// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal in the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; using System.Data; using System.IO; using System.Linq; using System.Text.Json; using System.Text.RegularExpressions; using System.Threading; using Duplicati.Library.Interface; using Duplicati.Library.Main.Database; using Duplicati.Library.SQLiteHelper; using Duplicati.Library.Utility; using RemoteSynchronization; #nullable enable namespace Duplicati.Library.Modules.Builtin; /// /// Trigger modes for remote synchronization. /// public enum RemoteSyncTriggerMode { /// /// Trigger after every successful backup. /// Inline, /// /// Trigger based on a time interval. /// Interval, /// /// Trigger after a certain number of backups. /// Counting } /// /// Configuration for a single remote synchronization destination. /// public record RemoteSyncDestinationConfig( RemoteSynchronization.Config Config, RemoteSyncTriggerMode Mode = RemoteSyncTriggerMode.Inline, TimeSpan? Interval = null, int Count = 0 ); /// /// Module for synchronizing backup data to remote destinations after a successful backup operation. /// public class RemoteSynchronizationModule : IGenericCallbackModule { private static readonly string LOGTAG = Logging.Log.LogTagFromType(); private const string OPTION_JSON_CONFIG = "remote-sync-json-config"; private string? m_dbpath; private List m_destinations = []; private bool m_enabled; private string? m_operationName; private string? m_source; private bool m_syncOnWarnings = true; // Default configuration for the remote synchronization runner private RemoteSynchronization.Config m_defaultRunnerConfig = new( Src: string.Empty, Dst: string.Empty, AutoCreateFolders: true, BackendRetries: 3, BackendRetryDelay: 1000, BackendRetryWithExponentialBackoff: true, Confirm: true, DryRun: false, DstOptions: [], Force: false, GlobalOptions: [], LogFile: string.Empty, LogLevel: "Information", ParseArgumentsOnly: false, Progress: false, Retention: false, Retry: 3, SrcOptions: [], VerifyContents: false, VerifyGetAfterPut: false ); /// /// Gets the key identifier for this module. /// public string Key => "remotesync"; /// /// Gets the display name for this module. /// public string DisplayName => Strings.RemoteSynchronization.DisplayName; /// /// Gets the description of this module. /// public string Description => Strings.RemoteSynchronization.Description; /// /// Gets whether this module should be loaded by default. /// public bool LoadAsDefault => true; /// /// Gets the list of supported command line arguments. /// public IList SupportedCommands => [ new CommandLineArgument(OPTION_JSON_CONFIG, CommandLineArgument.ArgumentType.String, "JSON configuration for remote synchronization", "JSON string or file path containing remote synchronization configuration"), ]; /// /// Gets a boolean value from the dictionary for the specified key. /// /// The dictionary to search. /// The key to look for in the dictionary. /// The default value to return if the key is not found or the value is not a boolean. /// The boolean value associated with the key, or the default value if not found or invalid. private static bool GetBoolFromDictionary(Dictionary dict, string key, bool defaultValue = false) { if (dict.TryGetValue(key, out var val)) { if (val is bool b) return b; else if (val is JsonElement elem && (elem.ValueKind == JsonValueKind.True || elem.ValueKind == JsonValueKind.False)) return elem.GetBoolean(); } return defaultValue; } /// /// Gets an integer value from the dictionary for the specified key. /// /// The dictionary to search. /// The key to look for in the dictionary. /// The default value to return if the key is not found or the value is not an integer. /// The integer value associated with the key, or the default value if not found or invalid. private static int GetIntFromDictionary(Dictionary dict, string key, int defaultValue = 0) { if (dict.TryGetValue(key, out var val)) { if (val is int l) return l; else if (val is JsonElement elem && elem.ValueKind == JsonValueKind.Number && elem.TryGetInt32(out var parsedInt)) return parsedInt; } return defaultValue; } /// /// Gets a list of strings from the dictionary for the specified key. /// /// The dictionary to search. /// The key to look for in the dictionary. /// The default list to return if the key is not found or the value is not a string. /// The list of strings associated with the key, or the default list if not found or invalid. private static List GetOptionsFromDictionary(Dictionary dict, string key, List defaultValue) { var options = GetStringFromDictionary(dict, key) .Split(' ') .Where(s => !string.IsNullOrWhiteSpace(s)) .ToList(); if (options.Count != 0) return [.. options]; return defaultValue; } /// /// Gets a string value from the dictionary for the specified key. /// /// The dictionary to search. /// The key to look for in the dictionary. /// The default value to return if the key is not found or the value is not a string. /// The string value associated with the key, or the default value if not found or invalid. private static string GetStringFromDictionary(Dictionary dict, string key, string defaultValue = "") { if (dict.TryGetValue(key, out var val)) { if (val is string str) return str; else if (val is JsonElement elem && elem.ValueKind == JsonValueKind.String) return elem.GetString() ?? defaultValue; } return defaultValue; } /// /// Configures the module with the provided command line options. /// /// The command line options dictionary. public void Configure(IDictionary commandlineOptions) { // Default is no valid JSON config provided, which disables the module m_enabled = false; m_destinations = []; if (commandlineOptions.TryGetValue("dbpath", out var dbpath)) m_dbpath = dbpath; // Check if JSON config is provided if (commandlineOptions.TryGetValue(OPTION_JSON_CONFIG, out var jsonConfigStr) && !string.IsNullOrWhiteSpace(jsonConfigStr)) { string jsonContent; if (jsonConfigStr.TrimStart().StartsWith("{")) { // It's a JSON string jsonContent = jsonConfigStr; } else { // It's a file path try { jsonContent = File.ReadAllText(jsonConfigStr); } catch (Exception ex) { Logging.Log.WriteErrorMessage(LOGTAG, "RemoteSyncJsonFileReadError", ex, "Failed to read JSON configuration file '{0}': {1}", jsonConfigStr, ex.Message); return; } } try { var deserializeOpts = new JsonSerializerOptions { PropertyNameCaseInsensitive = true }; var toplevel = JsonSerializer.Deserialize>(jsonContent, deserializeOpts); if (toplevel == null) { Logging.Log.WriteErrorMessage(LOGTAG, "RemoteSyncJsonParseError", null, "Failed to parse JSON configuration: top-level object is null."); return; } m_syncOnWarnings = GetBoolFromDictionary(toplevel, "sync-on-warnings", m_syncOnWarnings); if (toplevel?.TryGetValue("destinations", out var destinationsObj) == true && destinationsObj is JsonElement destinationsElem && destinationsElem.ValueKind == JsonValueKind.Array) { var destinations = JsonSerializer.Deserialize>>(destinationsElem.GetRawText(), deserializeOpts) ?? []; foreach (var destination in destinations) { var loglevel = GetStringFromDictionary(destination, "log-level"); var mode = GetStringFromDictionary(destination, "mode"); var interval = GetStringFromDictionary(destination, "interval"); m_destinations.Add(new( Config: m_defaultRunnerConfig with { Dst = GetStringFromDictionary(destination, "url", string.Empty), AutoCreateFolders = GetBoolFromDictionary(destination, "auto-create-folders", m_defaultRunnerConfig.AutoCreateFolders), BackendRetries = GetIntFromDictionary(destination, "backend-retries", m_defaultRunnerConfig.BackendRetries), BackendRetryDelay = GetIntFromDictionary(destination, "backend-retry-delay", m_defaultRunnerConfig.BackendRetryDelay), BackendRetryWithExponentialBackoff = GetBoolFromDictionary(destination, "backend-retry-with-exponential-backoff", m_defaultRunnerConfig.BackendRetryWithExponentialBackoff), Confirm = GetBoolFromDictionary(destination, "confirm", m_defaultRunnerConfig.Confirm), DryRun = GetBoolFromDictionary(destination, "dry-run", m_defaultRunnerConfig.DryRun), DstOptions = GetOptionsFromDictionary(destination, "dst-options", m_defaultRunnerConfig.DstOptions), Force = GetBoolFromDictionary(destination, "force", m_defaultRunnerConfig.Force), GlobalOptions = GetOptionsFromDictionary(destination, "global-options", m_defaultRunnerConfig.GlobalOptions), LogFile = GetStringFromDictionary(destination, "log-file"), LogLevel = loglevel ?? (commandlineOptions.TryGetValue("log-file-log-level", out var logLevel) ? logLevel : m_defaultRunnerConfig.LogLevel), ParseArgumentsOnly = GetBoolFromDictionary(destination, "parse-arguments-only", m_defaultRunnerConfig.ParseArgumentsOnly), Progress = GetBoolFromDictionary(destination, "progress", m_defaultRunnerConfig.Progress), Retention = GetBoolFromDictionary(destination, "retention", m_defaultRunnerConfig.Retention), Retry = GetIntFromDictionary(destination, "retry", m_defaultRunnerConfig.Retry), SrcOptions = GetOptionsFromDictionary(destination, "src-options", m_defaultRunnerConfig.SrcOptions), VerifyContents = GetBoolFromDictionary(destination, "verify-contents", m_defaultRunnerConfig.VerifyContents), VerifyGetAfterPut = GetBoolFromDictionary(destination, "verify-get-after-put", m_defaultRunnerConfig.VerifyGetAfterPut) }, Mode: Enum.TryParse(mode, true, out var parsedMode) ? parsedMode : RemoteSyncTriggerMode.Inline, Interval: string.IsNullOrWhiteSpace(interval) ? null : Duplicati.Library.Utility.Timeparser.ParseTimeSpan(interval), Count: GetIntFromDictionary(destination, "count", 0) )); } m_enabled = true; } else { Logging.Log.WriteErrorMessage(LOGTAG, "RemoteSyncJsonMissingDestinations", null, "JSON configuration is missing 'destinations' array."); m_enabled = false; return; } } catch (Exception ex) { Logging.Log.WriteErrorMessage(LOGTAG, "RemoteSyncJsonParseError", ex, "Failed to parse JSON configuration: {0}", ex.Message); } } } /// /// Called when an operation starts. /// /// The name of the operation. /// The remote URL. /// The local paths. public void OnStart(string operationname, ref string remoteurl, ref string[] localpath) { if (!m_enabled) return; m_operationName = operationname; if (string.IsNullOrWhiteSpace(m_source)) m_source = remoteurl; } /// /// Called when an operation finishes. /// /// The results of the operation. /// Any exception that occurred during the operation. public void OnFinish(IBasicResults result, Exception exception) { if (!m_enabled) return; if (!string.Equals(m_operationName, "Backup", StringComparison.OrdinalIgnoreCase)) return; if (exception != null) { Logging.Log.WriteWarningMessage(LOGTAG, "RemoteSyncSkipped", exception, "Remote synchronization skipped due to operation failure."); return; } if (result != null && (result.ParsedResult == ParsedResultType.Error || result.ParsedResult == ParsedResultType.Fatal)) { Logging.Log.WriteWarningMessage(LOGTAG, "RemoteSyncSkipped", null, "Remote synchronization skipped because backup reported errors."); return; } if (result != null && result.ParsedResult == ParsedResultType.Warning && !m_syncOnWarnings) { Logging.Log.WriteInformationMessage(LOGTAG, "RemoteSyncSkipped", "Remote synchronization skipped because backup reported warnings and sync on warnings is disabled."); return; } if (string.IsNullOrWhiteSpace(m_source)) { Logging.Log.WriteWarningMessage(LOGTAG, "RemoteSyncMissingBackends", null, "Remote synchronization skipped because source is missing."); return; } if (m_destinations.Count == 0) { Logging.Log.WriteWarningMessage(LOGTAG, "RemoteSyncNoDestinations", null, "Remote synchronization skipped because no destinations are configured."); return; } for (int i = 0; i < m_destinations.Count; i++) { var dest = m_destinations[i]; if (string.IsNullOrWhiteSpace(dest.Config.Dst)) continue; if (!ShouldTriggerSync(i, dest)) { Logging.Log.WriteInformationMessage(LOGTAG, "RemoteSyncSkipped", "Remote synchronization to {0} skipped due to trigger mode conditions not met.", dest); continue; } try { var config = dest.Config with { Src = m_source! }; var exitCode = RemoteSynchronizationRunner.Run(config, CancellationToken.None).ConfigureAwait(false).GetAwaiter().GetResult(); if (exitCode != 0) Logging.Log.WriteErrorMessage(LOGTAG, "RemoteSyncFailed", null, "Remote synchronization to {0} failed with exit code {1}.", dest, exitCode); RecordSyncOperation(i); } catch (Exception ex) { Logging.Log.WriteErrorMessage(LOGTAG, "RemoteSyncFailed", ex, "Remote synchronization to {0} failed: {1}", dest, ex.Message); } } } /// /// Checks if remote synchronization should be triggered for the specified destination based on the configured mode. /// /// The index of the destination in the list. /// True if synchronization should be triggered. private bool ShouldTriggerSync(int index, RemoteSyncDestinationConfig dest) { if (index < 0 || index >= m_destinations.Count) return false; var description = $"Rsync {index}"; switch (dest.Mode) { case RemoteSyncTriggerMode.Inline: return true; case RemoteSyncTriggerMode.Interval: { using var db = SQLiteLoader.LoadConnection(m_dbpath!); using var cmd = db.CreateCommand(); cmd.CommandText = @" SELECT ""Timestamp"" FROM ""Operation"" WHERE ""Description"" = @description ORDER BY ""Timestamp"" DESC LIMIT 1 "; cmd.AddNamedParameter("@description", description); var lastSync = cmd.ExecuteScalar(); if (lastSync is null) return true; var lastSyncTime = Utility.Utility.EPOCH.AddSeconds((long)lastSync); var now = DateTime.UtcNow; return (now - lastSyncTime) >= dest.Interval; } case RemoteSyncTriggerMode.Counting: { using var db = SQLiteLoader.LoadConnection(m_dbpath!); using var cmd = db.CreateCommand(); cmd.CommandText = @" SELECT COUNT(*) FROM ""Operation"" WHERE ""Description"" = @description "; cmd.AddNamedParameter("@description", description); var syncCount = (long)(cmd.ExecuteScalar() ?? 0L); if (syncCount == 0) return true; cmd.Parameters.Clear(); cmd.CommandText = @" SELECT COUNT(*) FROM ""Operation"" WHERE ""Description"" = 'Backup' AND ""Timestamp"" > COALESCE( ( SELECT ""Timestamp"" FROM ""Operation"" WHERE ""Description"" = @description ORDER BY ""Timestamp"" DESC LIMIT 1 ), 0 )"; cmd.AddNamedParameter("@description", description); var backupCount = (long)(cmd.ExecuteScalar() ?? 0L); return backupCount >= dest.Count; } default: return false; } } /// /// Records a remote synchronization operation in the database. /// /// The index of the destination. private void RecordSyncOperation(int index) { // Validate index if (index < 0 || index >= m_destinations.Count) { Logging.Log.WriteWarningMessage(LOGTAG, "RemoteSyncRecordInvalidIndex", null, "Cannot record remote synchronization operation: invalid index {0}.", index); return; } if (string.IsNullOrWhiteSpace(m_dbpath)) return; using var db = SQLiteLoader.LoadConnection(m_dbpath!); using var transaction = db.BeginTransaction(); using var cmd = db.CreateCommand(); cmd.CommandText = @" INSERT INTO ""Operation"" ( ""Description"", ""Timestamp"" ) VALUES ( @description, @timestamp )"; cmd.SetTransaction(transaction); cmd.AddNamedParameter("@description", $"Rsync {index}"); cmd.AddNamedParameter("@timestamp", Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow)); cmd.ExecuteNonQuery(); transaction.Commit(); } }