// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal in the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; using System.Data; using System.IO; using System.Linq; using System.Text.Json; using System.Text.RegularExpressions; using System.Threading; using Duplicati.Library.Interface; using Duplicati.Library.Main.Database; using Duplicati.Library.SQLiteHelper; using Duplicati.Library.Utility; using RemoteSynchronization; #nullable enable namespace Duplicati.Library.Modules.Builtin; /// /// Trigger modes for remote synchronization. /// public enum RemoteSyncTriggerMode { /// /// Trigger after every successful backup. /// Inline, /// /// Trigger based on a schedule. /// Scheduled, /// /// Trigger after a certain number of backups. /// Counting } /// /// Configuration for a single remote synchronization destination. /// public record RemoteSyncDestinationConfig( RemoteSynchronization.Config Config, RemoteSyncTriggerMode Mode = RemoteSyncTriggerMode.Inline, TimeSpan? Schedule = null, int Count = 0 ); /// /// Module for synchronizing backup data to remote destinations after a successful backup operation. /// public class RemoteSynchronizationModule : IGenericCallbackModule { private static readonly string LOGTAG = Logging.Log.LogTagFromType(); private const string OPTION_JSON_CONFIG = "remote-sync-json-config"; private string? m_dbpath; private List m_destinations = []; private bool m_enabled; private string? m_operationName; private string? m_source; private bool m_syncOnWarnings = true; // Default configuration for the remote synchronization runner private RemoteSynchronization.Config m_defaultRunnerConfig = new( Src: string.Empty, Dst: string.Empty, AutoCreateFolders: true, BackendRetries: 3, BackendRetryDelay: 1000, BackendRetryWithExponentialBackoff: true, Confirm: true, DryRun: false, DstOptions: [], Force: false, GlobalOptions: [], LogFile: string.Empty, LogLevel: "Information", ParseArgumentsOnly: false, Progress: false, Retention: false, Retry: 3, SrcOptions: [], VerifyContents: false, VerifyGetAfterPut: false ); /// /// Gets the key identifier for this module. /// public string Key => "remotesync"; /// /// Gets the display name for this module. /// public string DisplayName => Strings.RemoteSynchronization.DisplayName; /// /// Gets the description of this module. /// public string Description => Strings.RemoteSynchronization.Description; /// /// Gets whether this module should be loaded by default. /// public bool LoadAsDefault => true; /// /// Gets the list of supported command line arguments. /// public IList SupportedCommands => [ new CommandLineArgument(OPTION_JSON_CONFIG, CommandLineArgument.ArgumentType.String, "JSON configuration for remote synchronization", "JSON string or file path containing remote synchronization configuration"), ]; /// /// Configures the module with the provided command line options. /// /// The command line options dictionary. public void Configure(IDictionary commandlineOptions) { // Default is no valid JSON config provided, which disables the module m_enabled = false; m_destinations = []; if (commandlineOptions.TryGetValue("dbpath", out var dbpath)) m_dbpath = dbpath; // Check if JSON config is provided if (commandlineOptions.TryGetValue(OPTION_JSON_CONFIG, out var jsonConfigStr) && !string.IsNullOrWhiteSpace(jsonConfigStr)) { string jsonContent; if (jsonConfigStr.TrimStart().StartsWith("{")) { // It's a JSON string jsonContent = jsonConfigStr; } else { // It's a file path try { jsonContent = File.ReadAllText(jsonConfigStr); } catch (Exception ex) { Logging.Log.WriteErrorMessage(LOGTAG, "RemoteSyncJsonFileReadError", ex, "Failed to read JSON configuration file '{0}': {1}", jsonConfigStr, ex.Message); return; } } try { var deserializeOpts = new JsonSerializerOptions { PropertyNameCaseInsensitive = true }; var toplevel = JsonSerializer.Deserialize>(jsonContent, deserializeOpts); if (toplevel?.TryGetValue("sync-on-warnings", out var syncOnWarningsObj) == true) { if (syncOnWarningsObj is JsonElement elem && elem.ValueKind == JsonValueKind.True) m_syncOnWarnings = true; else if (syncOnWarningsObj is JsonElement elem2 && elem2.ValueKind == JsonValueKind.False) m_syncOnWarnings = false; else if (syncOnWarningsObj is bool b) m_syncOnWarnings = b; } if (toplevel?.TryGetValue("destinations", out var destinationsObj) == true && destinationsObj is JsonElement destinationsElem && destinationsElem.ValueKind == JsonValueKind.Array) { var destinations = JsonSerializer.Deserialize>>(destinationsElem.GetRawText(), deserializeOpts) ?? []; foreach (var destination in destinations) { m_destinations.Add(new( Config: m_defaultRunnerConfig with { Dst = (destination.GetValueOrDefault("url") as string) ?? (destination.GetValueOrDefault("url") is JsonElement elem && elem.ValueKind == JsonValueKind.String ? elem.GetString() : null) ?? string.Empty, AutoCreateFolders = destination.TryGetValue("auto-create-folders", out var autoCreateFoldersObj) && autoCreateFoldersObj is bool autoCreateFolders ? autoCreateFolders : m_defaultRunnerConfig.AutoCreateFolders, BackendRetries = destination.TryGetValue("backend-retries", out var backendRetriesObj) && backendRetriesObj is long backendRetriesLong ? (int)backendRetriesLong : m_defaultRunnerConfig.BackendRetries, BackendRetryDelay = destination.TryGetValue("backend-retry-delay", out var backendRetryDelayObj) && backendRetryDelayObj is long backendRetryDelayLong ? (int)backendRetryDelayLong : m_defaultRunnerConfig.BackendRetryDelay, BackendRetryWithExponentialBackoff = destination.TryGetValue("backend-retry-with-exponential-backoff", out var backendRetryWithExponentialBackoffObj) && backendRetryWithExponentialBackoffObj is bool backendRetryWithExponentialBackoff ? backendRetryWithExponentialBackoff : m_defaultRunnerConfig.BackendRetryWithExponentialBackoff, Confirm = destination.TryGetValue("confirm", out var confirmObj) && confirmObj is bool confirm ? confirm : m_defaultRunnerConfig.Confirm, DryRun = destination.TryGetValue("dry-run", out var dryRunObj) && dryRunObj is bool dryRun ? dryRun : m_defaultRunnerConfig.DryRun, DstOptions = destination.TryGetValue("dst-options", out var dstOptionsObj) && ((dstOptionsObj is string dstOptionsStr) || (dstOptionsObj is JsonElement elemDstOpt && elemDstOpt.ValueKind == JsonValueKind.String && (dstOptionsStr = elemDstOpt.GetString()) != null)) ? dstOptionsStr.Split(' ').ToList() : m_defaultRunnerConfig.DstOptions, Force = destination.TryGetValue("force", out var forceObj) && forceObj is bool force ? force : m_defaultRunnerConfig.Force, GlobalOptions = destination.TryGetValue("global-options", out var globalOptionsObj) && ((globalOptionsObj is string globalOptionsStr) || (globalOptionsObj is JsonElement elemGlob && elemGlob.ValueKind == JsonValueKind.String && (globalOptionsStr = elemGlob.GetString()) != null)) ? globalOptionsStr.Split(' ').ToList() : m_defaultRunnerConfig.GlobalOptions, LogFile = (destination.GetValueOrDefault("log-file") as string) ?? (destination.GetValueOrDefault("log-file") is JsonElement elemLog && elemLog.ValueKind == JsonValueKind.String ? elemLog.GetString() : null) ?? (commandlineOptions.TryGetValue("log-file", out var logFile) ? logFile : m_defaultRunnerConfig.LogFile), LogLevel = (destination.GetValueOrDefault("log-level") as string) ?? (destination.GetValueOrDefault("log-level") is JsonElement elemLvl && elemLvl.ValueKind == JsonValueKind.String ? elemLvl.GetString() : null) ?? (commandlineOptions.TryGetValue("log-file-log-level", out var logLevel) ? logLevel : m_defaultRunnerConfig.LogLevel), ParseArgumentsOnly = destination.TryGetValue("parse-arguments-only", out var parseArgumentsOnlyObj) && parseArgumentsOnlyObj is bool parseArgumentsOnly ? parseArgumentsOnly : m_defaultRunnerConfig.ParseArgumentsOnly, Progress = destination.TryGetValue("progress", out var progressObj) && progressObj is bool progress ? progress : m_defaultRunnerConfig.Progress, Retention = destination.TryGetValue("retention", out var retentionObj) && retentionObj is bool retention ? retention : m_defaultRunnerConfig.Retention, Retry = destination.TryGetValue("retry", out var retryObj) && retryObj is long retryLong ? (int)retryLong : m_defaultRunnerConfig.Retry, SrcOptions = destination.TryGetValue("src-options", out var srcOptionsObj) && ((srcOptionsObj is string srcOptionsStr) || (srcOptionsObj is JsonElement elemSrc && elemSrc.ValueKind == JsonValueKind.String && (srcOptionsStr = elemSrc.GetString()) != null)) ? srcOptionsStr.Split(' ').ToList() : m_defaultRunnerConfig.SrcOptions, VerifyContents = destination.TryGetValue("verify-contents", out var verifyContentsObj) && verifyContentsObj is bool verifyContents ? verifyContents : m_defaultRunnerConfig.VerifyContents, VerifyGetAfterPut = destination.TryGetValue("verify-get-after-put", out var verifyGetAfterPutObj) && verifyGetAfterPutObj is bool verifyGetAfterPut ? verifyGetAfterPut : m_defaultRunnerConfig.VerifyGetAfterPut }, Mode: Enum.TryParse((destination.GetValueOrDefault("mode") as string) ?? (destination.GetValueOrDefault("mode") is JsonElement elemMode && elemMode.ValueKind == JsonValueKind.String ? elemMode.GetString() : null), true, out var mode) ? mode : RemoteSyncTriggerMode.Inline, Schedule: destination.TryGetValue("schedule", out var scheduleObj) && ((scheduleObj is string scheduleStr) || (scheduleObj is JsonElement elemSch && elemSch.ValueKind == JsonValueKind.String && (scheduleStr = elemSch.GetString()) != null)) && TimeSpan.TryParse(scheduleStr, out var schedule) ? schedule : null, Count: destination.TryGetValue("count", out var countObj) && ((countObj is long countLong) || (countObj is JsonElement elemCnt && elemCnt.ValueKind == JsonValueKind.Number && elemCnt.TryGetInt64(out countLong))) ? (int)countLong : 0 )); } m_enabled = true; } else { Logging.Log.WriteErrorMessage(LOGTAG, "RemoteSyncJsonMissingDestinations", null, "JSON configuration is missing 'destinations' array."); m_enabled = false; return; } } catch (Exception ex) { Logging.Log.WriteErrorMessage(LOGTAG, "RemoteSyncJsonParseError", ex, "Failed to parse JSON configuration: {0}", ex.Message); } } } /// /// Called when an operation starts. /// /// The name of the operation. /// The remote URL. /// The local paths. public void OnStart(string operationname, ref string remoteurl, ref string[] localpath) { if (!m_enabled) return; m_operationName = operationname; if (string.IsNullOrWhiteSpace(m_source)) m_source = remoteurl; } /// /// Called when an operation finishes. /// /// The results of the operation. /// Any exception that occurred during the operation. public void OnFinish(IBasicResults result, Exception exception) { if (!m_enabled) return; if (!string.Equals(m_operationName, "Backup", StringComparison.OrdinalIgnoreCase)) return; if (exception != null) { Logging.Log.WriteWarningMessage(LOGTAG, "RemoteSyncSkipped", exception, "Remote synchronization skipped due to operation failure."); return; } if (result != null && (result.ParsedResult == ParsedResultType.Error || result.ParsedResult == ParsedResultType.Fatal)) { Logging.Log.WriteWarningMessage(LOGTAG, "RemoteSyncSkipped", null, "Remote synchronization skipped because backup reported errors."); return; } if (result != null && result.ParsedResult == ParsedResultType.Warning && !m_syncOnWarnings) { Logging.Log.WriteInformationMessage(LOGTAG, "RemoteSyncSkipped", "Remote synchronization skipped because backup reported warnings and sync on warnings is disabled."); return; } if (string.IsNullOrWhiteSpace(m_source)) { Logging.Log.WriteWarningMessage(LOGTAG, "RemoteSyncMissingBackends", null, "Remote synchronization skipped because source is missing."); return; } if (m_destinations.Count == 0) { Logging.Log.WriteWarningMessage(LOGTAG, "RemoteSyncNoDestinations", null, "Remote synchronization skipped because no destinations are configured."); return; } for (int i = 0; i < m_destinations.Count; i++) { var dest = m_destinations[i]; if (string.IsNullOrWhiteSpace(dest.Config.Dst)) continue; if (!ShouldTriggerSync(i, dest)) { Logging.Log.WriteInformationMessage(LOGTAG, "RemoteSyncSkipped", "Remote synchronization to {0} skipped due to trigger mode conditions not met.", dest); continue; } RecordSyncOperation(i); try { var config = dest.Config with { Src = m_source! }; var exitCode = RemoteSynchronizationRunner.Run(config, CancellationToken.None).ConfigureAwait(false).GetAwaiter().GetResult(); if (exitCode != 0) Logging.Log.WriteErrorMessage(LOGTAG, "RemoteSyncFailed", null, "Remote synchronization to {0} failed with exit code {1}.", dest, exitCode); } catch (Exception ex) { Logging.Log.WriteErrorMessage(LOGTAG, "RemoteSyncFailed", ex, "Remote synchronization to {0} failed: {1}", dest, ex.Message); } } } /// /// Checks if remote synchronization should be triggered for the specified destination based on the configured mode. /// /// The index of the destination in the list. /// True if synchronization should be triggered. private bool ShouldTriggerSync(int index, RemoteSyncDestinationConfig dest) { if (index < 0 || index >= m_destinations.Count) return false; var description = $"Rsync {index}"; switch (dest.Mode) { case RemoteSyncTriggerMode.Inline: return true; case RemoteSyncTriggerMode.Scheduled: { using var db = SQLiteLoader.LoadConnection(m_dbpath!); using var cmd = db.CreateCommand(); cmd.CommandText = @" SELECT ""Timestamp"" FROM ""Operation"" WHERE ""Description"" = @description ORDER BY ""Timestamp"" DESC LIMIT 1 "; cmd.AddNamedParameter("@description", description); var lastSync = cmd.ExecuteScalar(); if (lastSync is null) return true; var lastSyncTime = Utility.Utility.EPOCH.AddSeconds((long)lastSync); var now = DateTime.UtcNow; return (now - lastSyncTime) >= dest.Schedule; } case RemoteSyncTriggerMode.Counting: { using var db = SQLiteLoader.LoadConnection(m_dbpath!); using var cmd = db.CreateCommand(); cmd.CommandText = @" SELECT COUNT(*) FROM ""Operation"" WHERE ""Description"" = 'Backup' AND ""Timestamp"" > COALESCE( ( SELECT ""Timestamp"" FROM ""Operation"" WHERE ""Description"" = @description ORDER BY ""Timestamp"" DESC LIMIT 1 ), 0 )"; cmd.AddNamedParameter("@description", description); var backupCount = (long)(cmd.ExecuteScalar() ?? 0L); return backupCount >= dest.Count; } default: return false; } } /// /// Records a remote synchronization operation in the database. /// /// The index of the destination. private void RecordSyncOperation(int index) { // Validate index if (index < 0 || index >= m_destinations.Count) { Logging.Log.WriteWarningMessage(LOGTAG, "RemoteSyncRecordInvalidIndex", null, "Cannot record remote synchronization operation: invalid index {0}.", index); return; } if (string.IsNullOrWhiteSpace(m_dbpath)) return; using var db = SQLiteLoader.LoadConnection(m_dbpath!); using var transaction = db.BeginTransaction(); using var cmd = db.CreateCommand(); cmd.CommandText = @" INSERT INTO ""Operation"" ( ""Description"", ""Timestamp"" ) VALUES ( @description, @timestamp )"; cmd.SetTransaction(transaction); cmd.AddNamedParameter("@description", $"Rsync {index}"); cmd.AddNamedParameter("@timestamp", Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow)); cmd.ExecuteNonQuery(); transaction.Commit(); } }