From 52b38579b739066124fab47e4eead8eed3d8baea Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 8 Jan 2026 14:44:26 +0100 Subject: [PATCH] Added remote synchronization as a module --- .../Library/Modules/Builtin/GenericModules.cs | 1 + .../Builtin/RemoteSynchronizationModule.cs | 240 ++++++++++++++++++ Duplicati/Library/Modules/Builtin/Strings.cs | 45 ++++ 3 files changed, 286 insertions(+) create mode 100644 Duplicati/Library/Modules/Builtin/RemoteSynchronizationModule.cs diff --git a/Duplicati/Library/Modules/Builtin/GenericModules.cs b/Duplicati/Library/Modules/Builtin/GenericModules.cs index 79175c825..60e68216e 100644 --- a/Duplicati/Library/Modules/Builtin/GenericModules.cs +++ b/Duplicati/Library/Modules/Builtin/GenericModules.cs @@ -34,6 +34,7 @@ public static class GenericModules new CommonOptions(), new HyperVOptions(), new MSSQLOptions(), + new RemoteSynchronizationModule(), new RunScript(), new SendHttpMessage(), new SendJabberMessage(), diff --git a/Duplicati/Library/Modules/Builtin/RemoteSynchronizationModule.cs b/Duplicati/Library/Modules/Builtin/RemoteSynchronizationModule.cs new file mode 100644 index 000000000..949ed7692 --- /dev/null +++ b/Duplicati/Library/Modules/Builtin/RemoteSynchronizationModule.cs @@ -0,0 +1,240 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using Duplicati.Library.Interface; +using Duplicati.Library.Utility; +using RemoteSynchronization; + +namespace Duplicati.Library.Modules.Builtin; + +public class RemoteSynchronizationModule : IGenericCallbackModule +{ + private static readonly string LOGTAG = Logging.Log.LogTagFromType(); + + private static readonly Regex ARGREGEX = new Regex( + @"(?(?<=\s|^)(""(?[^""\\]*(?:\\.[^""\\]*)*)""|'(?[^'\\]*(?:\\.[^'\\]*)*)'|(?[^\s]+))\s?)", + RegexOptions.Compiled | RegexOptions.ExplicitCapture + ); + + private const string OPTION_BACKEND_SRC = "remote-sync-src"; + private const string OPTION_BACKEND_DST = "remote-sync-dst"; + private const string OPTION_AUTO_CREATE_FOLDERS = "remote-sync-auto-create-folders"; + private const string OPTION_BACKEND_RETRIES = "remote-sync-backend-retries"; + private const string OPTION_BACKEND_RETRY_DELAY = "remote-sync-backend-retry-delay"; + private const string OPTION_BACKEND_RETRY_BACKOFF = "remote-sync-backend-retry-with-exponential-backoff"; + private const string OPTION_CONFIRM = "remote-sync-confirm"; + private const string OPTION_DRY_RUN = "remote-sync-dry-run"; + private const string OPTION_DST_OPTIONS = "remote-sync-dst-options"; + private const string OPTION_FORCE = "remote-sync-force"; + private const string OPTION_GLOBAL_OPTIONS = "remote-sync-global-options"; + private const string OPTION_LOG_FILE = "remote-sync-log-file"; + private const string OPTION_LOG_LEVEL = "remote-sync-log-level"; + private const string OPTION_PARSE_ARGUMENTS_ONLY = "remote-sync-parse-arguments-only"; + private const string OPTION_PROGRESS = "remote-sync-progress"; + private const string OPTION_RETENTION = "remote-sync-retention"; + private const string OPTION_RETRY = "remote-sync-retry"; + private const string OPTION_SRC_OPTIONS = "remote-sync-src-options"; + private const string OPTION_VERIFY_CONTENTS = "remote-sync-verify-contents"; + private const string OPTION_VERIFY_GET_AFTER_PUT = "remote-sync-verify-get-after-put"; + + private IReadOnlyDictionary m_options = new Dictionary(); + private string m_source; + private string m_destination; + private string m_operationName; + private string m_remoteUrl; + private bool m_enabled; + + public string Key => "remotesync"; + public string DisplayName => Strings.RemoteSynchronization.DisplayName; + public string Description => Strings.RemoteSynchronization.Description; + public bool LoadAsDefault => true; + + public IList SupportedCommands => + [ + new CommandLineArgument(OPTION_BACKEND_SRC, CommandLineArgument.ArgumentType.String, Strings.RemoteSynchronization.BackendSourceShort, Strings.RemoteSynchronization.BackendSourceLong), + new CommandLineArgument(OPTION_BACKEND_DST, CommandLineArgument.ArgumentType.String, Strings.RemoteSynchronization.BackendDestinationShort, Strings.RemoteSynchronization.BackendDestinationLong), + new CommandLineArgument(OPTION_AUTO_CREATE_FOLDERS, CommandLineArgument.ArgumentType.Boolean, Strings.RemoteSynchronization.AutoCreateFoldersShort, Strings.RemoteSynchronization.AutoCreateFoldersLong, "false"), + new CommandLineArgument(OPTION_BACKEND_RETRIES, CommandLineArgument.ArgumentType.Integer, Strings.RemoteSynchronization.BackendRetriesShort, Strings.RemoteSynchronization.BackendRetriesLong, "3"), + new CommandLineArgument(OPTION_BACKEND_RETRY_DELAY, CommandLineArgument.ArgumentType.Integer, Strings.RemoteSynchronization.BackendRetryDelayShort, Strings.RemoteSynchronization.BackendRetryDelayLong, "1000"), + new CommandLineArgument(OPTION_BACKEND_RETRY_BACKOFF, CommandLineArgument.ArgumentType.Boolean, Strings.RemoteSynchronization.BackendRetryBackoffShort, Strings.RemoteSynchronization.BackendRetryBackoffLong, "false"), + new CommandLineArgument(OPTION_CONFIRM, CommandLineArgument.ArgumentType.Boolean, Strings.RemoteSynchronization.ConfirmShort, Strings.RemoteSynchronization.ConfirmLong, "false"), + new CommandLineArgument(OPTION_DRY_RUN, CommandLineArgument.ArgumentType.Boolean, Strings.RemoteSynchronization.DryRunShort, Strings.RemoteSynchronization.DryRunLong, "false"), + new CommandLineArgument(OPTION_DST_OPTIONS, CommandLineArgument.ArgumentType.String, Strings.RemoteSynchronization.DestinationOptionsShort, Strings.RemoteSynchronization.DestinationOptionsLong), + new CommandLineArgument(OPTION_FORCE, CommandLineArgument.ArgumentType.Boolean, Strings.RemoteSynchronization.ForceShort, Strings.RemoteSynchronization.ForceLong, "false"), + new CommandLineArgument(OPTION_GLOBAL_OPTIONS, CommandLineArgument.ArgumentType.String, Strings.RemoteSynchronization.GlobalOptionsShort, Strings.RemoteSynchronization.GlobalOptionsLong), + new CommandLineArgument(OPTION_LOG_FILE, CommandLineArgument.ArgumentType.Path, Strings.RemoteSynchronization.LogFileShort, Strings.RemoteSynchronization.LogFileLong, string.Empty), + new CommandLineArgument(OPTION_LOG_LEVEL, CommandLineArgument.ArgumentType.String, Strings.RemoteSynchronization.LogLevelShort, Strings.RemoteSynchronization.LogLevelLong, "Information"), + new CommandLineArgument(OPTION_PARSE_ARGUMENTS_ONLY, CommandLineArgument.ArgumentType.Boolean, Strings.RemoteSynchronization.ParseArgumentsOnlyShort, Strings.RemoteSynchronization.ParseArgumentsOnlyLong, "false"), + new CommandLineArgument(OPTION_PROGRESS, CommandLineArgument.ArgumentType.Boolean, Strings.RemoteSynchronization.ProgressShort, Strings.RemoteSynchronization.ProgressLong, "false"), + new CommandLineArgument(OPTION_RETENTION, CommandLineArgument.ArgumentType.Boolean, Strings.RemoteSynchronization.RetentionShort, Strings.RemoteSynchronization.RetentionLong, "false"), + new CommandLineArgument(OPTION_RETRY, CommandLineArgument.ArgumentType.Integer, Strings.RemoteSynchronization.RetryShort, Strings.RemoteSynchronization.RetryLong, "3"), + new CommandLineArgument(OPTION_SRC_OPTIONS, CommandLineArgument.ArgumentType.String, Strings.RemoteSynchronization.SourceOptionsShort, Strings.RemoteSynchronization.SourceOptionsLong), + new CommandLineArgument(OPTION_VERIFY_CONTENTS, CommandLineArgument.ArgumentType.Boolean, Strings.RemoteSynchronization.VerifyContentsShort, Strings.RemoteSynchronization.VerifyContentsLong, "false"), + new CommandLineArgument(OPTION_VERIFY_GET_AFTER_PUT, CommandLineArgument.ArgumentType.Boolean, Strings.RemoteSynchronization.VerifyGetAfterPutShort, Strings.RemoteSynchronization.VerifyGetAfterPutLong, "false"), + ]; + + public void Configure(IDictionary commandlineOptions) + { + m_options = commandlineOptions.AsReadOnly(); + commandlineOptions.TryGetValue(OPTION_BACKEND_SRC, out m_source); + commandlineOptions.TryGetValue(OPTION_BACKEND_DST, out m_destination); + m_enabled = !string.IsNullOrWhiteSpace(m_destination); + } + + public void OnStart(string operationname, ref string remoteurl, ref string[] localpath) + { + if (!m_enabled) + return; + + m_operationName = operationname; + m_remoteUrl = remoteurl; + + if (string.IsNullOrWhiteSpace(m_source)) + m_source = remoteurl; + } + + public void OnFinish(IBasicResults result, Exception exception) + { + if (!m_enabled) + return; + + if (!string.Equals(m_operationName, "Backup", StringComparison.OrdinalIgnoreCase)) + return; + + if (exception != null) + { + Logging.Log.WriteWarningMessage(LOGTAG, "RemoteSyncSkipped", exception, "Remote synchronization skipped due to operation failure."); + return; + } + + if (result != null && (result.ParsedResult == ParsedResultType.Error || result.ParsedResult == ParsedResultType.Fatal)) + { + Logging.Log.WriteWarningMessage(LOGTAG, "RemoteSyncSkipped", null, "Remote synchronization skipped because backup reported errors."); + return; + } + + if (string.IsNullOrWhiteSpace(m_source) || string.IsNullOrWhiteSpace(m_destination)) + { + Logging.Log.WriteWarningMessage(LOGTAG, "RemoteSyncMissingBackends", null, "Remote synchronization skipped because source or destination is missing."); + return; + } + + var args = BuildArguments(); + try + { + var exitCode = RemoteSynchronizationRunner.RunAsync(args).ConfigureAwait(false).GetAwaiter().GetResult(); + if (exitCode != 0) + Logging.Log.WriteErrorMessage(LOGTAG, "RemoteSyncFailed", null, "Remote synchronization failed with exit code {0}.", exitCode); + } + catch (Exception ex) + { + Logging.Log.WriteErrorMessage(LOGTAG, "RemoteSyncFailed", ex, "Remote synchronization failed: {0}", ex.Message); + } + } + + private string[] BuildArguments() + { + var args = new List + { + m_source, + m_destination + }; + + AddBoolOptionWithDefaultTrue(OPTION_AUTO_CREATE_FOLDERS, "--auto-create-folders", args); + AddOption(OPTION_BACKEND_RETRIES, "--backend-retries", args); + AddOption(OPTION_BACKEND_RETRY_DELAY, "--backend-retry-delay", args); + AddBoolOptionWithDefaultTrue(OPTION_BACKEND_RETRY_BACKOFF, "--backend-retry-with-exponential-backoff", args); + AddBoolOptionWithDefaultTrue(OPTION_CONFIRM, "--confirm", args); + AddBoolOption(OPTION_DRY_RUN, "--dry-run", args); + AddMultiValueOption(OPTION_DST_OPTIONS, "--dst-options", args); + AddBoolOption(OPTION_FORCE, "--force", args); + AddMultiValueOption(OPTION_GLOBAL_OPTIONS, "--global-options", args); + AddOption(OPTION_LOG_FILE, "--log-file", args); + AddOption(OPTION_LOG_LEVEL, "--log-level", args); + AddBoolOption(OPTION_PARSE_ARGUMENTS_ONLY, "--parse-arguments-only", args); + AddBoolOption(OPTION_PROGRESS, "--progress", args); + AddBoolOption(OPTION_RETENTION, "--retention", args); + AddOption(OPTION_RETRY, "--retry", args); + AddMultiValueOption(OPTION_SRC_OPTIONS, "--src-options", args); + AddBoolOption(OPTION_VERIFY_CONTENTS, "--verify-contents", args); + AddBoolOption(OPTION_VERIFY_GET_AFTER_PUT, "--verify-get-after-put", args); + + return args.ToArray(); + } + + private void AddBoolOptionWithDefaultTrue(string optionKey, string toolOption, List args) + { + if (m_options.ContainsKey(optionKey)) + AddBoolOption(optionKey, toolOption, args); + else + args.Add(toolOption); + } + + private void AddBoolOption(string optionKey, string toolOption, List args) + { + if (!m_options.TryGetValue(optionKey, out var value)) + return; + + args.Add(toolOption); + if (!string.IsNullOrWhiteSpace(value)) + args.Add(value); + } + + private void AddOption(string optionKey, string toolOption, List args) + { + if (!m_options.TryGetValue(optionKey, out var value)) + return; + + if (string.IsNullOrWhiteSpace(value)) + return; + + args.Add(toolOption); + args.Add(value); + } + + private void AddMultiValueOption(string optionKey, string toolOption, List args) + { + if (!m_options.TryGetValue(optionKey, out var value)) + return; + + var tokens = SplitArguments(value); + if (tokens.Count == 0) + return; + + args.Add(toolOption); + args.AddRange(tokens); + } + + private static List SplitArguments(string value) + { + if (string.IsNullOrWhiteSpace(value)) + return new List(); + + return ARGREGEX.Matches(value) + .Select(match => match.Groups["value"].Value) + .Where(token => !string.IsNullOrWhiteSpace(token)) + .ToList(); + } +} diff --git a/Duplicati/Library/Modules/Builtin/Strings.cs b/Duplicati/Library/Modules/Builtin/Strings.cs index a3ac504ee..399f5662d 100644 --- a/Duplicati/Library/Modules/Builtin/Strings.cs +++ b/Duplicati/Library/Modules/Builtin/Strings.cs @@ -53,6 +53,51 @@ namespace Duplicati.Library.Modules.Builtin.Strings public static string Description { get { return LC.L(@"This module works internaly to parse source parameters to backup Microsoft SQL Server databases"); } } public static string DisplayName { get { return LC.L(@"Configure Microsoft SQL Server module"); } } } + internal static class RemoteSynchronization + { + public static string Description { get { return LC.L(@"This module runs remote synchronization after a successful backup to create a secondary copy (3-2-1)."); } } + public static string DisplayName { get { return LC.L(@"Remote synchronization (3-2-1)"); } } + public static string BackendSourceShort { get { return LC.L(@"Remote sync source backend"); } } + public static string BackendSourceLong { get { return LC.L(@"Specify the source backend string for remote synchronization. If not set, the current backup destination is used."); } } + public static string BackendDestinationShort { get { return LC.L(@"Remote sync destination backend"); } } + public static string BackendDestinationLong { get { return LC.L(@"Specify the destination backend string for remote synchronization."); } } + public static string AutoCreateFoldersShort { get { return LC.L(@"Create destination folders"); } } + public static string AutoCreateFoldersLong { get { return LC.L(@"Automatically create folders in the destination backend if they do not exist."); } } + public static string BackendRetriesShort { get { return LC.L(@"Backend retries"); } } + public static string BackendRetriesLong { get { return LC.L(@"Number of times to recreate a backend on backend errors."); } } + public static string BackendRetryDelayShort { get { return LC.L(@"Backend retry delay"); } } + public static string BackendRetryDelayLong { get { return LC.L(@"Delay in milliseconds between backend retries."); } } + public static string BackendRetryBackoffShort { get { return LC.L(@"Exponential retry backoff"); } } + public static string BackendRetryBackoffLong { get { return LC.L(@"Use exponential backoff for backend retries."); } } + public static string ConfirmShort { get { return LC.L(@"Auto confirm remote sync"); } } + public static string ConfirmLong { get { return LC.L(@"Automatically confirm the remote synchronization plan to avoid interactive prompts."); } } + public static string DryRunShort { get { return LC.L(@"Remote sync dry run"); } } + public static string DryRunLong { get { return LC.L(@"Do not write or delete files during remote synchronization."); } } + public static string DestinationOptionsShort { get { return LC.L(@"Destination backend options"); } } + public static string DestinationOptionsLong { get { return LC.L(@"Space-separated key=value pairs for the destination backend, e.g. ""key1=value1 key2=value2""."); } } + public static string ForceShort { get { return LC.L(@"Force remote sync"); } } + public static string ForceLong { get { return LC.L(@"Force synchronization, skipping comparisons and overwriting the destination."); } } + public static string GlobalOptionsShort { get { return LC.L(@"Global backend options"); } } + public static string GlobalOptionsLong { get { return LC.L(@"Space-separated key=value pairs applied to both backends unless overridden."); } } + public static string LogFileShort { get { return LC.L(@"Remote sync log file"); } } + public static string LogFileLong { get { return LC.L(@"The log file to write to. If not set here, the global options are checked."); } } + public static string LogLevelShort { get { return LC.L(@"Remote sync log level"); } } + public static string LogLevelLong { get { return LC.L(@"The log level to use. If not set here, the global options are checked."); } } + public static string ParseArgumentsOnlyShort { get { return LC.L(@"Parse arguments only"); } } + public static string ParseArgumentsOnlyLong { get { return LC.L(@"Only parse the arguments and then exit."); } } + public static string ProgressShort { get { return LC.L(@"Show progress"); } } + public static string ProgressLong { get { return LC.L(@"Print progress to STDOUT."); } } + public static string RetentionShort { get { return LC.L(@"Retention mode"); } } + public static string RetentionLong { get { return LC.L(@"Keep old files by renaming instead of deleting them."); } } + public static string RetryShort { get { return LC.L(@"Operation retries"); } } + public static string RetryLong { get { return LC.L(@"Number of times to retry on errors."); } } + public static string SourceOptionsShort { get { return LC.L(@"Source backend options"); } } + public static string SourceOptionsLong { get { return LC.L(@"Space-separated key=value pairs for the source backend, e.g. ""key1=value1 key2=value2""."); } } + public static string VerifyContentsShort { get { return LC.L(@"Verify contents"); } } + public static string VerifyContentsLong { get { return LC.L(@"Verify file contents before overwriting destination files."); } } + public static string VerifyGetAfterPutShort { get { return LC.L(@"Verify after upload"); } } + public static string VerifyGetAfterPutLong { get { return LC.L(@"Verify files after uploading them."); } } + } internal static class RunScript { public static string Description { get { return LC.L(@"Execute a script before starting an operation, and again on completion"); } }