From 3626d3e4ed978f52c23fc515d20bb207058fb0a1 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 8 Jan 2026 14:36:36 +0100 Subject: [PATCH] Moved core functionality of the remote synchronization tool from the tool to a shared library, making the tool a thin wrapper. --- ...icati.Library.RemoteSynchronization.csproj | 25 + .../RemoteSynchronizationRunner.cs | 808 ++++++++++++++++++ Tools/RemoteSynchronization/Program.cs | 758 +--------------- .../RemoteSynchronization.csproj | 8 +- 4 files changed, 840 insertions(+), 759 deletions(-) create mode 100644 Duplicati/Library/RemoteSynchronization/Duplicati.Library.RemoteSynchronization.csproj create mode 100644 Duplicati/Library/RemoteSynchronization/RemoteSynchronizationRunner.cs diff --git a/Duplicati/Library/RemoteSynchronization/Duplicati.Library.RemoteSynchronization.csproj b/Duplicati/Library/RemoteSynchronization/Duplicati.Library.RemoteSynchronization.csproj new file mode 100644 index 000000000..3f48c0758 --- /dev/null +++ b/Duplicati/Library/RemoteSynchronization/Duplicati.Library.RemoteSynchronization.csproj @@ -0,0 +1,25 @@ + + + + Library + net10.0 + disable + enable + RemoteSynchronization + + + + + + + + + + + + + + + + + diff --git a/Duplicati/Library/RemoteSynchronization/RemoteSynchronizationRunner.cs b/Duplicati/Library/RemoteSynchronization/RemoteSynchronizationRunner.cs new file mode 100644 index 000000000..b9bc7126c --- /dev/null +++ b/Duplicati/Library/RemoteSynchronization/RemoteSynchronizationRunner.cs @@ -0,0 +1,808 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using Duplicati.Library.Backend; +using Duplicati.Library.Common.IO; +using Duplicati.Library.Interface; +using Duplicati.Library.Main.Backend; +using Duplicati.Library.Logging; +using Duplicati.Library.Utility; +using System; +using System.Collections.Generic; +using System.CommandLine; +using System.CommandLine.NamingConventionBinder; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + + +namespace RemoteSynchronization +{ + /// + /// Remote synchronization tool. + /// + public static class RemoteSynchronizationRunner + { + /// + /// Global configuration for the tool. Should be set after parsing the commandline arguments. + /// + private sealed record Config + ( + // Arguments + string Src, + string Dst, + + // Options + bool AutoCreateFolders, + int BackendRetries, + int BackendRetryDelay, + bool BackendRetryWithExponentialBackoff, + bool Confirm, + bool DryRun, + List DstOptions, + bool Force, + List GlobalOptions, + string LogFile, + string LogLevel, + bool ParseArgumentsOnly, + bool Progress, + bool Retention, + int Retry, + List SrcOptions, + bool VerifyContents, + bool VerifyGetAfterPut + ); + + /// + /// The log tag for this tool. + /// + private static readonly string LOGTAG = Duplicati.Library.Logging.Log.LogTagFromType(typeof(RemoteSynchronizationRunner)); + + /// + /// Entry point for running the tool logic with parsed arguments. + /// + /// The commandline arguments + /// 0 on success, -1 on abort, and the number of errors encountered otherwise. + public static async Task RunAsync(string[] args) + { + var arg_src = new Argument(name: "backend_src", description: "The source backend string"); + var arg_dst = new Argument(name: "backend_dst", description: "The destination backend string"); + + var root_cmd = new RootCommand(@"Remote Synchronization Tool + +This tool synchronizes two remote backends. The tool assumes that the intent is +to have the destination match the source. + +If the destination has files that are not in the source, they will be deleted +(or renamed if the retention option is set). + +If the destination has files that are also present in the source, but the files +differ in size, or if the source files have a newer (more recent) timestamp, +the destination files will be overwritten by the source files. Given that some +backends do not allow for metadata or timestamp modification, and that the tool +is run after backup, the destination files should always have a timestamp that +is newer (or the same if run promptly) compared to the source files. + +If the force option is set, the destination will be overwritten by the source, +regardless of the state of the files. It will also skip the initial comparison, +and delete (or rename) all files in the destination. + +If the verify option is set, the files will be downloaded and compared after +uploading to ensure that the files are correct. Files that already exist in the +destination will be verified before being overwritten (if they seemingly match). + ") + { + arg_src, + arg_dst, + + new Option(aliases: ["--auto-create-folders"], description: "Automatically create folders in the destination backend if they do not exist", getDefaultValue: () => true), + new Option(aliases: ["--backend-retries"], description: "Number of times to recreate a backend on backend errors", getDefaultValue: () => 3) { Arity = ArgumentArity.ExactlyOne }, + new Option(aliases: ["--backend-retry-delay"], description: "Delay in milliseconds between backend retries", getDefaultValue: () => 1000) { Arity = ArgumentArity.ExactlyOne }, + new Option(aliases: ["--backend-retry-with-exponential-backoff"], description: "Use exponential backoff for backend retries, multiplying the delay by two for each failure.", getDefaultValue: () => true), + new Option(aliases: ["--confirm", "--yes", "-y"], description: "Automatically confirm the operation", getDefaultValue: () => false), + new Option(aliases: ["--dry-run", "-d"], description: "Do not actually write or delete files. If not set here, the global options will be checked", getDefaultValue: () => false), + OptionWithMultipleTokens(aliases: ["--dst-options"], description: "Options for the destination backend. Each option is a key-value pair separated by an equals sign, e.g. --dst-options key1=value1 key2=value2 [default: empty]", getDefaultValue: () => []), + new Option(aliases: ["--force", "-f"], description: "Force the synchronization", getDefaultValue: () => false), + OptionWithMultipleTokens(aliases: ["--global-options"], description: "Global options all backends. May be overridden by backend specific options (src-options, dst-options). Each option is a key-value pair separated by an equals sign, e.g. --global-options key1=value1 key2=value2 [default: empty]", getDefaultValue: () => []), + new Option(aliases: ["--log-file"], description: "The log file to write to. If not set here, global options will be checked [default: \"\"]", getDefaultValue: () => "") { Arity = ArgumentArity.ExactlyOne }, + new Option(aliases: ["--log-level"], description: "The log level to use. If not set here, global options will be checked", getDefaultValue: () => "Information") { Arity = ArgumentArity.ExactlyOne }, + new Option(aliases: ["--parse-arguments-only"], description: "Only parse the arguments and then exit", getDefaultValue: () => false), + new Option(aliases: ["--progress"], description: "Print progress to STDOUT", getDefaultValue: () => false), + new Option(aliases: ["--retention"], description: "Toggles whether to keep old files. Any deletes will be renames instead", getDefaultValue: () => false), + new Option(aliases: ["--retry"], description: "Number of times to retry on errors", getDefaultValue: () => 3) { Arity = ArgumentArity.ExactlyOne }, + OptionWithMultipleTokens(aliases: ["--src-options"], description: "Options for the source backend. Each option is a key-value pair separated by an equals sign, e.g. --src-options key1=value1 key2=value2 [default: empty]", getDefaultValue: () => []), + new Option(aliases: ["--verify-contents"], description: "Verify the contents of the files to decide whether the pre-existing destination files should be overwritten", getDefaultValue: () => false), + new Option(aliases: ["--verify-get-after-put"], description: "Verify the files after uploading them to ensure that they were uploaded correctly", getDefaultValue: () => false), + }; + + root_cmd.Handler = CommandHandler.Create((string backend_src, string backend_dst, Config config, CancellationToken token) => + { + var config_with_args = config with { Dst = backend_dst, Src = backend_src }; + + return Run(config_with_args, token); + }); + + return await root_cmd.InvokeAsync(args).ConfigureAwait(false); + } + + /// + /// The main logic of the tool. + /// + /// The parsed configuration for the tool. + /// The cancellation token to use for the asynchronous operations. + /// The return code for the main entry; 0 on success. + private static async Task Run(Config config, CancellationToken token) + { + // Unpack and parse the multi token options + var global_options = ParseOptions(config.GlobalOptions); + + // Parse the log level + var log_level_parsed = Enum.TryParse(config.LogLevel, true, out var log_level_enum); + log_level_enum = log_level_parsed ? log_level_enum : Duplicati.Library.Logging.LogMessageType.Information; + using var console_sink = new StreamLogDestination(Console.OpenStandardOutput()); + + // Parse the log file + StreamLogDestination? log_file_sink = null; + if (!string.IsNullOrEmpty(config.LogFile)) + { + string log_file_dir = SystemIO.IO_OS.PathGetDirectoryName(config.LogFile); + if (!string.IsNullOrEmpty(log_file_dir) && !SystemIO.IO_OS.DirectoryExists(log_file_dir)) + SystemIO.IO_OS.DirectoryCreate(log_file_dir); + log_file_sink = new StreamLogDestination(config.LogFile); + } + using var multi_sink = new MultiLogDestination([console_sink, log_file_sink]); + + // Start the logging scope + using var _ = Duplicati.Library.Logging.Log.StartScope(multi_sink, log_level_enum); + + var src_opts = ParseOptions(config.SrcOptions); + var dst_opts = ParseOptions(config.DstOptions); + + // Merge the global options into the source and destination options. The global options will be overridden by the source and destination options. + foreach (var x in global_options) + { + if (!src_opts.ContainsKey(x.Key)) + src_opts[x.Key] = x.Value; + if (!dst_opts.ContainsKey(x.Key)) + dst_opts[x.Key] = x.Value; + } + + // Check if we only had to parse the arguments + if (config.ParseArgumentsOnly) + { + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Arguments parsed successfully; {0}; exiting.", config); + return 0; + } + + using var b1m = new LightWeightBackendManager(config.Src, src_opts, config.BackendRetries, config.BackendRetryDelay, config.BackendRetryWithExponentialBackoff); + using var b2m = new LightWeightBackendManager(config.Dst, dst_opts, config.BackendRetries, config.BackendRetryDelay, config.BackendRetryWithExponentialBackoff); + + // Prepare the operations + var (to_copy, to_delete, to_verify) = await PrepareFileLists(b1m, b2m, config, token).ConfigureAwait(false); + + // Verify the files if requested. If the files are not verified, they will be deleted and copied again. + long verified = 0, failed_verify = 0; + if (config.VerifyContents) + { + // As this is a potentially slow operation, ask for confirmation of the verification) + if (!config.Confirm) + { + Console.WriteLine($"This will verify {to_verify.Count()} files before copying them. Do you want to continue? [y/N]"); + var response = Console.ReadLine(); + if (!response?.Equals("y", StringComparison.CurrentCultureIgnoreCase) ?? true) + { + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Aborted"); + return -1; + } + } + + var not_verified = await VerifyAsync(b1m, b2m, to_verify, config, token).ConfigureAwait(false); + failed_verify = not_verified.Count(); + verified = to_verify.Count() - failed_verify; + + if (not_verified.Any()) + { + Duplicati.Library.Logging.Log.WriteWarningMessage(LOGTAG, "rsync", null, + "{0} files failed verification. They will be deleted and copied again.", + not_verified.Count()); + to_delete = to_delete.Concat(not_verified); + to_copy = to_copy.Concat(not_verified); + } + } + + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "The remote synchronization plan is to {0} {1} files from {2}, then copy {3} files from {4} to {2}.", + config.Retention ? "rename" : "delete", + to_delete.Count(), b2m.DisplayName, to_copy.Count(), b1m.DisplayName); + + // As this is a potentially destructive operation, ask for confirmation + if (!config.Confirm) + { + var delete_rename = config.Retention ? "Rename" : "Delete"; + Console.WriteLine($"This will perform the following actions (in order):"); + Console.WriteLine($" - {delete_rename} {to_delete.Count()} files from {config.Dst}"); + Console.WriteLine($" - Copy {to_copy.Count()} files from {config.Src} to {config.Dst}"); + Console.WriteLine(); + Console.WriteLine("Do you want to continue? [y/N]"); + + var response = Console.ReadLine(); + if (!response?.Equals("y", StringComparison.CurrentCultureIgnoreCase) ?? true) + { + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Aborted"); + return -1; + } + } + + // Delete or rename the files that are not needed + long renamed = 0, deleted = 0; + if (config.Retention) + { + renamed = await RenameAsync(b2m, to_delete, config, token).ConfigureAwait(false); + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "Renamed {0} files in {1}", renamed, b2m.DisplayName); + } + else + { + deleted = await DeleteAsync(b2m, to_delete, config, token).ConfigureAwait(false); + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "Deleted {0} files from {1}", deleted, b2m.DisplayName); + } + + // Copy the files + var (copied, copy_errors) = await CopyAsync(b1m, b2m, to_copy, config, token).ConfigureAwait(false); + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "Copied {0} files from {1} to {2}", copied, b1m.DisplayName, b2m.DisplayName); + + // If there are still errors, retry a few times + if (copy_errors.Any()) + { + Duplicati.Library.Logging.Log.WriteWarningMessage(LOGTAG, "rsync", null, + "Could not copy {0} files.", copy_errors.Count()); + if (config.Retry > 0) + { + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "Retrying {0} more times to copy the {1} files that failed", + config.Retry, copy_errors.Count()); + for (int i = 0; i < config.Retry; i++) + { + await Task.Delay(5000).ConfigureAwait(false); // Wait 5 seconds before retrying + (copied, copy_errors) = await CopyAsync(b1m, b2m, copy_errors, config, token).ConfigureAwait(false); + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "Copied {0} files from {1} to {2}", copied, b1m.DisplayName, b2m.DisplayName); + if (!copy_errors.Any()) + break; + } + } + + if (copy_errors.Any()) + { + Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", null, + "Could not copy {0} files. Not retrying any more.", copy_errors.Count()); + return copy_errors.Count(); + } + } + + // Results reporting + if (verified > 0) + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "Verified {0} files in {1} that didn't need to be copied", + verified, b2m.DisplayName); + if (failed_verify > 0) + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "Failed to verify {0} files in {1}, which were then attempted to be copied", + failed_verify, b2m.DisplayName); + if (copied > 0) + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "Copied {0} files from {1} to {2}", copied, b1m.DisplayName, b2m.DisplayName); + if (deleted > 0) + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "Deleted {0} files from {1}", deleted, b2m.DisplayName); + if (renamed > 0) + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "Renamed {0} files in {1}", renamed, b2m.DisplayName); + + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "Remote synchronization completed successfully"); + + return 0; + } + + // TODO have concurrency parameters: uploaders, downloaders + // TODO low memory mode, where things aren't kept in memory. Maybe utilize SQLite? + // TODO For convenience, have the option to launch a "duplicati test" on the destination backend after the synchronization + // TODO Save hash to minimize redownload + // TODO Duplicati Results + + /// + /// Copies the files from one backend to another. + /// The files are copied one by one, and each file is verified after uploading if the verify flag is set. + /// + /// The source backend. + /// The destination backend. + /// The files that will be copied. + /// The parsed configuration for the tool. + /// The cancellation token to use for the asynchronous operations. + /// A tuple holding the number of succesful copies and a List of the files that failed. + private static async Task<(long, IEnumerable)> CopyAsync(LightWeightBackendManager b_src, LightWeightBackendManager b_dst, IEnumerable files, Config config, CancellationToken token) + { + long successful_copies = 0; + List errors = []; + long i = 0, n = files.Count(); + var sw_get_src = new System.Diagnostics.Stopwatch(); + var sw_put_dst = new System.Diagnostics.Stopwatch(); + var sw_get_dst = new System.Diagnostics.Stopwatch(); + var sw_get_cmp = new System.Diagnostics.Stopwatch(); + + foreach (var f in files) + { + if (config.Progress) + Console.Write($"\rCopying: {i}/{n}"); + + Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", + "Copying {0} from {1} to {2}", f.Name, b_src.DisplayName, b_dst.DisplayName); + + using var s_src = Duplicati.Library.Utility.TempFileStream.Create(); + + try + { + sw_get_src.Start(); + await b_src.GetAsync(f.Name, s_src, token).ConfigureAwait(false); + s_src.Position = 0; + sw_get_src.Stop(); + if (config.DryRun) + { + Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", + "Would write {0} bytes of {1} to {2}", + Duplicati.Library.Utility.Utility.FormatSizeString(s_src.Length), + f.Name, b_dst.DisplayName); + } + else + { + sw_put_dst.Start(); + await b_dst.PutAsync(f.Name, s_src, token).ConfigureAwait(false); + s_src.Position = 0; + sw_put_dst.Stop(); + if (config.VerifyGetAfterPut) + { + // Start calculating the hash of the source file while we are downloading + var srchashtask = Task.Run(() => + { + using var hasher = HashFactory.CreateHasher("SHA256"); + return Convert.ToBase64String(hasher.ComputeHash(s_src)); + }); + + using var s_dst = Duplicati.Library.Utility.TempFileStream.Create(); + + sw_get_dst.Start(); + await b_dst.GetAsync(f.Name, s_dst, token).ConfigureAwait(false); + s_dst.Position = 0; + sw_get_dst.Stop(); + + sw_get_cmp.Start(); + string? err_string = null; + if (s_src.Length != s_dst.Length) + { + err_string = $"The sizes of the files do not match: {s_src.Length} != {s_dst.Length}."; + } + + using var hasher = HashFactory.CreateHasher("SHA256"); + var dsthash = Convert.ToBase64String(hasher.ComputeHash(s_dst)); + + if (await srchashtask.ConfigureAwait(false) != dsthash) + { + err_string = (err_string is null ? "" : err_string + " ") + "The contents of the files do not match."; + } + sw_get_cmp.Stop(); + + if (err_string is not null) + { + throw new Exception(err_string); + } + } + } + + successful_copies++; + } + catch (Exception e) + { + Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e, + "Error copying {0}: {1}", f.Name, e.Message); + errors.Add(f); + } + finally + { + i++; + + // Stop any running timers + sw_get_src.Stop(); + sw_put_dst.Stop(); + sw_get_dst.Stop(); + sw_get_cmp.Stop(); + } + } + + if (config.Progress) + Console.WriteLine($"\rCopying: {n}/{n}"); + + Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", + "Copy | Get source: {0} ms, Put destination: {1} ms, Get destination: {2} ms, Get compare: {3} ms", + TimeSpan.FromMilliseconds(sw_get_src.ElapsedMilliseconds), + TimeSpan.FromMilliseconds(sw_put_dst.ElapsedMilliseconds), + TimeSpan.FromMilliseconds(sw_get_dst.ElapsedMilliseconds), + TimeSpan.FromMilliseconds(sw_get_cmp.ElapsedMilliseconds)); + + return (successful_copies, errors); + } + + /// + /// Deletes the files from a backend. + /// + /// The backend to delete the files from. + /// The files to delete. + /// The parsed configuration for the tool. + /// The cancellation token to use for the asynchronous operations. + /// The number of successful deletions. + private static async Task DeleteAsync(LightWeightBackendManager b, IEnumerable files, Config config, CancellationToken token) + { + long successful_deletes = 0; + long i = 0, n = files.Count(); + using var timer = new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Delete operation"); + + foreach (var f in files) + { + if (n > 1 && config.Progress) + { + Console.Write($"\rDeleting: {i}/{n}"); + } + + Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", + "Deleting {0} from {1}", f.Name, b.DisplayName); + + try + { + if (config.DryRun) + { + Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", + "Would delete {0} from {1}", f.Name, b.DisplayName); + } + else + { + await b.DeleteAsync(f.Name, token).ConfigureAwait(false); + } + successful_deletes++; + } + catch (Exception e) + { + Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e, + "Error deleting {0}: {1}", f.Name, e.Message); + } + + i++; + } + + if (config.Progress) + Console.WriteLine($"\rDeleting: {n}/{n}"); + return successful_deletes; + } + + /// + /// Creates an option that allows multiple tokens and multiple arguments per token. + /// + /// The aliases for the option. + /// The description for the option. + /// The created option. + private static Option> OptionWithMultipleTokens(string[] aliases, string description, Func> getDefaultValue) + { + return new Option>(aliases: aliases, description: description, getDefaultValue: getDefaultValue) + { + Arity = ArgumentArity.OneOrMore, + AllowMultipleArgumentsPerToken = true + }; + } + + private sealed class MultiLogDestination : ILogDestination, IDisposable + { + private readonly List m_destinations; + + public MultiLogDestination(IEnumerable destinations) + { + m_destinations = destinations.Where(x => x != null).Cast().ToList(); + } + + public void WriteMessage(LogEntry entry) + { + foreach (var destination in m_destinations) + destination.WriteMessage(entry); + } + + public void Dispose() + { + foreach (var destination in m_destinations) + if (destination is IDisposable disposable) + disposable.Dispose(); + } + } + + /// + /// Parses the options from a list of strings. + /// Each option should be in the format "key=value". If the value contains spaces, + /// it should be enclosed in quotes, e.g. "key=\"value with spaces\"". + /// + /// The list of string options to parse + /// A dictionary with the parsed options, where the key is the option name and the value is the option value. + /// If an option was not parsed correctly. + private static Dictionary ParseOptions(IEnumerable options) + { + var result = options + .Select(x => x.Split('=')) + .ToDictionary(x => x[0], x => string.Join("=", x.Skip(1))); + + // Double check that the options are valid by reconstructing them from the dictionary + foreach (var opt in result.Select(x => $"{x.Key}={x.Value}")) + { + if (!options.Contains(opt)) + { + Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", null, + "The source option '{0}' is not valid. Please check the syntax.", opt); + throw new ArgumentException($"The source option '{opt}' has not been parsed correctly."); + } + } + + return result; + } + + /// + /// Prepares the lists of files to copy, delete and verify. + /// The files to copy are the files that are not in the destination, have a different size or have a more recent modification date. + /// The files to delete are the files that are found in the destination but not found in the source. + /// The files to verify are the files that are found in both the source and the destination, and that have the same size and modification date. + /// + /// The source lightweight backend manager. + /// The destination lightweight backend manager. + /// The parsed configuration for the tool. + /// The cancellation token to use for the asynchronous operations. + /// A tuple of Lists each holding the files to copy, delete and verify. + private static async Task<(IEnumerable, IEnumerable, IEnumerable)> PrepareFileLists(LightWeightBackendManager b_src, LightWeightBackendManager b_dst, Config config, CancellationToken token) + { + IEnumerable files_src, files_dst; + + using (new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Prepare | List source")) + files_src = await b_src.ListAsync(token).ConfigureAwait(false); + + using (new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Prepare | List destination")) + files_dst = await b_dst.ListAsync(token).ConfigureAwait(false); + + // Shortcut for force + if (config.Force) + { + return (files_src, files_dst, []); + } + + // Shortcut for empty destination + if (!files_dst.Any()) + { + return (files_src, [], []); + } + + Dictionary lookup_src, lookup_dst; + using (new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Prepare | Build lookup for source and destination")) + { + lookup_src = files_src.ToDictionary(x => x.Name); + lookup_dst = files_dst.ToDictionary(x => x.Name); + } + + var to_copy = new List(); + var to_delete = new HashSet(); + var to_verify = new List(); + + // Find all of the files in src that are not in dst, where the dst has a different size than src or src a more recent modification date than dst + using (new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Prepare | Check the files that are present in source against destination")) + foreach (var f_src in files_src) + { + if (lookup_dst.TryGetValue(f_src.Name, out var f_dst)) + { + if (f_src.Size != f_dst.Size || f_src.LastModification > f_dst.LastModification) + { + // The file is different, so we need to copy it + to_copy.Add(f_src); + to_delete.Add(f_dst.Name); + } + else + { + // The file seems to be the same, so we need to verify it if the user wants to + to_verify.Add(f_src); + } + } + else + { + // The file is not in the destination, so we need to copy it + to_copy.Add(f_src); + } + } + + // Find all of the files in dst that are not in src + using (new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Prepare | Check the files that are present in destination against source")) + foreach (var f_dst in files_dst) + { + if (to_delete.Contains(f_dst.Name)) + continue; + + if (!lookup_src.ContainsKey(f_dst.Name)) + { + to_delete.Add(f_dst.Name); + } + } + + List to_delete_lookedup; + using (new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Prepare | Lookup the files to delete")) + to_delete_lookedup = [.. to_delete.Select(x => lookup_dst[x])]; + + return (to_copy, to_delete_lookedup, to_verify); + } + + /// + /// Renames the files in a backend. + /// The renaming is done by deleting the file and re-uploading it with a new name. + /// + /// The lightweight backend manager to issue rename operations to. + /// The files to rename. + /// The parsed configuration for the tool. + /// The cancellation token to use for the asynchronous operations. + /// The number of successful renames. + private static async Task RenameAsync(LightWeightBackendManager bm, IEnumerable files, Config config, CancellationToken token) + { + long successful_renames = 0; + string prefix = $"{System.DateTime.UtcNow:yyyyMMddHHmmss}.old"; + using var downloaded = new MemoryStream(); + long i = 0, n = files.Count(); + + var sw = new System.Diagnostics.Stopwatch(); + + foreach (var f in files) + { + if (config.Progress) + Console.Write($"\rRenaming: {i}/{n}"); + + Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", + "Renaming {0} to {1}.{0} by calling Rename on {2}", + f.Name, prefix, bm.DisplayName); + + try + { + if (config.DryRun) + { + Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", + "Would rename {0} to {1}.{0} by calling Rename on {2}", + f.Name, prefix, bm.DisplayName); + } + else + { + sw.Start(); + await bm.RenameAsync(f.Name, $"{prefix}.{f.Name}", token).ConfigureAwait(false); + sw.Stop(); + } + successful_renames++; + } + catch (Exception e) + { + Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e, + "Error renaming {0}: {1}", f.Name, e.Message); + } + finally + { + // Ensure the timer is stopped + sw.Stop(); + } + + i++; + } + + Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", + "Rename: {0} ms", + TimeSpan.FromMilliseconds(sw.ElapsedMilliseconds)); + + if (config.Progress) + Console.WriteLine($"\rRenaming: {n}/{n}"); + + return successful_renames; + } + + /// + /// Verifies the files in the destination backend. + /// The verification is done by downloading the files from the destination backend and comparing them to the source files. + /// + /// The source lightweight backend manager. + /// The destination lightweight backend manager. + /// The files to verify. + /// The parsed configuration for the tool. + /// The cancellation token to use for the asynchronous operations. + /// A list of the files that failed verification. + private static async Task> VerifyAsync(LightWeightBackendManager b_src, LightWeightBackendManager b_dst, IEnumerable files, Config config, CancellationToken token) + { + var errors = new List(); + using var s_src = new MemoryStream(); + using var s_dst = new MemoryStream(); + long i = 0, n = files.Count(); + var sw_get = new System.Diagnostics.Stopwatch(); + var sw_cmp = new System.Diagnostics.Stopwatch(); + + foreach (var f in files) + { + if (config.Progress) + Console.Write($"\rVerifying: {i}/{n}"); + + Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", + "Verifying {0} by downloading and comparing {1} bytes from {2} and {3}", + f.Name, + Duplicati.Library.Utility.Utility.FormatSizeString(s_src.Length), + b_dst.DisplayName, b_src.DisplayName); + + try + { + // Get both files + sw_get.Start(); + var fs = b_src.GetAsync(f.Name, s_src, token); + var ds = b_dst.GetAsync(f.Name, s_dst, token); + await Task.WhenAll(fs, ds).ConfigureAwait(false); + sw_get.Stop(); + + // Compare the contents + sw_cmp.Start(); + if (s_src.Length != s_dst.Length || !s_src.ToArray().SequenceEqual(s_dst.ToArray())) + { + errors.Add(f); + } + sw_cmp.Stop(); + } + catch (Exception e) + { + errors.Add(f); + Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e, + "Error during verification of {0}: {1}", f.Name, e.Message); + } + finally + { + // Reset the streams + s_src.SetLength(0); + s_dst.SetLength(0); + + // Stop any running timers + sw_get.Stop(); + sw_cmp.Stop(); + } + + i++; + } + + Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", + "Verify | Get: {0} ms, Compare: {1} ms", + TimeSpan.FromMilliseconds(sw_get.ElapsedMilliseconds), + TimeSpan.FromMilliseconds(sw_cmp.ElapsedMilliseconds)); + + if (config.Progress) + Console.WriteLine($"\rVerifying: {n}/{n}"); + + return errors; + } + + } +} diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 8ec7d506c..51cb7ee98 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2025, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a @@ -19,767 +19,21 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using Duplicati.Library.Common.IO; -using Duplicati.Library.Interface; -using Duplicati.Library.Utility; -using System; -using System.Collections.Generic; -using System.CommandLine; -using System.CommandLine.NamingConventionBinder; -using System.IO; -using System.Linq; -using System.Threading; using System.Threading.Tasks; - namespace RemoteSynchronization { /// - /// Remote synchronization tool. + /// Remote synchronization tool entry point. /// - public class Program + public static class Program { - /// - /// Global configuration for the tool. Should be set after parsing the commandline arguments. - /// - private sealed record Config - ( - // Arguments - string Src, - string Dst, - - // Options - bool AutoCreateFolders, - int BackendRetries, - int BackendRetryDelay, - bool BackendRetryWithExponentialBackoff, - bool Confirm, - bool DryRun, - List DstOptions, - bool Force, - List GlobalOptions, - string LogFile, - string LogLevel, - bool ParseArgumentsOnly, - bool Progress, - bool Retention, - int Retry, - List SrcOptions, - bool VerifyContents, - bool VerifyGetAfterPut - ); - - /// - /// The log tag for this tool. - /// - private static readonly string LOGTAG = Duplicati.Library.Logging.Log.LogTagFromType(); - /// /// Main entry point for the tool. /// - /// The commandline arguments + /// The commandline arguments. /// 0 on success, -1 on abort, and the number of errors encountered otherwise. - public static async Task Main(string[] args) - { - var arg_src = new Argument(name: "backend_src", description: "The source backend string"); - var arg_dst = new Argument(name: "backend_dst", description: "The destination backend string"); - - var root_cmd = new RootCommand(@"Remote Synchronization Tool - -This tool synchronizes two remote backends. The tool assumes that the intent is -to have the destination match the source. - -If the destination has files that are not in the source, they will be deleted -(or renamed if the retention option is set). - -If the destination has files that are also present in the source, but the files -differ in size, or if the source files have a newer (more recent) timestamp, -the destination files will be overwritten by the source files. Given that some -backends do not allow for metadata or timestamp modification, and that the tool -is run after backup, the destination files should always have a timestamp that -is newer (or the same if run promptly) compared to the source files. - -If the force option is set, the destination will be overwritten by the source, -regardless of the state of the files. It will also skip the initial comparison, -and delete (or rename) all files in the destination. - -If the verify option is set, the files will be downloaded and compared after -uploading to ensure that the files are correct. Files that already exist in the -destination will be verified before being overwritten (if they seemingly match). - ") - { - arg_src, - arg_dst, - - new Option(aliases: ["--auto-create-folders"], description: "Automatically create folders in the destination backend if they do not exist", getDefaultValue: () => true), - new Option(aliases: ["--backend-retries"], description: "Number of times to recreate a backend on backend errors", getDefaultValue: () => 3) { Arity = ArgumentArity.ExactlyOne }, - new Option(aliases: ["--backend-retry-delay"], description: "Delay in milliseconds between backend retries", getDefaultValue: () => 1000) { Arity = ArgumentArity.ExactlyOne }, - new Option(aliases: ["--backend-retry-with-exponential-backoff"], description: "Use exponential backoff for backend retries, multiplying the delay by two for each failure.", getDefaultValue: () => true), - new Option(aliases: ["--confirm", "--yes", "-y"], description: "Automatically confirm the operation", getDefaultValue: () => false), - new Option(aliases: ["--dry-run", "-d"], description: "Do not actually write or delete files. If not set here, the global options will be checked", getDefaultValue: () => false), - OptionWithMultipleTokens(aliases: ["--dst-options"], description: "Options for the destination backend. Each option is a key-value pair separated by an equals sign, e.g. --dst-options key1=value1 key2=value2 [default: empty]", getDefaultValue: () => []), - new Option(aliases: ["--force", "-f"], description: "Force the synchronization", getDefaultValue: () => false), - OptionWithMultipleTokens(aliases: ["--global-options"], description: "Global options all backends. May be overridden by backend specific options (src-options, dst-options). Each option is a key-value pair separated by an equals sign, e.g. --global-options key1=value1 key2=value2 [default: empty]", getDefaultValue: () => []), - new Option(aliases: ["--log-file"], description: "The log file to write to. If not set here, global options will be checked [default: \"\"]", getDefaultValue: () => "") { Arity = ArgumentArity.ExactlyOne }, - new Option(aliases: ["--log-level"], description: "The log level to use. If not set here, global options will be checked", getDefaultValue: () => "Information") { Arity = ArgumentArity.ExactlyOne }, - new Option(aliases: ["--parse-arguments-only"], description: "Only parse the arguments and then exit", getDefaultValue: () => false), - new Option(aliases: ["--progress"], description: "Print progress to STDOUT", getDefaultValue: () => false), - new Option(aliases: ["--retention"], description: "Toggles whether to keep old files. Any deletes will be renames instead", getDefaultValue: () => false), - new Option(aliases: ["--retry"], description: "Number of times to retry on errors", getDefaultValue: () => 3) { Arity = ArgumentArity.ExactlyOne }, - OptionWithMultipleTokens(aliases: ["--src-options"], description: "Options for the source backend. Each option is a key-value pair separated by an equals sign, e.g. --src-options key1=value1 key2=value2 [default: empty]", getDefaultValue: () => []), - new Option(aliases: ["--verify-contents"], description: "Verify the contents of the files to decide whether the pre-existing destination files should be overwritten", getDefaultValue: () => false), - new Option(aliases: ["--verify-get-after-put"], description: "Verify the files after uploading them to ensure that they were uploaded correctly", getDefaultValue: () => false), - }; - - root_cmd.Handler = CommandHandler.Create((string backend_src, string backend_dst, Config config, CancellationToken token) => - { - var config_with_args = config with { Dst = backend_dst, Src = backend_src }; - - return Run(config_with_args, token); - }); - - return await root_cmd.InvokeAsync(args).ConfigureAwait(false); - } - - /// - /// The main logic of the tool. - /// - /// The parsed configuration for the tool. - /// The cancellation token to use for the asynchronous operations. - /// The return code for the main entry; 0 on success. - private static async Task Run(Config config, CancellationToken token) - { - // Unpack and parse the multi token options - var global_options = ParseOptions(config.GlobalOptions); - - // Parse the log level - var log_level_parsed = Enum.TryParse(config.LogLevel, true, out var log_level_enum); - log_level_enum = log_level_parsed ? log_level_enum : Duplicati.Library.Logging.LogMessageType.Information; - - using var console_sink = new Duplicati.CommandLine.ConsoleOutput(Console.Out, global_options); - using var multi_sink = new Duplicati.Library.Main.ControllerMultiLogTarget(console_sink, log_level_enum, null, null); - - // Parse the log file - // The log file sink doesn't have to be disposed, as the multi_sink will take care of it - Duplicati.Library.Logging.StreamLogDestination? log_file_sink = null; - if (!string.IsNullOrEmpty(config.LogFile)) - { - string log_file_dir = SystemIO.IO_OS.PathGetDirectoryName(config.LogFile); - if (!string.IsNullOrEmpty(log_file_dir) && !SystemIO.IO_OS.DirectoryExists(log_file_dir)) - SystemIO.IO_OS.DirectoryCreate(log_file_dir); - log_file_sink = new Duplicati.Library.Logging.StreamLogDestination(config.LogFile); - } - multi_sink.AddTarget(log_file_sink, log_level_enum, null); - - // Start the logging scope - using var _ = Duplicati.Library.Logging.Log.StartScope(multi_sink, log_level_enum); - - var src_opts = ParseOptions(config.SrcOptions); - var dst_opts = ParseOptions(config.DstOptions); - - // Merge the global options into the source and destination options. The global options will be overridden by the source and destination options. - foreach (var x in global_options) - { - if (!src_opts.ContainsKey(x.Key)) - src_opts[x.Key] = x.Value; - if (!dst_opts.ContainsKey(x.Key)) - dst_opts[x.Key] = x.Value; - } - - // Check if we only had to parse the arguments - if (config.ParseArgumentsOnly) - { - Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Arguments parsed successfully; {0}; exiting.", config); - return 0; - } - - using var b1m = new LightWeightBackendManager(config.Src, src_opts, config.BackendRetries, config.BackendRetryDelay, config.BackendRetryWithExponentialBackoff); - using var b2m = new LightWeightBackendManager(config.Dst, dst_opts, config.BackendRetries, config.BackendRetryDelay, config.BackendRetryWithExponentialBackoff); - - // Prepare the operations - var (to_copy, to_delete, to_verify) = await PrepareFileLists(b1m, b2m, config, token).ConfigureAwait(false); - - // Verify the files if requested. If the files are not verified, they will be deleted and copied again. - long verified = 0, failed_verify = 0; - if (config.VerifyContents) - { - // As this is a potentially slow operation, ask for confirmation of the verification) - if (!config.Confirm) - { - Console.WriteLine($"This will verify {to_verify.Count()} files before copying them. Do you want to continue? [y/N]"); - var response = Console.ReadLine(); - if (!response?.Equals("y", StringComparison.CurrentCultureIgnoreCase) ?? true) - { - Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Aborted"); - return -1; - } - } - - var not_verified = await VerifyAsync(b1m, b2m, to_verify, config, token).ConfigureAwait(false); - failed_verify = not_verified.Count(); - verified = to_verify.Count() - failed_verify; - - if (not_verified.Any()) - { - Duplicati.Library.Logging.Log.WriteWarningMessage(LOGTAG, "rsync", null, - "{0} files failed verification. They will be deleted and copied again.", - not_verified.Count()); - to_delete = to_delete.Concat(not_verified); - to_copy = to_copy.Concat(not_verified); - } - } - - Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", - "The remote synchronization plan is to {0} {1} files from {2}, then copy {3} files from {4} to {2}.", - config.Retention ? "rename" : "delete", - to_delete.Count(), b2m.DisplayName, to_copy.Count(), b1m.DisplayName); - - // As this is a potentially destructive operation, ask for confirmation - if (!config.Confirm) - { - var delete_rename = config.Retention ? "Rename" : "Delete"; - Console.WriteLine($"This will perform the following actions (in order):"); - Console.WriteLine($" - {delete_rename} {to_delete.Count()} files from {config.Dst}"); - Console.WriteLine($" - Copy {to_copy.Count()} files from {config.Src} to {config.Dst}"); - Console.WriteLine(); - Console.WriteLine("Do you want to continue? [y/N]"); - - var response = Console.ReadLine(); - if (!response?.Equals("y", StringComparison.CurrentCultureIgnoreCase) ?? true) - { - Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Aborted"); - return -1; - } - } - - // Delete or rename the files that are not needed - long renamed = 0, deleted = 0; - if (config.Retention) - { - renamed = await RenameAsync(b2m, to_delete, config, token).ConfigureAwait(false); - Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", - "Renamed {0} files in {1}", renamed, b2m.DisplayName); - } - else - { - deleted = await DeleteAsync(b2m, to_delete, config, token).ConfigureAwait(false); - Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", - "Deleted {0} files from {1}", deleted, b2m.DisplayName); - } - - // Copy the files - var (copied, copy_errors) = await CopyAsync(b1m, b2m, to_copy, config, token).ConfigureAwait(false); - Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", - "Copied {0} files from {1} to {2}", copied, b1m.DisplayName, b2m.DisplayName); - - // If there are still errors, retry a few times - if (copy_errors.Any()) - { - Duplicati.Library.Logging.Log.WriteWarningMessage(LOGTAG, "rsync", null, - "Could not copy {0} files.", copy_errors.Count()); - if (config.Retry > 0) - { - Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", - "Retrying {0} more times to copy the {1} files that failed", - config.Retry, copy_errors.Count()); - for (int i = 0; i < config.Retry; i++) - { - await Task.Delay(5000).ConfigureAwait(false); // Wait 5 seconds before retrying - (copied, copy_errors) = await CopyAsync(b1m, b2m, copy_errors, config, token).ConfigureAwait(false); - Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", - "Copied {0} files from {1} to {2}", copied, b1m.DisplayName, b2m.DisplayName); - if (!copy_errors.Any()) - break; - } - } - - if (copy_errors.Any()) - { - Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", null, - "Could not copy {0} files. Not retrying any more.", copy_errors.Count()); - return copy_errors.Count(); - } - } - - // Results reporting - if (verified > 0) - Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", - "Verified {0} files in {1} that didn't need to be copied", - verified, b2m.DisplayName); - if (failed_verify > 0) - Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", - "Failed to verify {0} files in {1}, which were then attempted to be copied", - failed_verify, b2m.DisplayName); - if (copied > 0) - Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", - "Copied {0} files from {1} to {2}", copied, b1m.DisplayName, b2m.DisplayName); - if (deleted > 0) - Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", - "Deleted {0} files from {1}", deleted, b2m.DisplayName); - if (renamed > 0) - Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", - "Renamed {0} files in {1}", renamed, b2m.DisplayName); - - Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", - "Remote synchronization completed successfully"); - - return 0; - } - - // TODO have concurrency parameters: uploaders, downloaders - // TODO low memory mode, where things aren't kept in memory. Maybe utilize SQLite? - // TODO For convenience, have the option to launch a "duplicati test" on the destination backend after the synchronization - // TODO Save hash to minimize redownload - // TODO Duplicati Results - - /// - /// Copies the files from one backend to another. - /// The files are copied one by one, and each file is verified after uploading if the verify flag is set. - /// - /// The source backend. - /// The destination backend. - /// The files that will be copied. - /// The parsed configuration for the tool. - /// The cancellation token to use for the asynchronous operations. - /// A tuple holding the number of succesful copies and a List of the files that failed. - private static async Task<(long, IEnumerable)> CopyAsync(LightWeightBackendManager b_src, LightWeightBackendManager b_dst, IEnumerable files, Config config, CancellationToken token) - { - long successful_copies = 0; - List errors = []; - long i = 0, n = files.Count(); - var sw_get_src = new System.Diagnostics.Stopwatch(); - var sw_put_dst = new System.Diagnostics.Stopwatch(); - var sw_get_dst = new System.Diagnostics.Stopwatch(); - var sw_get_cmp = new System.Diagnostics.Stopwatch(); - - foreach (var f in files) - { - if (config.Progress) - Console.Write($"\rCopying: {i}/{n}"); - - Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", - "Copying {0} from {1} to {2}", f.Name, b_src.DisplayName, b_dst.DisplayName); - - using var s_src = Duplicati.Library.Utility.TempFileStream.Create(); - - try - { - sw_get_src.Start(); - await b_src.GetAsync(f.Name, s_src, token).ConfigureAwait(false); - s_src.Position = 0; - sw_get_src.Stop(); - if (config.DryRun) - { - Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", - "Would write {0} bytes of {1} to {2}", - Duplicati.Library.Utility.Utility.FormatSizeString(s_src.Length), - f.Name, b_dst.DisplayName); - } - else - { - sw_put_dst.Start(); - await b_dst.PutAsync(f.Name, s_src, token).ConfigureAwait(false); - s_src.Position = 0; - sw_put_dst.Stop(); - if (config.VerifyGetAfterPut) - { - // Start calculating the hash of the source file while we are downloading - var srchashtask = Task.Run(() => - { - using var hasher = HashFactory.CreateHasher("SHA256"); - return Convert.ToBase64String(hasher.ComputeHash(s_src)); - }); - - using var s_dst = Duplicati.Library.Utility.TempFileStream.Create(); - - sw_get_dst.Start(); - await b_dst.GetAsync(f.Name, s_dst, token).ConfigureAwait(false); - s_dst.Position = 0; - sw_get_dst.Stop(); - - sw_get_cmp.Start(); - string? err_string = null; - if (s_src.Length != s_dst.Length) - { - err_string = $"The sizes of the files do not match: {s_src.Length} != {s_dst.Length}."; - } - - using var hasher = HashFactory.CreateHasher("SHA256"); - var dsthash = Convert.ToBase64String(hasher.ComputeHash(s_dst)); - - if (await srchashtask.ConfigureAwait(false) != dsthash) - { - err_string = (err_string is null ? "" : err_string + " ") + "The contents of the files do not match."; - } - sw_get_cmp.Stop(); - - if (err_string is not null) - { - throw new Exception(err_string); - } - } - } - - successful_copies++; - } - catch (Exception e) - { - Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e, - "Error copying {0}: {1}", f.Name, e.Message); - errors.Add(f); - } - finally - { - i++; - - // Stop any running timers - sw_get_src.Stop(); - sw_put_dst.Stop(); - sw_get_dst.Stop(); - sw_get_cmp.Stop(); - } - } - - if (config.Progress) - Console.WriteLine($"\rCopying: {n}/{n}"); - - Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", - "Copy | Get source: {0} ms, Put destination: {1} ms, Get destination: {2} ms, Get compare: {3} ms", - TimeSpan.FromMilliseconds(sw_get_src.ElapsedMilliseconds), - TimeSpan.FromMilliseconds(sw_put_dst.ElapsedMilliseconds), - TimeSpan.FromMilliseconds(sw_get_dst.ElapsedMilliseconds), - TimeSpan.FromMilliseconds(sw_get_cmp.ElapsedMilliseconds)); - - return (successful_copies, errors); - } - - /// - /// Deletes the files from a backend. - /// - /// The backend to delete the files from. - /// The files to delete. - /// The parsed configuration for the tool. - /// The cancellation token to use for the asynchronous operations. - /// The number of successful deletions. - private static async Task DeleteAsync(LightWeightBackendManager b, IEnumerable files, Config config, CancellationToken token) - { - long successful_deletes = 0; - long i = 0, n = files.Count(); - using var timer = new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Delete operation"); - - foreach (var f in files) - { - if (n > 1 && config.Progress) - { - Console.Write($"\rDeleting: {i}/{n}"); - } - - Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", - "Deleting {0} from {1}", f.Name, b.DisplayName); - - try - { - if (config.DryRun) - { - Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", - "Would delete {0} from {1}", f.Name, b.DisplayName); - } - else - { - await b.DeleteAsync(f.Name, token).ConfigureAwait(false); - } - successful_deletes++; - } - catch (Exception e) - { - Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e, - "Error deleting {0}: {1}", f.Name, e.Message); - } - - i++; - } - - if (config.Progress) - Console.WriteLine($"\rDeleting: {n}/{n}"); - return successful_deletes; - } - - /// - /// Creates an option that allows multiple tokens and multiple arguments per token. - /// - /// The aliases for the option. - /// The description for the option. - /// The created option. - private static Option> OptionWithMultipleTokens(string[] aliases, string description, Func> getDefaultValue) - { - return new Option>(aliases: aliases, description: description, getDefaultValue: getDefaultValue) - { - Arity = ArgumentArity.OneOrMore, - AllowMultipleArgumentsPerToken = true - }; - } - - /// - /// Parses the options from a list of strings. - /// Each option should be in the format "key=value". If the value contains spaces, - /// it should be enclosed in quotes, e.g. "key=\"value with spaces\"". - /// - /// The list of string options to parse - /// A dictionary with the parsed options, where the key is the option name and the value is the option value. - /// If an option was not parsed correctly. - private static Dictionary ParseOptions(IEnumerable options) - { - var result = options - .Select(x => x.Split('=')) - .ToDictionary(x => x[0], x => string.Join("=", x.Skip(1))); - - // Double check that the options are valid by reconstructing them from the dictionary - foreach (var opt in result.Select(x => $"{x.Key}={x.Value}")) - { - if (!options.Contains(opt)) - { - Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", null, - "The source option '{0}' is not valid. Please check the syntax.", opt); - throw new ArgumentException($"The source option '{opt}' has not been parsed correctly."); - } - } - - return result; - } - - /// - /// Prepares the lists of files to copy, delete and verify. - /// The files to copy are the files that are not in the destination, have a different size or have a more recent modification date. - /// The files to delete are the files that are found in the destination but not found in the source. - /// The files to verify are the files that are found in both the source and the destination, and that have the same size and modification date. - /// - /// The source lightweight backend manager. - /// The destination lightweight backend manager. - /// The parsed configuration for the tool. - /// The cancellation token to use for the asynchronous operations. - /// A tuple of Lists each holding the files to copy, delete and verify. - private static async Task<(IEnumerable, IEnumerable, IEnumerable)> PrepareFileLists(LightWeightBackendManager b_src, LightWeightBackendManager b_dst, Config config, CancellationToken token) - { - IEnumerable files_src, files_dst; - - using (new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Prepare | List source")) - files_src = await b_src.ListAsync(token).ConfigureAwait(false); - - using (new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Prepare | List destination")) - files_dst = await b_dst.ListAsync(token).ConfigureAwait(false); - - // Shortcut for force - if (config.Force) - { - return (files_src, files_dst, []); - } - - // Shortcut for empty destination - if (!files_dst.Any()) - { - return (files_src, [], []); - } - - Dictionary lookup_src, lookup_dst; - using (new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Prepare | Build lookup for source and destination")) - { - lookup_src = files_src.ToDictionary(x => x.Name); - lookup_dst = files_dst.ToDictionary(x => x.Name); - } - - var to_copy = new List(); - var to_delete = new HashSet(); - var to_verify = new List(); - - // Find all of the files in src that are not in dst, where the dst has a different size than src or src a more recent modification date than dst - using (new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Prepare | Check the files that are present in source against destination")) - foreach (var f_src in files_src) - { - if (lookup_dst.TryGetValue(f_src.Name, out var f_dst)) - { - if (f_src.Size != f_dst.Size || f_src.LastModification > f_dst.LastModification) - { - // The file is different, so we need to copy it - to_copy.Add(f_src); - to_delete.Add(f_dst.Name); - } - else - { - // The file seems to be the same, so we need to verify it if the user wants to - to_verify.Add(f_src); - } - } - else - { - // The file is not in the destination, so we need to copy it - to_copy.Add(f_src); - } - } - - // Find all of the files in dst that are not in src - using (new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Prepare | Check the files that are present in destination against source")) - foreach (var f_dst in files_dst) - { - if (to_delete.Contains(f_dst.Name)) - continue; - - if (!lookup_src.ContainsKey(f_dst.Name)) - { - to_delete.Add(f_dst.Name); - } - } - - List to_delete_lookedup; - using (new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Prepare | Lookup the files to delete")) - to_delete_lookedup = [.. to_delete.Select(x => lookup_dst[x])]; - - return (to_copy, to_delete_lookedup, to_verify); - } - - /// - /// Renames the files in a backend. - /// The renaming is done by deleting the file and re-uploading it with a new name. - /// - /// The lightweight backend manager to issue rename operations to. - /// The files to rename. - /// The parsed configuration for the tool. - /// The cancellation token to use for the asynchronous operations. - /// The number of successful renames. - private static async Task RenameAsync(LightWeightBackendManager bm, IEnumerable files, Config config, CancellationToken token) - { - long successful_renames = 0; - string prefix = $"{System.DateTime.UtcNow:yyyyMMddHHmmss}.old"; - using var downloaded = new MemoryStream(); - long i = 0, n = files.Count(); - - var sw = new System.Diagnostics.Stopwatch(); - - foreach (var f in files) - { - if (config.Progress) - Console.Write($"\rRenaming: {i}/{n}"); - - Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", - "Renaming {0} to {1}.{0} by calling Rename on {2}", - f.Name, prefix, bm.DisplayName); - - try - { - if (config.DryRun) - { - Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", - "Would rename {0} to {1}.{0} by calling Rename on {2}", - f.Name, prefix, bm.DisplayName); - } - else - { - sw.Start(); - await bm.RenameAsync(f.Name, $"{prefix}.{f.Name}", token).ConfigureAwait(false); - sw.Stop(); - } - successful_renames++; - } - catch (Exception e) - { - Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e, - "Error renaming {0}: {1}", f.Name, e.Message); - } - finally - { - // Ensure the timer is stopped - sw.Stop(); - } - - i++; - } - - Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", - "Rename: {0} ms", - TimeSpan.FromMilliseconds(sw.ElapsedMilliseconds)); - - if (config.Progress) - Console.WriteLine($"\rRenaming: {n}/{n}"); - - return successful_renames; - } - - /// - /// Verifies the files in the destination backend. - /// The verification is done by downloading the files from the destination backend and comparing them to the source files. - /// - /// The source lightweight backend manager. - /// The destination lightweight backend manager. - /// The files to verify. - /// The parsed configuration for the tool. - /// The cancellation token to use for the asynchronous operations. - /// A list of the files that failed verification. - private static async Task> VerifyAsync(LightWeightBackendManager b_src, LightWeightBackendManager b_dst, IEnumerable files, Config config, CancellationToken token) - { - var errors = new List(); - using var s_src = new MemoryStream(); - using var s_dst = new MemoryStream(); - long i = 0, n = files.Count(); - var sw_get = new System.Diagnostics.Stopwatch(); - var sw_cmp = new System.Diagnostics.Stopwatch(); - - foreach (var f in files) - { - if (config.Progress) - Console.Write($"\rVerifying: {i}/{n}"); - - Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", - "Verifying {0} by downloading and comparing {1} bytes from {2} and {3}", - f.Name, - Duplicati.Library.Utility.Utility.FormatSizeString(s_src.Length), - b_dst.DisplayName, b_src.DisplayName); - - try - { - // Get both files - sw_get.Start(); - var fs = b_src.GetAsync(f.Name, s_src, token); - var ds = b_dst.GetAsync(f.Name, s_dst, token); - await Task.WhenAll(fs, ds).ConfigureAwait(false); - sw_get.Stop(); - - // Compare the contents - sw_cmp.Start(); - if (s_src.Length != s_dst.Length || !s_src.ToArray().SequenceEqual(s_dst.ToArray())) - { - errors.Add(f); - } - sw_cmp.Stop(); - } - catch (Exception e) - { - errors.Add(f); - Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e, - "Error during verification of {0}: {1}", f.Name, e.Message); - } - finally - { - // Reset the streams - s_src.SetLength(0); - s_dst.SetLength(0); - - // Stop any running timers - sw_get.Stop(); - sw_cmp.Stop(); - } - - i++; - } - - Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", - "Verify | Get: {0} ms, Compare: {1} ms", - TimeSpan.FromMilliseconds(sw_get.ElapsedMilliseconds), - TimeSpan.FromMilliseconds(sw_cmp.ElapsedMilliseconds)); - - if (config.Progress) - Console.WriteLine($"\rVerifying: {n}/{n}"); - - return errors; - } - + public static Task Main(string[] args) + => RemoteSynchronizationRunner.RunAsync(args); } } diff --git a/Tools/RemoteSynchronization/RemoteSynchronization.csproj b/Tools/RemoteSynchronization/RemoteSynchronization.csproj index 9565912f3..5fcdc0d82 100644 --- a/Tools/RemoteSynchronization/RemoteSynchronization.csproj +++ b/Tools/RemoteSynchronization/RemoteSynchronization.csproj @@ -8,13 +8,7 @@ - - - - - - - +