duplicati/Duplicati/Library/Main/Operation/RemoteSynchronization/RemoteSynchronizationRunner.cs

// Copyright (C) 2026, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.

using System;
using System.Collections.Generic;
using System.CommandLine;
using System.CommandLine.NamingConventionBinder;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.Logging;
using Duplicati.Library.Main.Backend;
using Duplicati.Library.Utility;

namespace Duplicati.Library.Main.Operation.RemoteSynchronization;

#nullable enable

/// <summary>
/// Remote synchronization tool.
/// </summary>
public static class RemoteSynchronizationRunner
{

    /// <summary>
    /// The log tag for this tool.
    /// </summary>
    private static readonly string LOGTAG = Duplicati.Library.Logging.Log.LogTagFromType(typeof(RemoteSynchronizationRunner));

    /// <summary>
    /// Entry point for running the tool logic with parsed arguments.
    /// </summary>
    /// <param name="args">The commandline arguments</param>
    /// <returns>0 on success, -1 on abort, and the number of errors encountered otherwise.</returns>
    public static async Task<int> RunAsync(string[] args)
    {
        var arg_src = new Argument<string>(name: "backend_src", description: Strings.SourceBackendDescription);
        var arg_dst = new Argument<string>(name: "backend_dst", description: Strings.DestinationBackendDescription);

        var root_cmd = new RootCommand(Strings.RootCommandDescription)
            {
                arg_src,
                arg_dst,

                new Option<bool>(aliases: ["--auto-create-folders"], description: Strings.AutoCreateFoldersDescription, getDefaultValue: () => true),
                new Option<int>(aliases: ["--backend-retries"], description: Strings.BackendRetriesDescription, getDefaultValue: () => 3) { Arity = ArgumentArity.ExactlyOne },
                new Option<int>(aliases: ["--backend-retry-delay"], description: Strings.BackendRetryDelayDescription, getDefaultValue: () => 1000) { Arity = ArgumentArity.ExactlyOne },
                new Option<bool>(aliases: ["--backend-retry-with-exponential-backoff"], description: Strings.BackendRetryWithExponentialBackoffDescription, getDefaultValue: () => true),
                new Option<bool>(aliases: ["--confirm", "--yes", "-y"], description: Strings.ConfirmDescription, getDefaultValue: () => false),
                new Option<bool>(aliases: ["--dry-run", "-d"], description: Strings.DryRunDescription, getDefaultValue: () => false),
                OptionWithMultipleTokens(aliases: ["--dst-options"], description: Strings.DstOptionsDescription, getDefaultValue: () => []),
                new Option<bool>(aliases: ["--force", "-f"], description: Strings.ForceDescription, getDefaultValue: () => false),
                OptionWithMultipleTokens(aliases: ["--global-options"], description: Strings.GlobalOptionsDescription, getDefaultValue: () => []),
                new Option<string>(aliases: ["--log-file"], description: Strings.LogFileDescription, getDefaultValue: () => "") { Arity = ArgumentArity.ExactlyOne },
                new Option<string>(aliases: ["--log-level"], description: Strings.LogLevelDescription, getDefaultValue: () => "Information") { Arity = ArgumentArity.ExactlyOne },
                new Option<bool>(aliases: ["--parse-arguments-only"], description: Strings.ParseArgumentsOnlyDescription, getDefaultValue: () => false),
                new Option<bool>(aliases: ["--progress"], description: Strings.ProgressDescription, getDefaultValue: () => false),
                new Option<bool>(aliases: ["--retention"], description: Strings.RetentionDescription, getDefaultValue: () => false),
                new Option<int>(aliases: ["--retry"], description: Strings.RetryDescription, getDefaultValue: () => 3) { Arity = ArgumentArity.ExactlyOne },
                OptionWithMultipleTokens(aliases: ["--src-options"], description: Strings.SrcOptionsDescription, getDefaultValue: () => []),
                new Option<bool>(aliases: ["--verify-contents"], description: Strings.VerifyContentsDescription, getDefaultValue: () => false),
                new Option<bool>(aliases: ["--verify-get-after-put"], description: Strings.VerifyGetAfterPutDescription, getDefaultValue: () => false),
            };

        root_cmd.Handler = CommandHandler.Create((string backend_src, string backend_dst, RemoteSynchronizationConfig config, CancellationToken token) =>
        {
            var config_with_args = config with { Dst = backend_dst, Src = backend_src };

            return RunAsync(config_with_args, token);
        });

        return await root_cmd.InvokeAsync(args).ConfigureAwait(false);
    }

    /// <summary>
    /// The main logic of the tool.
    /// </summary>
    /// <param name="config">The parsed configuration for the tool.</param>
    /// <param name="token">The cancellation token to use for the asynchronous operations.</param>
    /// <param name="progressUpdater">Optional progress updater for reporting file count and transfer progress to the UI.</param>
    /// <param name="backendProgressUpdater">Optional backend progress updater for reporting transfer speed to the UI.</param>
    /// <returns>The return code (0 on success).</returns>
    internal static async Task<int> RunAsync(RemoteSynchronizationConfig config, CancellationToken token, IOperationProgressUpdater? progressUpdater = null, IBackendProgressUpdater? backendProgressUpdater = null, IBasicResults? results = null)
    {
        // Parse the log level
        var log_level_parsed = Enum.TryParse<Duplicati.Library.Logging.LogMessageType>(config.LogLevel, true, out var log_level_enum);
        log_level_enum = log_level_parsed ? log_level_enum : Duplicati.Library.Logging.LogMessageType.Information;
        var rsyncResults = results as RemoteSynchronizationResults;

        if (!string.IsNullOrEmpty(config.LogFile))
        {
            using var console_sink = new StreamLogDestination(Console.OpenStandardOutput());

            // Parse the log file
            StreamLogDestination? log_file_sink = null;
            if (!string.IsNullOrEmpty(config.LogFile))
            {
                string log_file_dir = SystemIO.IO_OS.PathGetDirectoryName(config.LogFile);
                if (!string.IsNullOrEmpty(log_file_dir) && !SystemIO.IO_OS.DirectoryExists(log_file_dir))
                    SystemIO.IO_OS.DirectoryCreate(log_file_dir);
                log_file_sink = new StreamLogDestination(config.LogFile);
            }

            if (rsyncResults is not null)
            {
                using var multi_log_target = new ControllerMultiLogTarget(rsyncResults, log_level_enum, null, null);
                multi_log_target.AddTarget(console_sink, log_level_enum, null);
                multi_log_target.AddTarget(log_file_sink, log_level_enum, null);

                // Start the logging scope with both file logging and message capture
                using var scope = Duplicati.Library.Logging.Log.StartScope(multi_log_target.WriteMessage);

                return await RunCoreAsync(config, token, progressUpdater, backendProgressUpdater, rsyncResults).ConfigureAwait(false);
            }
            else
            {
                using var multi_sink = new MultiLogDestination([console_sink, log_file_sink]);

                // Start the logging scope
                using var _ = Duplicati.Library.Logging.Log.StartScope(multi_sink, log_level_enum);

                return await RunCoreAsync(config, token, progressUpdater, backendProgressUpdater).ConfigureAwait(false);
            }
        }
        else
        {
            if (rsyncResults is not null)
            {
                // Set up logging to capture messages to the results even without a log file
                using var log_target = new ControllerMultiLogTarget(rsyncResults, log_level_enum, null, null);
                using var scope = Duplicati.Library.Logging.Log.StartScope(log_target.WriteMessage);

                return await RunCoreAsync(config, token, progressUpdater, backendProgressUpdater, rsyncResults).ConfigureAwait(false);
            }
            else
            {
                return await RunCoreAsync(config, token, progressUpdater, backendProgressUpdater).ConfigureAwait(false);
            }
        }
    }

    /// <summary>
    /// The core logic of the tool, without logging setup.
    /// </summary>
    /// <param name="config">The parsed configuration for the tool.</param>
    /// <param name="token">The cancellation token to use for the asynchronous operations.</param>
    /// <param name="progressUpdater">Optional progress updater for reporting file count and transfer progress to the UI.</param>
    /// <param name="backendProgressUpdater">Optional backend progress updater for reporting transfer speed to the UI.</param>
    /// <returns>The return code (0 on success).</returns>
    private static async Task<int> RunCoreAsync(RemoteSynchronizationConfig config, CancellationToken token, IOperationProgressUpdater? progressUpdater = null, IBackendProgressUpdater? backendProgressUpdater = null, RemoteSynchronizationResults? results = null)
    {
        // Unpack and parse the multi token options
        var global_options = ParseOptions(config.GlobalOptions);

        var src_opts = ParseOptions(config.SrcOptions);
        var dst_opts = ParseOptions(config.DstOptions);

        // Merge the global options into the source and destination options. The global options will be overridden by the source and destination options.
        foreach (var x in global_options)
        {
            if (!src_opts.ContainsKey(x.Key))
                src_opts[x.Key] = x.Value;
            if (!dst_opts.ContainsKey(x.Key))
                dst_opts[x.Key] = x.Value;
        }

        // Check if we only had to parse the arguments
        if (config.ParseArgumentsOnly)
        {
            Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "ArgumentsParsed", "Arguments parsed successfully; {0}; exiting.", config);
            return 0;
        }

        using var b1m = new LightWeightBackendManager(config.Src, src_opts, config.BackendRetries, config.BackendRetryDelay, config.BackendRetryWithExponentialBackoff, progressUpdater: progressUpdater, backendProgressUpdater: backendProgressUpdater);
        using var b2m = new LightWeightBackendManager(config.Dst, dst_opts, config.BackendRetries, config.BackendRetryDelay, config.BackendRetryWithExponentialBackoff, progressUpdater: progressUpdater, backendProgressUpdater: backendProgressUpdater);

        // Prepare the operations
        var (to_copy, to_delete, to_verify) = await PrepareFileListsAsync(b1m, b2m, config, token).ConfigureAwait(false);
        var disableQuota = Library.Utility.Utility.ParseBoolOption(dst_opts, "quota-disable");

        // Check if we have enough free space in the destination to perform the synchronization.
        var dst_quota = disableQuota ? null : await b2m.GetQuotaInfoAsync(token).ConfigureAwait(false);
        if (dst_quota is not null)
        {
            var total_delete_size = to_delete.Sum(x => Math.Max(x.Size, 0));
            var total_copy_size = to_copy.Sum(x => Math.Max(x.Size, 0));
            if (dst_quota.FreeQuotaSpace < total_copy_size - total_delete_size)
            {
                Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "NotEnoughDestinationSpace", null,
                    "Not enough free space in destination to perform the synchronization. Required: {0}, Available: {1}. Aborting.",
                    Duplicati.Library.Utility.Utility.FormatSizeString(total_copy_size - total_delete_size),
                    Duplicati.Library.Utility.Utility.FormatSizeString(dst_quota.FreeQuotaSpace));
                throw new UserInformationException("Not enough free space in destination to perform the synchronization.", "NotEnoughDestinationSpace");
            }
        }

        // Verify the files if requested. If the files are not verified, they will be deleted and copied again.
        long verified = 0, failed_verify = 0;
        if (config.VerifyContents)
        {
            // As this is a potentially slow operation, ask for confirmation of the verification)
            if (!config.Confirm)
            {
                Console.WriteLine($"This will verify {to_verify.Count()} files before copying them. Do you want to continue? [y/N]");
                var response = Console.ReadLine();
                if (!response?.Equals("y", StringComparison.CurrentCultureIgnoreCase) ?? true)
                {
                    Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "AbortedByUser", "Aborted");
                    return -1;
                }
            }

            var not_verified = await VerifyAsync(b1m, b2m, to_verify, config, token).ConfigureAwait(false);
            failed_verify = not_verified.Count();
            verified = to_verify.Count() - failed_verify;

            if (not_verified.Any())
            {
                Duplicati.Library.Logging.Log.WriteWarningMessage(LOGTAG, "VerificationFailed", null,
                    "{0} files failed verification. They will be deleted and copied again.",
                    not_verified.Count());
                to_delete = to_delete.Concat(not_verified);
                to_copy = to_copy.Concat(not_verified);
            }
        }

        Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "SynchronizationPlan",
            "The remote synchronization plan is to {0} {1} files from {2}, then copy {3} files from {4} to {2}.",
            config.Retention ? "rename" : "delete",
            to_delete.Count(), b2m.DisplayName, to_copy.Count(), b1m.DisplayName);

        // As this is a potentially destructive operation, ask for confirmation
        if (!config.Confirm)
        {
            var delete_rename = config.Retention ? "Rename" : "Delete";
            Console.WriteLine($"This will perform the following actions (in order):");
            Console.WriteLine($"    - {delete_rename} {to_delete.Count()} files from {config.Dst}");
            Console.WriteLine($"    - Copy {to_copy.Count()} files from {config.Src} to {config.Dst}");
            Console.WriteLine();
            Console.WriteLine("Do you want to continue? [y/N]");

            var response = Console.ReadLine();
            if (!response?.Equals("y", StringComparison.CurrentCultureIgnoreCase) ?? true)
            {
                Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "AbortedByUser", "Aborted");
                return -1;
            }
        }

        // Report total file count to the progress updater (delete + copy operations)
        var totalFileCount = to_delete.Count() + to_copy.Count();
        var totalFileSize = to_copy.Sum(f => Math.Max(f.Size, 0));
        if (progressUpdater != null)
        {
            progressUpdater.UpdatefileCount(totalFileCount, totalFileSize, true);
            progressUpdater.UpdatefilesProcessed(0, 0);
        }

        // Delete or rename the files that are not needed
        long renamed = 0, deleted = 0;
        var deleteCount = to_delete.Count();
        if (config.Retention)
        {
            renamed = await RenameAsync(b2m, to_delete, config, token, 0, totalFileCount, progressUpdater, backendProgressUpdater).ConfigureAwait(false);
            Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "RenameComplete",
                "Renamed {0} files in {1}", renamed, b2m.DisplayName);
        }
        else
        {
            deleted = await DeleteAsync(b2m, to_delete, config, token, 0, totalFileCount, progressUpdater, backendProgressUpdater).ConfigureAwait(false);
            Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "DeleteComplete",
                "Deleted {0} files from {1}", deleted, b2m.DisplayName);
        }

        // The delete/rename phase now reports progress incrementally, so we don't need a separate update here
        var deletedOrRenamed = Math.Max(deleted, renamed);

        // Copy the files
        var (copied, copy_errors) = await CopyAsync(b1m, b2m, to_copy, config, deletedOrRenamed, totalFileCount, token, progressUpdater, backendProgressUpdater).ConfigureAwait(false);
        Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "CopyComplete",
            "Copied {0} files from {1} to {2}", copied, b1m.DisplayName, b2m.DisplayName);

        // If there are still errors, retry a few times
        if (copy_errors.Any())
        {
            Duplicati.Library.Logging.Log.WriteWarningMessage(LOGTAG, "CopyErrors", null,
                "Could not copy {0} files.", copy_errors.Count());
            if (config.Retry > 0)
            {
                Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "CopyRetry",
                    "Retrying {0} more times to copy the {1} files that failed",
                    config.Retry, copy_errors.Count());
                for (int i = 0; i < config.Retry; i++)
                {
                    await Task.Delay(5000).ConfigureAwait(false); // Wait 5 seconds before retrying
                    (copied, copy_errors) = await CopyAsync(b1m, b2m, copy_errors, config, deletedOrRenamed, totalFileCount, token, progressUpdater, backendProgressUpdater).ConfigureAwait(false);
                    Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "CopyPartialComplete",
                        "Copied {0} files from {1} to {2}", copied, b1m.DisplayName, b2m.DisplayName);
                    if (!copy_errors.Any())
                        break;
                }
            }

            if (copy_errors.Any())
            {
                Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "CopyFailed", null,
                    "Could not copy {0} files. Not retrying any more.", copy_errors.Count());
                results?.DeletedFileCount = deleted;
                results?.RenamedFileCount = renamed;
                results?.CopiedFileCount = copied;
                results?.VerifiedFileCount = verified;
                results?.FailedVerificationCount = failed_verify;
                results?.CopiedFileSize = totalFileSize;
                return copy_errors.Count();
            }
        }

        // Results reporting
        if (results is not null)
        {
            if (verified > 0)
                Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "VerificationSuccess",
                    "Verified {0} files in {1} that didn't need to be copied",
                    verified, b2m.DisplayName);
            if (failed_verify > 0)
                Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "VerificationFailedRetry",
                    "Failed to verify {0} files in {1}, which were then attempted to be copied",
                    failed_verify, b2m.DisplayName);
            if (copied > 0)
                Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "CopyComplete",
                    "Copied {0} files from {1} to {2}", copied, b1m.DisplayName, b2m.DisplayName);
            if (deleted > 0)
                Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "DeleteComplete",
                    "Deleted {0} files from {1}", deleted, b2m.DisplayName);
            if (renamed > 0)
                Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "RenameComplete",
                    "Renamed {0} files in {1}", renamed, b2m.DisplayName);
        }

        Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "SynchronizationComplete",
            "Remote synchronization completed successfully");

        results?.DeletedFileCount = deleted;
        results?.RenamedFileCount = renamed;
        results?.CopiedFileCount = copied;
        results?.VerifiedFileCount = verified;
        results?.FailedVerificationCount = failed_verify;
        results?.CopiedFileSize = totalFileSize;

        return 0;
    }

    // TODO have concurrency parameters: uploaders, downloaders
    // TODO low memory mode, where things aren't kept in memory. Maybe utilize SQLite?
    // TODO For convenience, have the option to launch a "duplicati test" on the destination backend after the synchronization
    // TODO Save hash to minimize redownload

    /// <summary>
    /// Copies the files from one backend to another.
    /// The files are copied one by one, and each file is verified after uploading if the verify flag is set.
    /// </summary>
    /// <param name="b_src">The source backend.</param>
    /// <param name="b_dst">The destination backend.</param>
    /// <param name="files">The files that will be copied.</param>
    /// <param name="config">The parsed configuration for the tool.</param>
    /// <param name="filesProcessedOffset">The number of files already processed before this copy phase (e.g., from delete/rename), used for progress reporting.</param>
    /// <param name="totalFiles">The total number of files to be processed in the entire operation (delete + copy), used for progress reporting.</param>
    /// <param name="token">The cancellation token to use for the asynchronous operations.</param>
    /// <param name="progressUpdater">Optional progress updater for reporting file-level progress to the UI.</param>
    /// <param name="backendProgressUpdater">Optional backend progress updater for reporting transfer speed to the UI.</param>
    /// <returns>A tuple holding the number of succesful copies and a List of the files that failed.</returns>
    private static async Task<(long, IEnumerable<IFileEntry>)> CopyAsync(LightWeightBackendManager b_src, LightWeightBackendManager b_dst, IEnumerable<IFileEntry> files, RemoteSynchronizationConfig config, long filesProcessedOffset, long totalFiles, CancellationToken token, IOperationProgressUpdater? progressUpdater = null, IBackendProgressUpdater? backendProgressUpdater = null)
    {
        long successful_copies = 0;
        long sizeProcessed = 0;
        List<IFileEntry> errors = [];
        long i = 0, n = files.Count();
        var sw_get_src = new System.Diagnostics.Stopwatch();
        var sw_put_dst = new System.Diagnostics.Stopwatch();
        var sw_get_dst = new System.Diagnostics.Stopwatch();
        var sw_get_cmp = new System.Diagnostics.Stopwatch();

        foreach (var f in files)
        {
            if (config.Progress)
                Console.Write($"\rCopying: {i}/{n}");

            Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "CopyingFile",
                "Copying {0} from {1} to {2}", f.Name, b_src.DisplayName, b_dst.DisplayName);

            try
            {
                // Report downloading current file with real-time progress
                var fileSize = Math.Max(f.Size, 0);
                progressUpdater?.StartFile($"Downloading {f.Name}", fileSize);
                backendProgressUpdater?.StartAction(BackendActionType.Get, f.Name, fileSize);

                sw_get_src.Start();
                using var tmp_src = await b_src.GetAsync(f.Name, token).ConfigureAwait(false);
                sw_get_src.Stop();

                backendProgressUpdater?.EndAction(BackendActionType.Get, f.Name);

                var s_src_length = SystemIO.IO_OS.FileLength(tmp_src);

                if (config.DryRun)
                {
                    Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "DryRunCopy",
                        "Would write {0} bytes of {1} to {2}",
                        Duplicati.Library.Utility.Utility.FormatSizeString(s_src_length),
                        f.Name, b_dst.DisplayName);
                }
                else
                {
                    // Compute hash BEFORE uploading, so we don't need to seek back after PutAsync
                    byte[]? sourceHash = null;
                    if (config.VerifyGetAfterPut)
                    {
                        using var hasher = HashFactory.CreateHasher("SHA256");
                        using var s = SystemIO.IO_OS.FileOpenRead(tmp_src);
                        sourceHash = hasher.ComputeHash(s);
                    }

                    // Report uploading current file with real-time progress
                    progressUpdater?.StartFile($"Uploading {f.Name}", s_src_length);
                    backendProgressUpdater?.StartAction(BackendActionType.Put, f.Name, s_src_length);

                    sw_put_dst.Start();
                    await b_dst.PutAsync(f.Name, tmp_src, token).ConfigureAwait(false);
                    sw_put_dst.Stop();

                    backendProgressUpdater?.EndAction(BackendActionType.Put, f.Name);

                    if (config.VerifyGetAfterPut)
                    {
                        using var s_dst = Duplicati.Library.Utility.TempFileStream.Create();

                        sw_get_dst.Start();
                        await b_dst.GetAsync(f.Name, s_dst, token).ConfigureAwait(false);
                        s_dst.Position = 0;
                        sw_get_dst.Stop();

                        sw_get_cmp.Start();
                        string? err_string = null;
                        if (s_src_length != s_dst.Length)
                        {
                            err_string = $"The sizes of the files do not match: {s_src_length} != {s_dst.Length}.";
                        }

                        using var hasher = HashFactory.CreateHasher("SHA256");
                        var dsthash = Convert.ToBase64String(hasher.ComputeHash(s_dst));

                        if (sourceHash != null && Convert.ToBase64String(sourceHash) != dsthash)
                        {
                            err_string = (err_string is null ? "" : err_string + " ") + "The contents of the files do not match.";
                        }
                        sw_get_cmp.Stop();

                        if (err_string is not null)
                        {
                            throw new Exception(err_string);
                        }
                    }
                }

                successful_copies++;
                sizeProcessed += s_src_length;

                // Report overall progress: files processed so far (offset + current index)
                var total_progress = filesProcessedOffset + successful_copies;
                progressUpdater?.UpdatefilesProcessed(total_progress, sizeProcessed);
                progressUpdater?.UpdateProgress(totalFiles > 0 ? (float)total_progress / totalFiles : 1.0f);
            }
            catch (Exception e)
            {
                Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "CopyError", e,
                    "Error copying {0}: {1}", f.Name, e.Message);
                errors.Add(f);
            }
            finally
            {
                i++;

                // Stop any running timers
                sw_get_src.Stop();
                sw_put_dst.Stop();
                sw_get_dst.Stop();
                sw_get_cmp.Stop();
            }
        }

        if (config.Progress)
            Console.WriteLine($"\rCopying: {n}/{n}");

        Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "CopyTiming",
            "Copy | Get source: {0} ms, Put destination: {1} ms, Get destination: {2} ms, Get compare: {3} ms",
            TimeSpan.FromMilliseconds(sw_get_src.ElapsedMilliseconds),
            TimeSpan.FromMilliseconds(sw_put_dst.ElapsedMilliseconds),
            TimeSpan.FromMilliseconds(sw_get_dst.ElapsedMilliseconds),
            TimeSpan.FromMilliseconds(sw_get_cmp.ElapsedMilliseconds));

        return (successful_copies, errors);
    }

    /// <summary>
    /// Deletes the files from a backend.
    /// </summary>
    /// <param name="b">The backend to delete the files from.</param>
    /// <param name="files">The files to delete.</param>
    /// <param name="config">The parsed configuration for the tool.</param>
    /// <param name="token">The cancellation token to use for the asynchronous operations.</param>
    /// <param name="filesProcessedOffset">The number of files already processed before this delete phase, used for overall progress reporting.</param>
    /// <param name="totalFiles">The total number of files to be processed in the entire operation (delete + copy), used for overall progress reporting.</param>
    /// <param name="progressUpdater">Optional progress updater for reporting file-level progress to the UI.</param>
    /// <param name="backendProgressUpdater">Optional backend progress updater for reporting delete speed to the UI.</param>
    /// <returns>The number of successful deletions.</returns>
    private static async Task<long> DeleteAsync(LightWeightBackendManager b, IEnumerable<IFileEntry> files, RemoteSynchronizationConfig config, CancellationToken token, long filesProcessedOffset, long totalFiles, IOperationProgressUpdater? progressUpdater = null, IBackendProgressUpdater? backendProgressUpdater = null)
    {
        long successful_deletes = 0;
        long i = 0, n = files.Count();
        using var timer = new Duplicati.Library.Logging.Timer(LOGTAG, "DeleteOperation", "Delete operation");

        foreach (var f in files)
        {
            if (n > 1 && config.Progress)
            {
                Console.Write($"\rDeleting: {i}/{n}");
            }

            Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "DeletingFile",
                "Deleting {0} from {1}", f.Name, b.DisplayName);

            try
            {
                // Report delete operation with progress
                var fileSize = Math.Max(f.Size, 0);
                progressUpdater?.StartFile($"Deleting {f.Name}", fileSize);
                backendProgressUpdater?.StartAction(BackendActionType.Delete, f.Name, fileSize);

                if (config.DryRun)
                {
                    Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "DryRunDelete",
                        "Would delete {0} from {1}", f.Name, b.DisplayName);
                }
                else
                {
                    using (var timer_delete = new Duplicati.Library.Logging.Timer(LOGTAG, "DeleteFile", $"Delete {f.Name}"))
                        await b.DeleteAsync(f.Name, token).ConfigureAwait(false);
                }
                successful_deletes++;

                backendProgressUpdater?.EndAction(BackendActionType.Delete, f.Name);

                // Report overall progress: files processed so far (with offset for overall progress)
                var total_progress = filesProcessedOffset + successful_deletes;
                progressUpdater?.UpdatefilesProcessed(total_progress, 0);
                progressUpdater?.UpdateProgress(totalFiles > 0 ? (float)total_progress / totalFiles : 1.0f);
            }
            catch (Exception e)
            {
                Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "DeleteError", e,
                    "Error deleting {0}: {1}", f.Name, e.Message);
            }
            finally
            {
                i++;
            }
        }

        if (config.Progress)
            Console.WriteLine($"\rDeleting: {n}/{n}");
        return successful_deletes;
    }

    /// <summary>
    /// Creates an option that allows multiple tokens and multiple arguments per token.
    /// </summary>
    /// <param name="aliases">The aliases for the option.</param>
    /// <param name="description">The description for the option.</param>
    /// <returns>The created option.</returns>
    private static Option<List<string>> OptionWithMultipleTokens(string[] aliases, string description, Func<List<string>> getDefaultValue)
    {
        return new Option<List<string>>(aliases: aliases, description: description, getDefaultValue: getDefaultValue)
        {
            Arity = ArgumentArity.OneOrMore,
            AllowMultipleArgumentsPerToken = true
        };
    }

    /// <summary>
    /// A log destination that writes to multiple destinations.
    /// </summary>
    private sealed class MultiLogDestination : ILogDestination, IDisposable
    {
        private readonly List<ILogDestination> m_destinations;

        /// <summary>
        /// Initializes a new instance of the MultiLogDestination class.
        /// </summary>
        /// <param name="destinations">The log destinations to write to.</param>
        public MultiLogDestination(IEnumerable<ILogDestination?> destinations)
        {
            m_destinations = destinations.Where(x => x != null).Cast<ILogDestination>().ToList();
        }

        /// <summary>
        /// Writes a log entry to all destinations.
        /// </summary>
        /// <param name="entry">The log entry to write.</param>
        public void WriteMessage(LogEntry entry)
        {
            foreach (var destination in m_destinations)
                destination.WriteMessage(entry);
        }

        /// <summary>
        /// Disposes of the resources used by this instance.
        /// </summary>
        public void Dispose()
        {
            foreach (var destination in m_destinations)
                if (destination is IDisposable disposable)
                    disposable.Dispose();
        }
    }

    /// <summary>
    /// Parses the options from a list of strings.
    /// Each option should be in the format "key=value". If the value contains spaces,
    /// it should be enclosed in quotes, e.g. "key=\"value with spaces\"".
    /// </summary>
    /// <param name="options">The list of string options to parse</param>
    /// <returns>A dictionary with the parsed options, where the key is the option name and the value is the option value.</returns>
    /// <exception cref="ArgumentException">If an option was not parsed correctly.</exception>
    private static Dictionary<string, string?> ParseOptions(IEnumerable<string> options)
    {
        var result = options
            .Select(x => x.Split('='))
            .ToDictionary(x => x[0], x => (string?)string.Join("=", x.Skip(1)));

        // Double check that the options are valid by reconstructing them from the dictionary
        foreach (var opt in result.Select(x => $"{x.Key}={x.Value}"))
        {
            if (!options.Contains(opt))
            {
                Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "InvalidOption", null,
                    "The source option '{0}' is not valid. Please check the syntax.", opt);
                throw new ArgumentException($"The source option '{opt}' has not been parsed correctly.");
            }
        }

        return result;
    }

    /// <summary>
    /// Prepares the lists of files to copy, delete and verify.
    /// The files to copy are the files that are not in the destination, have a different size or have a more recent modification date.
    /// The files to delete are the files that are found in the destination but not found in the source.
    /// The files to verify are the files that are found in both the source and the destination, and that have the same size and modification date.
    /// </summary>
    /// <param name="b_src">The source lightweight backend manager.</param>
    /// <param name="b_dst">The destination lightweight backend manager.</param>
    /// <param name="config">The parsed configuration for the tool.</param>
    /// <param name="token">The cancellation token to use for the asynchronous operations.</param>
    /// <returns>A tuple of Lists each holding the files to copy, delete and verify.</returns>
    private static async Task<(IEnumerable<IFileEntry>, IEnumerable<IFileEntry>, IEnumerable<IFileEntry>)> PrepareFileListsAsync(LightWeightBackendManager b_src, LightWeightBackendManager b_dst, RemoteSynchronizationConfig config, CancellationToken token)
    {
        IEnumerable<IFileEntry> files_src, files_dst;

        using (new Duplicati.Library.Logging.Timer(LOGTAG, "ListSource", "Prepare | List source"))
            files_src = await b_src.ListAsync(token).ConfigureAwait(false);

        using (new Duplicati.Library.Logging.Timer(LOGTAG, "ListDestination", "Prepare | List destination"))
            files_dst = await b_dst.ListAsync(token).ConfigureAwait(false);

        // Shortcut for force
        if (config.Force)
        {
            return (files_src, files_dst, []);
        }

        // Shortcut for empty destination
        if (!files_dst.Any())
        {
            return (files_src, [], []);
        }

        Dictionary<string, IFileEntry> lookup_src, lookup_dst;
        using (new Duplicati.Library.Logging.Timer(LOGTAG, "BuildLookup", "Prepare | Build lookup for source and destination"))
        {
            lookup_src = files_src.ToDictionary(x => x.Name);
            lookup_dst = files_dst.ToDictionary(x => x.Name);
        }

        var to_copy = new List<IFileEntry>();
        var to_delete = new HashSet<string>();
        var to_verify = new List<IFileEntry>();

        // Find all of the files in src that are not in dst, where the dst has a different size than src or src a more recent modification date than dst
        using (new Duplicati.Library.Logging.Timer(LOGTAG, "CheckSourceAgainstDestination", "Prepare | Check the files that are present in source against destination"))
            foreach (var f_src in files_src)
            {
                if (lookup_dst.TryGetValue(f_src.Name, out var f_dst))
                {
                    if (f_src.Size != f_dst.Size || f_src.LastModification > f_dst.LastModification)
                    {
                        // The file is different, so we need to copy it
                        to_copy.Add(f_src);
                        to_delete.Add(f_dst.Name);
                    }
                    else
                    {
                        // The file seems to be the same, so we need to verify it if the user wants to
                        to_verify.Add(f_src);
                    }
                }
                else
                {
                    // The file is not in the destination, so we need to copy it
                    to_copy.Add(f_src);
                }
            }

        // Find all of the files in dst that are not in src
        using (new Duplicati.Library.Logging.Timer(LOGTAG, "CheckDestinationAgainstSource", "Prepare | Check the files that are present in destination against source"))
            foreach (var f_dst in files_dst)
            {
                if (to_delete.Contains(f_dst.Name))
                    continue;

                if (!lookup_src.ContainsKey(f_dst.Name))
                {
                    to_delete.Add(f_dst.Name);
                }
            }

        List<IFileEntry> to_delete_lookedup;
        using (new Duplicati.Library.Logging.Timer(LOGTAG, "LookupFilesToDelete", "Prepare | Lookup the files to delete"))
            to_delete_lookedup = [.. to_delete.Select(x => lookup_dst[x])];

        return (to_copy, to_delete_lookedup, to_verify);
    }

    /// <summary>
    /// Renames the files in a backend.
    /// The renaming is done by deleting the file and re-uploading it with a new name.
    /// </summary>
    /// <param name="bm">The lightweight backend manager to issue rename operations to.</param>
    /// <param name="files">The files to rename.</param>
    /// <param name="config">The parsed configuration for the tool.</param>
    /// <param name="token">The cancellation token to use for the asynchronous operations.</param>
    /// <param name="filesProcessedOffset">The number of files already processed before this rename phase, used for overall progress reporting.</param>
    /// <param name="totalFiles">The total number of files to be processed in the entire operation (delete + copy), used for progress reporting.</param>
    /// <param name="progressUpdater">Optional progress updater for reporting file-level progress to the UI.</param>
    /// <param name="backendProgressUpdater">Optional backend progress updater for reporting transfer speed to the UI.</param>
    /// <returns>The number of successful renames.</returns>
    private static async Task<long> RenameAsync(LightWeightBackendManager bm, IEnumerable<IFileEntry> files, RemoteSynchronizationConfig config, CancellationToken token, long filesProcessedOffset, long totalFiles, IOperationProgressUpdater? progressUpdater = null, IBackendProgressUpdater? backendProgressUpdater = null)
    {
        long successful_renames = 0;
        string prefix = $"{System.DateTime.UtcNow:yyyyMMddHHmmss}.old";
        using var downloaded = new MemoryStream();
        long i = 0, n = files.Count();

        var sw = new System.Diagnostics.Stopwatch();

        foreach (var f in files)
        {
            if (config.Progress)
                Console.Write($"\rRenaming: {i}/{n}");

            Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "RenamingFile",
                "Renaming {0} to {1}.{0} by calling Rename on {2}",
                f.Name, prefix, bm.DisplayName);

            try
            {
                // Report rename operation with progress
                var fileSize = Math.Max(f.Size, 0);
                progressUpdater?.StartFile($"Renaming {f.Name}", fileSize);
                // Use Delete action type since rename typically involves delete operation
                backendProgressUpdater?.StartAction(BackendActionType.Delete, f.Name, fileSize);

                if (config.DryRun)
                {
                    Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "DryRunRename",
                        "Would rename {0} to {1}.{0} by calling Rename on {2}",
                        f.Name, prefix, bm.DisplayName);
                }
                else
                {
                    sw.Start();
                    await bm.RenameAsync(f.Name, $"{prefix}.{f.Name}", token).ConfigureAwait(false);
                    sw.Stop();
                }
                successful_renames++;

                backendProgressUpdater?.EndAction(BackendActionType.Delete, f.Name);

                // Report overall progress: files processed so far (with offset for overall progress)
                var total_progress = filesProcessedOffset + successful_renames;
                progressUpdater?.UpdatefilesProcessed(total_progress, 0);
                progressUpdater?.UpdateProgress(totalFiles > 0 ? (float)total_progress / totalFiles : 1.0f);
            }
            catch (Exception e)
            {
                Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "RenameError", e,
                    "Error renaming {0}: {1}", f.Name, e.Message);
            }
            finally
            {
                // Ensure the timer is stopped
                sw.Stop();

                i++;
            }
        }

        Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "RenameTiming",
            "Rename: {0} ms",
            TimeSpan.FromMilliseconds(sw.ElapsedMilliseconds));

        if (config.Progress)
            Console.WriteLine($"\rRenaming: {n}/{n}");

        return successful_renames;
    }

    /// <summary>
    /// Verifies the files in the destination backend.
    /// The verification is done by downloading the files from the destination backend and comparing them to the source files.
    /// </summary>
    /// <param name="b_src">The source lightweight backend manager.</param>
    /// <param name="b_dst">The destination lightweight backend manager.</param>
    /// <param name="files">The files to verify.</param>
    /// <param name="config">The parsed configuration for the tool.</param>
    /// <param name="token">The cancellation token to use for the asynchronous operations.</param>
    /// <returns>A list of the files that failed verification.</returns>
    private static async Task<IEnumerable<IFileEntry>> VerifyAsync(LightWeightBackendManager b_src, LightWeightBackendManager b_dst, IEnumerable<IFileEntry> files, RemoteSynchronizationConfig config, CancellationToken token)
    {
        var errors = new List<IFileEntry>();
        using var s_src = new MemoryStream();
        using var s_dst = new MemoryStream();
        long i = 0, n = files.Count();
        var sw_get = new System.Diagnostics.Stopwatch();
        var sw_cmp = new System.Diagnostics.Stopwatch();

        foreach (var f in files)
        {
            if (config.Progress)
                Console.Write($"\rVerifying: {i}/{n}");

            Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "VerifyingFile",
                "Verifying {0} by downloading and comparing {1} bytes from {2} and {3}",
                f.Name,
                Duplicati.Library.Utility.Utility.FormatSizeString(s_src.Length),
                b_dst.DisplayName, b_src.DisplayName);

            try
            {
                // Get both files
                sw_get.Start();
                var fs = b_src.GetAsync(f.Name, s_src, token);
                var ds = b_dst.GetAsync(f.Name, s_dst, token);
                await Task.WhenAll(fs, ds).ConfigureAwait(false);
                sw_get.Stop();

                // Compare the contents
                sw_cmp.Start();
                if (s_src.Length != s_dst.Length || !s_src.ToArray().SequenceEqual(s_dst.ToArray()))
                {
                    errors.Add(f);
                }
                sw_cmp.Stop();
            }
            catch (Exception e)
            {
                errors.Add(f);
                Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "VerificationError", e,
                    "Error during verification of {0}: {1}", f.Name, e.Message);
            }
            finally
            {
                // Reset the streams
                s_src.SetLength(0);
                s_dst.SetLength(0);

                // Stop any running timers
                sw_get.Stop();
                sw_cmp.Stop();
            }

            i++;
        }

        Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "VerifyTiming",
            "Verify | Get: {0} ms, Compare: {1} ms",
            TimeSpan.FromMilliseconds(sw_get.ElapsedMilliseconds),
            TimeSpan.FromMilliseconds(sw_cmp.ElapsedMilliseconds));

        if (config.Progress)
            Console.WriteLine($"\rVerifying: {n}/{n}");

        return errors;
    }

}