Merge branch 'master' into feature/prevent-windows-subfolder-storage

Duplicati/Library/AutoUpdater/UpdaterManager.cs

@@ -231,7 +231,7 @@ namespace Duplicati.Library.AutoUpdater
         /// <summary>
         /// The machine name
         /// </summary>
-        public static string MachineName => Environment.MachineName;
+        public static readonly string MachineName = MachineNameReader.GetMachineName();
 
         /// <summary>
         /// The package type ID

Duplicati/Library/Interface/QuotaInfo.cs

@@ -18,11 +18,6 @@
 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
 // DEALINGS IN THE SOFTWARE.
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace Duplicati.Library.Interface
 {

Duplicati/Library/Main/AsyncDownloader.cs

@@ -1,174 +0,0 @@
-// Copyright (C) 2025, The Duplicati Team
-// https://duplicati.com, hello@duplicati.com
-// 
-// Permission is hereby granted, free of charge, to any person obtaining a 
-// copy of this software and associated documentation files (the "Software"), 
-// to deal in the Software without restriction, including without limitation 
-// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
-// and/or sell copies of the Software, and to permit persons to whom the 
-// Software is furnished to do so, subject to the following conditions:
-// 
-// The above copyright notice and this permission notice shall be included in 
-// all copies or substantial portions of the Software.
-// 
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
-// DEALINGS IN THE SOFTWARE.
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using Duplicati.Library.Main.Database;
-
-namespace Duplicati.Library.Main
-{
-    internal interface IAsyncDownloadedFile : IRemoteVolume
-    {
-        Library.Utility.TempFile TempFile { get; }
-    }
-
-    internal class AsyncDownloader : IEnumerable<IAsyncDownloadedFile>
-    {
-        private static readonly string LOGTAG = Logging.Log.LogTagFromType<AsyncDownloader>();
-
-        private class AsyncDownloaderEnumerator : IEnumerator<IAsyncDownloadedFile>
-        {
-            private class AsyncDownloadedFile : IAsyncDownloadedFile
-            {
-                private readonly Exception m_exception;
-                private Library.Utility.TempFile m_file;
-                
-                public string Name { get; private set; }
-                public string Hash { get; private set; }
-                public long Size { get; private set; }
-                
-                public void DisposeTempFile()
-                {
-                    if (m_file != null)
-                        try { m_file.Dispose(); }
-                        finally { m_file = null; }
-                }
-                
-
-                public Library.Utility.TempFile TempFile
-                {
-                    get
-                    {
-                        if (m_exception != null)
-                            throw m_exception;
-
-                        return m_file;
-                    }
-                }
-                
-                public AsyncDownloadedFile(string name, string hash, long size, Library.Utility.TempFile tempfile, Exception exception)
-                {
-                    this.Name = name;
-                    this.Hash = hash;
-                    this.Size = size;
-                    m_exception = exception;
-                    m_file = tempfile;
-                }
-            }
-        
-            private readonly IList<IRemoteVolume> m_volumes;
-            private BackendManager.IDownloadWaitHandle m_handle;
-            private readonly BackendManager m_backend;
-            private int m_index;
-            private AsyncDownloadedFile m_current;
-
-            public AsyncDownloaderEnumerator(IList<IRemoteVolume> volumes, BackendManager backend)
-            {
-                m_volumes = volumes;
-                m_backend = backend;
-                m_index = 0;
-            }
-
-            public IAsyncDownloadedFile Current
-            {
-                get { return m_current; }
-            }
-
-            public void Dispose()
-            {
-                if (m_current != null)
-                {
-                    m_current.DisposeTempFile();
-                    m_current = null;
-                }
-            }
-
-            object System.Collections.IEnumerator.Current
-            {
-                get { return this.Current; }
-            }
-
-            public bool MoveNext()
-            {
-                if (m_current != null)
-                {
-                    m_current.DisposeTempFile();
-                    m_current = null;
-                }
-
-                if (m_index >= m_volumes.Count)
-                    return false;
-
-                if (m_handle == null)
-                    m_handle = m_backend.GetAsync(m_volumes[m_index].Name, m_volumes[m_index].Size, m_volumes[m_index].Hash);
-                
-                string hash = null;
-                long size = -1;
-                Library.Utility.TempFile file = null;
-                Exception exception = null;
-                try
-                {
-                    file = m_handle.Wait(out hash, out size);
-                    
-                }
-                catch (Exception ex)
-                {
-                    Logging.Log.WriteErrorMessage(LOGTAG, "FailedToRetrieveFile", ex, "Failed to retrieve file {0}", m_volumes[m_index].Name);
-                    exception = ex;
-                }
-                
-                m_current = new AsyncDownloadedFile(m_volumes[m_index].Name, hash, size, file, exception);
-                m_handle = null;
-
-                m_index++;
-                if (m_index < m_volumes.Count)
-                    m_handle = m_backend.GetAsync(m_volumes[m_index].Name, m_volumes[m_index].Size, m_volumes[m_index].Hash);
-
-                return true;
-            }
-
-            public void Reset()
-            {
-                throw new NotSupportedException("Cannot reset " + this.GetType().FullName);
-            }
-        }
-
-        private readonly IList<IRemoteVolume> m_volumes;
-        private readonly BackendManager m_backend;
-
-        public AsyncDownloader(IList<IRemoteVolume> volumes, BackendManager backend)
-        {
-            m_volumes = volumes;
-            m_backend = backend;
-        }
-
-        public IEnumerator<IAsyncDownloadedFile> GetEnumerator()
-        {
-            return new AsyncDownloaderEnumerator(m_volumes, m_backend);
-        }
-
-        System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
-        {
-            return this.GetEnumerator();
-        }
-    }
-}

Duplicati/Library/Main/Backend/BackendManager.DatabaseCollector.cs

@@ -0,0 +1,179 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Duplicati.Library.Logging;
+using Duplicati.Library.Main.Database;
+
+namespace Duplicati.Library.Main.Backend;
+
+#nullable enable
+
+partial class BackendManager
+{
+    /// <summary>
+    /// A class to collect operations performed on the remote destination.
+    /// This class is used to log operations so they can later be written to the database.
+    /// The goal is to keep the database state as close as possible to the remote operation state.
+    /// Since the backend communication is done in parallel with the actual operations,
+    /// we store the performed operations in a queue and flush them to the database when the database is available.
+    /// The main operations are responsible for flushing the messages when comitting a transaction.
+    /// If the operation fails, the logged messages here should still be flushed to the database, 
+    /// as they have already been performed on the remote destination.
+    /// </summary>
+    private class DatabaseCollector
+    {
+        /// <summary>
+        /// The log tag for this class
+        /// </summary>
+        private static readonly string LOGTAG = Log.LogTagFromType<DatabaseCollector>();
+        /// <summary>
+        /// The lock object for the database queue
+        /// </summary>
+        private readonly object m_dbqueuelock = new object();
+        /// <summary>
+        /// The queue of database operations
+        /// </summary>
+        private List<IRemoteOperationEntry> m_dbqueue = [];
+
+        /// <summary>
+        /// Interface for database entries
+        /// </summary>
+        private interface IRemoteOperationEntry { }
+
+        /// <summary>
+        /// Logs an operation performed on the remote destination
+        /// </summary>
+        /// <param name="Action">The performed action</param>
+        /// <param name="File">The file the operation is performed on</param>
+        /// <param name="Result">The result of the operation</param>
+        private sealed record RemoteOperationLogEntry(string Action, string File, string? Result) : IRemoteOperationEntry;
+
+
+        /// <summary>
+        /// Logs a database update after a remote file operation
+        /// </summary>
+        /// <param name="Remotename">The remote name of the volume</param>
+        /// <param name="State">The new state of the volume</param>
+        /// <param name="Size">The new size of the volume</param>
+        /// <param name="Hash">The new hash of the volume</param>
+        private sealed record RemoteVolumeUpdate(string Remotename, RemoteVolumeState State, long Size, string? Hash) : IRemoteOperationEntry;
+
+        /// <summary>
+        /// Logs a rename of a remote file
+        /// </summary>
+        /// <param name="Oldname">The old name of the file</param>
+        /// <param name="Newname">The new name of the file</param>
+        private sealed record RenameRemoteVolume(string Oldname, string Newname) : IRemoteOperationEntry;
+
+        /// <summary>
+        /// Logs an operation performed on the remote destination
+        /// </summary>
+        /// <param name="action">The action of the operation</param>
+        /// <param name="file">The file of the operation</param>
+        /// <param name="result">The result of the operation</param>
+        public void LogRemoteOperation(string action, string file, string? result)
+        {
+            lock (m_dbqueuelock)
+                m_dbqueue.Add(new RemoteOperationLogEntry(action, file, result));
+        }
+
+        /// <summary>
+        /// Logs a database update after a remote file operation
+        /// </summary>
+        /// <param name="remotename">The remote name of the volume</param>
+        /// <param name="state">The new state of the volume</param>
+        /// <param name="size">The new size of the volume</param>
+        /// <param name="hash">The new hash of the volume</param>
+        public void LogRemoteVolumeUpdated(string remotename, RemoteVolumeState state, long size, string? hash)
+        {
+            lock (m_dbqueuelock)
+                m_dbqueue.Add(new RemoteVolumeUpdate(remotename, state, size, hash));
+        }
+
+        /// <summary>
+        /// Logs a remote file rename
+        /// </summary>
+        /// <param name="oldname">The old name of the file</param>
+        /// <param name="newname">The new name of the file</param>
+        public void LogRemoteVolumeRenamed(string oldname, string newname)
+        {
+            lock (m_dbqueuelock)
+                m_dbqueue.Add(new RenameRemoteVolume(oldname, newname));
+        }
+
+        /// <summary>
+        /// Drops all pending messages from the queue
+        /// </summary>
+        public void ClearPendingMessages()
+        {
+            lock (m_dbqueuelock)
+                m_dbqueue = [];
+        }
+
+        /// <summary>
+        /// Flushes all messages to the database
+        /// </summary>
+        /// <param name="db">The database to write pending messages to</param>
+        /// <param name="transaction">The transaction to use, if any</param>
+        /// <returns>Whether any messages were flushed</returns>
+        public bool FlushPendingMessages(LocalDatabase db, System.Data.IDbTransaction? transaction)
+        {
+            List<IRemoteOperationEntry> entries;
+            lock (m_dbqueuelock)
+                if (m_dbqueue.Count == 0)
+                    return false;
+                else
+                {
+                    entries = m_dbqueue;
+                    m_dbqueue = [];
+                }
+
+            // Collect removed volumes for final db cleanup.
+            var volsRemoved = new HashSet<string>();
+
+            // As we replace the list, we can now freely access the elements without locking
+            foreach (var e in entries)
+                if (e is RemoteOperationLogEntry operation)
+                    db.LogRemoteOperation(operation.Action, operation.File, operation.Result, transaction);
+                else if (e is RemoteVolumeUpdate update && update.State == RemoteVolumeState.Deleted)
+                {
+                    db.UpdateRemoteVolume(update.Remotename, RemoteVolumeState.Deleted, update.Size, update.Hash, true, TimeSpan.FromHours(2), transaction);
+                    volsRemoved.Add(update.Remotename);
+                }
+                else if (e is RemoteVolumeUpdate dbUpdate)
+                    db.UpdateRemoteVolume(dbUpdate.Remotename, dbUpdate.State, dbUpdate.Size, dbUpdate.Hash, transaction);
+                else if (e is RenameRemoteVolume rename)
+                    db.RenameRemoteFile(rename.Oldname, rename.Newname, transaction);
+                else if (e != null)
+                    Log.WriteErrorMessage(LOGTAG, "InvalidQueueElement", null, "Queue had element of type: {0}, {1}", e.GetType(), e);
+
+            // Finally remove volumes from DB.
+            if (volsRemoved.Count > 0)
+                db.RemoveRemoteVolumes(volsRemoved);
+
+            return true;
+        }
+
+        /// <summary>
+        /// Flushes all messages to the log after stopping the processing
+        /// </summary>
+        public void FlushMessagesToLog()
+        {
+            if (m_dbqueue.Count == 0)
+                return;
+
+            string message;
+            lock (m_dbqueuelock)
+                message = string.Join("\n", m_dbqueue.Select(e => e switch
+                {
+                    RemoteOperationLogEntry operation => $"Operation: {operation.Action} File: {operation.File} Result: {operation.Result}",
+                    RemoteVolumeUpdate update => $"Update: {update.Remotename} State: {update.State} Size: {update.Size} Hash: {update.Hash}",
+                    RenameRemoteVolume rename => $"Rename: {rename.Oldname} -> {rename.Newname}",
+                    _ => $"InvalidQueueElement: {e.GetType()} {e}"
+                }));
+
+            Log.WriteWarningMessage(LOGTAG, "FlushingMessagesToLog", null, message);
+
+        }
+    }
+}

Duplicati/Library/Main/Backend/BackendManager.DeleteOperation.cs

@@ -0,0 +1,107 @@
+using System;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Duplicati.Library.Interface;
+using Duplicati.Library.Localization.Short;
+
+namespace Duplicati.Library.Main.Backend;
+
+#nullable enable
+
+partial class BackendManager
+{
+    /// <summary>
+    /// Represents a pending DELETE operation
+    /// </summary>
+    private class DeleteOperation : PendingOperation<bool>
+    {
+        /// <summary>
+        /// The log tag for this class
+        /// </summary>
+        private static readonly string LOGTAG = Logging.Log.LogTagFromType<DeleteOperation>();
+
+        /// <summary>
+        /// The remote filename that is to be deleted
+        /// </summary>
+        public override string RemoteFilename { get; }
+        /// <summary>
+        /// The size of the remote file, or -1 if unknown
+        /// </summary>
+        public override long Size { get; }
+
+        /// <summary>
+        /// The operation type
+        /// </summary>
+        public override BackendActionType Operation => BackendActionType.Delete;
+
+        /// <summary>
+        /// Creates a new DeleteOperation
+        /// </summary>
+        /// <param name="remoteName">The remote filename to delete</param>
+        /// <param name="size">The size of the remote file, or -1 if unknown</param>
+        /// <param name="context">The execution context</param>
+        /// <param name="waitForComplete">Whether to wait for the operation to complete</param>
+        public DeleteOperation(string remoteName, long size, ExecuteContext context, bool waitForComplete, CancellationToken cancelToken)
+            : base(context, waitForComplete, cancelToken)
+        {
+            RemoteFilename = remoteName;
+            Size = size;
+        }
+
+        /// <inheritdoc/>
+        public override async Task<bool> ExecuteAsync(IBackend backend, CancellationToken cancelToken)
+        {
+            Context.Statwriter.SendEvent(BackendActionType.Delete, BackendEventType.Started, RemoteFilename, Size);
+
+            string? result = null;
+            try
+            {
+                await backend.DeleteAsync(RemoteFilename, cancelToken).ConfigureAwait(false);
+            }
+            catch (Exception ex)
+            {
+                // Check if the file was not found, and if so, check if it was actually deleted
+                var isFileMissingException = ex is Library.Interface.FileMissingException || ex is System.IO.FileNotFoundException;
+                var wr = (ex as System.Net.WebException)?.Response as System.Net.HttpWebResponse;
+                bool recovered = false;
+
+                if (isFileMissingException || (wr != null && wr.StatusCode == System.Net.HttpStatusCode.NotFound))
+                {
+                    Logging.Log.WriteInformationMessage(LOGTAG, "DeleteRemoteFileFailed", LC.L($"Delete operation failed for {RemoteFilename} with FileNotFound, listing contents"));
+
+                    try
+                    {
+                        recovered = !backend.List().Select(x => x.Name).Contains(RemoteFilename);
+                    }
+                    catch
+                    {
+                    }
+
+                    if (recovered)
+                        Logging.Log.WriteInformationMessage(LOGTAG, "DeleteRemoteFileSuccess", LC.L($"Listing indicates file {RemoteFilename} was deleted correctly"));
+                    else
+                        Logging.Log.WriteWarningMessage(LOGTAG, "DeleteRemoteFileFailed", ex, LC.L($"Listing confirms file {RemoteFilename} was not deleted"));
+                }
+
+                if (!recovered)
+                {
+                    result = ex.ToString();
+                    throw;
+                }
+            }
+            finally
+            {
+                // Here, we do not know if the file was actually deleted or not
+                // We log that the operation was performed, and the result
+                Context.Database.LogRemoteOperation("delete", RemoteFilename, result);
+
+                // We also log the new state of the file, so it will be attempted to be re-deleted on later listings
+                Context.Database.LogRemoteVolumeUpdated(RemoteFilename, RemoteVolumeState.Deleted, -1, null);
+            }
+
+            Context.Statwriter.SendEvent(BackendActionType.Delete, BackendEventType.Completed, RemoteFilename, Size);
+            return true;
+        }
+    }
+}

Duplicati/Library/Main/Backend/BackendManager.GetOperation.cs

@@ -0,0 +1,250 @@
+using System;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Duplicati.Library.Interface;
+using Duplicati.Library.Utility;
+using SharpAESCrypt;
+
+namespace Duplicati.Library.Main.Backend;
+
+#nullable enable
+
+partial class BackendManager
+{
+    /// <summary>
+    /// Represents a pending GET operation
+    /// </summary>
+    private class GetOperation : PendingOperation<(TempFile File, string Hash, long Size)>
+    {
+        /// <summary>
+        /// The log tag for this class
+        /// </summary>
+        private static readonly string LOGTAG = Logging.Log.LogTagFromType<GetOperation>();
+
+        /// <summary>
+        /// The remote filename that is to be downloaded
+        /// </summary>
+        public override string RemoteFilename { get; }
+        /// <summary>
+        /// The size of the remote file, or -1 if unknown
+        /// </summary>
+        public override long Size { get; }
+        /// <summary>
+        /// The hash of the remote file, or null if unknown
+        /// </summary>
+        public required string? Hash { get; set; }
+
+        /// <summary>
+        /// The operation type
+        /// </summary>
+        public override BackendActionType Operation => BackendActionType.Get;
+
+        /// <summary>
+        /// Creates a new GetOperation
+        /// </summary>
+        /// <param name="remotefilename">The remote filename to download</param>
+        /// <param name="size">The size of the remote file, or -1 if unknown</param>
+        /// <param name="context">The execution context</param>
+        /// <param name="cancelToken">The cancellation token</param>
+        public GetOperation(string remotefilename, long size, ExecuteContext context, CancellationToken cancelToken)
+            : base(context, true, cancelToken)
+        {
+            RemoteFilename = remotefilename;
+            Size = size;
+        }
+
+        /// <summary>
+        /// Executes the operation
+        /// </summary>
+        /// <param name="backend">The backend to download from</param>
+        /// <param name="cancelToken">The cancellation token</param>
+        /// <returns>The file, hash and size of the downloaded file</returns>
+        public override async Task<(TempFile File, string Hash, long Size)> ExecuteAsync(IBackend backend, CancellationToken cancelToken)
+        {
+            TempFile? tmpfile = null;
+            Context.Statwriter.SendEvent(BackendActionType.Get, BackendEventType.Started, RemoteFilename, Size);
+            using var encryption = Context.Options.NoEncryption
+                ? null
+                : (DynamicLoader.EncryptionLoader.GetModule(Context.Options.EncryptionModule, Context.Options.Passphrase, Context.Options.RawOptions)
+                    ?? throw new Exception(Strings.BackendMananger.EncryptionModuleNotFound(Context.Options.EncryptionModule))
+            );
+
+            try
+            {
+                // Start and time the donwload
+                var begin = DateTime.Now;
+
+                (tmpfile, var dataSizeDownloaded, var fileHash) = await DoGetFileAsync(backend, cancelToken).ConfigureAwait(false);
+
+                var duration = DateTime.Now - begin;
+                Logging.Log.WriteProfilingMessage(LOGTAG, "DownloadSpeed", "Downloaded {0} in {1}, {2}/s", Library.Utility.Utility.FormatSizeString(dataSizeDownloaded),
+                    duration, Library.Utility.Utility.FormatSizeString((long)(dataSizeDownloaded / duration.TotalSeconds)));
+
+                Context.Database.LogRemoteOperation("get", RemoteFilename, System.Text.Json.JsonSerializer.Serialize(new { Size = dataSizeDownloaded, Hash = fileHash }));
+                Context.Statwriter.SendEvent(BackendActionType.Get, BackendEventType.Completed, RemoteFilename, dataSizeDownloaded);
+
+                if (!Context.Options.SkipFileHashChecks)
+                {
+                    if (Size >= 0 && dataSizeDownloaded != Size)
+                        throw new Exception(Strings.Controller.DownloadedFileSizeError(RemoteFilename, dataSizeDownloaded, Size));
+
+                    if (!string.IsNullOrEmpty(Hash) && fileHash != Hash)
+                        throw new HashMismatchException(Strings.Controller.HashMismatchError(tmpfile, Hash, fileHash));
+                }
+
+                // Perform decryption after hash validation, if needed
+                tmpfile = DecryptFile(tmpfile, DetectEncryptionModule(encryption));
+
+                return (tmpfile, fileHash, dataSizeDownloaded);
+            }
+            catch
+            {
+                tmpfile?.Dispose();
+                throw;
+            }
+        }
+
+        /// <summary>
+        /// Downloads a file from the backend
+        /// </summary>
+        /// <param name="backend">The backend to download from</param>
+        /// <param name="cancelToken">The cancellation token</param>
+        /// <returns>The downloaded file, the size of the file, and the hash of the file</returns>
+        private async Task<(TempFile tempFile, long downloadSize, string remotehash)> DoGetFileAsync(IBackend backend, CancellationToken cancelToken)
+        {
+            TempFile? dlTarget = null;
+            try
+            {
+                long retDownloadSize;
+                string retHashcode;
+                dlTarget = new TempFile();
+                if (backend is IStreamingBackend streamingBackend && !Context.Options.DisableStreamingTransfers)
+                {
+                    // extended to use stacked streams
+                    using (var fs = System.IO.File.OpenWrite(dlTarget))
+                    using (var act = new StreamUtil.TimeoutObservingStream(fs) { WriteTimeout = backend is ITimeoutExemptBackend ? Timeout.Infinite : Context.Options.ReadWriteTimeout })
+                    using (var hasher = HashFactory.CreateHasher(Context.Options.FileHashAlgorithm))
+                    using (var hs = new HashCalculatingStream(act, hasher))
+                    using (var ss = new ShaderStream(hs, true))
+                    using (var linkedToken = CancellationTokenSource.CreateLinkedTokenSource(cancelToken, act.TimeoutToken))
+                    {
+                        using (var ts = new ThrottledStream(ss, 0, Context.Options.MaxDownloadPrSecond))
+                        using (var pgs = new ProgressReportingStream(ts, pg => Context.HandleProgress(ts, pg, RemoteFilename)))
+                        {
+                            await streamingBackend.GetAsync(RemoteFilename, pgs, linkedToken.Token).ConfigureAwait(false);
+                        }
+                        ss.Flush();
+                        retDownloadSize = ss.TotalBytesWritten;
+                        retHashcode = Convert.ToBase64String(hs.GetFinalHash());
+                    }
+                }
+                else
+                {
+                    await backend.GetAsync(RemoteFilename, dlTarget, cancelToken).ConfigureAwait(false);
+                    retDownloadSize = new System.IO.FileInfo(dlTarget).Length;
+                    retHashcode = CalculateFileHash(dlTarget, Context.Options);
+                }
+
+                var retTarget = dlTarget;
+                dlTarget = null;
+                return (retTarget, retDownloadSize, retHashcode);
+            }
+            finally
+            {
+                // Remove temp files on failure
+                dlTarget?.Dispose();
+            }
+        }
+
+        /// <summary>
+        /// Detects the encryption module to use, based on the filename.
+        /// This makes it possible to restore from a folder with mixed encryption modules.
+        /// </summary>
+        /// <param name="encryption">The default encryption module</param>
+        /// <returns>The encryption module to use</returns>
+        private IEncryption? DetectEncryptionModule(IEncryption? encryption)
+        {
+            try
+            {
+                // Auto-guess the encryption module
+                var ext = (System.IO.Path.GetExtension(RemoteFilename) ?? "").TrimStart('.');
+                if (!ext.Equals(encryption?.FilenameExtension, StringComparison.OrdinalIgnoreCase))
+                {
+                    // Check if the file is not encrypted
+                    if (DynamicLoader.CompressionLoader.Keys.Contains(ext, StringComparer.OrdinalIgnoreCase))
+                    {
+                        if (encryption != null)
+                            Logging.Log.WriteVerboseMessage(LOGTAG, "AutomaticDecryptionDetection", "Filename extension \"{0}\" does not match encryption module \"{1}\", guessing that it is not encrypted", ext, Context.Options.EncryptionModule);
+                        return null;
+                    }
+                    // Check if the file is encrypted with something else
+                    else if (DynamicLoader.EncryptionLoader.Keys.Contains(ext, StringComparer.OrdinalIgnoreCase))
+                    {
+                        Logging.Log.WriteVerboseMessage(LOGTAG, "AutomaticDecryptionDetection", "Filename extension \"{0}\" does not match encryption module \"{1}\", attempting to use matching encryption module", ext, Context.Options.EncryptionModule);
+
+                        try
+                        {
+                            return DynamicLoader.EncryptionLoader.GetModule(ext, Context.Options.Passphrase, Context.Options.RawOptions)
+                                ?? encryption;
+                        }
+                        catch (Exception ex)
+                        {
+                            Logging.Log.WriteWarningMessage(LOGTAG, "AutomaticDecryptionDetection", ex, "Failed to load encryption module \"{0}\", using specified encryption module \"{1}\"", ext, Context.Options.EncryptionModule);
+                        }
+                    }
+                    // Fallback, lets see what happens...
+                    else
+                    {
+                        Logging.Log.WriteVerboseMessage(LOGTAG, "AutomaticDecryptionDetection", "Filename extension \"{0}\" does not match encryption module \"{1}\", attempting to use specified encryption module as no others match", ext, Context.Options.EncryptionModule);
+                    }
+                }
+
+                return encryption;
+            }
+            // If we fail here, make sure that we throw a crypto exception
+            catch (System.Security.Cryptography.CryptographicException) { throw; }
+            catch (Exception ex) { throw new System.Security.Cryptography.CryptographicException(ex.Message, ex); }
+        }
+
+        /// <summary>
+        /// Performs decryption of a file. This could be more efficient if we decrypt while downloading,
+        /// but that would prevent us from verifying the hash of the encrypted file, before decrypting it.
+        /// Decrypting afterwards also ensures we can control the thrown exceptions.
+        /// </summary>
+        /// <param name="tempFile">The encrypted file</param>
+        /// <param name="decrypter">Then encryption module to use, or <c>null</c> for no encryption</param>
+        /// <returns>The decrypted file</returns>
+        private TempFile DecryptFile(TempFile tempFile, IEncryption? decrypter)
+        {
+            // Support no encryption
+            if (decrypter == null)
+                return tempFile;
+
+            TempFile? decryptTarget = null;
+
+            // Always dispose the source file
+            using (tempFile)
+            using (new Logging.Timer(LOGTAG, "DecryptFile", "Decrypting " + tempFile))
+            {
+                try
+                {
+                    decryptTarget = new TempFile();
+                    try { decrypter.Decrypt(tempFile, decryptTarget); }
+                    // If we fail here, make sure that we throw a crypto exception
+                    catch (System.Security.Cryptography.CryptographicException) { throw; }
+                    catch (Exception ex) { throw new System.Security.Cryptography.CryptographicException(ex.Message, ex); }
+
+                    var result = decryptTarget;
+                    decryptTarget = null;
+                    return result;
+                }
+                finally
+                {
+                    // Remove temp files on failure
+                    decryptTarget?.Dispose();
+                }
+            }
+        }
+    }
+}

Duplicati/Library/Main/Backend/BackendManager.Handler.cs

@@ -0,0 +1,443 @@
+#nullable enable
+
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using CoCoL;
+using Duplicati.Library.Interface;
+
+namespace Duplicati.Library.Main.Backend;
+
+partial class BackendManager
+{
+    /// <summary>
+    /// Wrapper class for making a backend disposable and reclaimable
+    /// </summary>
+    private sealed class ReclaimableBackend : IDisposable
+    {
+        /// <summary>
+        /// The tag used for logging
+        /// </summary>
+        private static readonly string LOGTAG = Logging.Log.LogTagFromType<ReclaimableBackend>();
+
+        /// <summary>
+        /// The backend being wrapped
+        /// </summary>
+        public IBackend Backend { get; }
+        /// <summary>
+        /// The pool where the backend should be returned to
+        /// </summary>
+        private readonly ConcurrentQueue<IBackend> pool;
+        /// <summary>
+        /// Whether the backend should be reused
+        /// </summary>
+        private bool reuse;
+        /// <summary>
+        /// Whether the backend wrapper has been disposed
+        /// </summary>
+        private bool disposed;
+
+        /// <summary>
+        /// Creates a new instance of the <see cref="ReclaimableBackend"/> class
+        /// </summary>
+        /// <param name="backend">The backend to wrap</param>
+        /// <param name="pool">The pool where the backend should be returned to</param>
+        /// <param name="reuse">Whether the backend should be reused or disposed</param>
+        public ReclaimableBackend(IBackend backend, ConcurrentQueue<IBackend> pool, bool reuse)
+        {
+            Backend = backend;
+            this.pool = pool;
+            this.reuse = reuse;
+        }
+
+        /// <summary>
+        /// Prevents the backend from being reclaimed
+        /// </summary>
+        public void PreventReuse()
+        {
+            reuse = false;
+        }
+
+        /// <summary>
+        /// Disposes the backend wrapper
+        /// </summary>
+        public void Dispose()
+        {
+            if (disposed)
+                return;
+            disposed = true;
+
+            if (reuse)
+                pool.Enqueue(Backend);
+            else
+                try { Backend.Dispose(); }
+                catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "BackendDisposeError", ex, "Failed to dispose backend instance: {0}", ex.Message); }
+        }
+    }
+
+    /// <summary>
+    /// The handler for processing backend operations
+    /// </summary>
+    private class Handler
+    {
+        /// <summary>
+        /// The tag used for logging
+        /// </summary>
+        private static readonly string LOGTAG = Logging.Log.LogTagFromType<Handler>();
+
+        /// <summary>
+        /// The list of active uploads
+        /// </summary>
+        private readonly List<Task> activeUploads = new();
+        /// <summary>
+        /// The pool of backends currently created
+        /// </summary>
+        private readonly ConcurrentQueue<IBackend> backendPool = new();
+        /// <summary>
+        /// The URL of the backend
+        /// </summary>
+        private readonly string backendUrl;
+        /// <summary>
+        /// The context for the handler
+        /// </summary>
+        private readonly ExecuteContext context;
+        /// <summary>
+        /// The maximum number of parallel uploads
+        /// </summary>
+        private readonly int maxParallelUploads;
+        /// <summary>
+        /// The maximum number of retries
+        /// </summary>
+        private readonly int maxRetries;
+        /// <summary>
+        /// The delay between retries
+        /// </summary>
+        private readonly TimeSpan retryDelay;
+        /// <summary>
+        /// Whether to retry with exponential backoff
+        /// </summary>
+        private readonly bool retryWithExponentialBackoff;
+        /// <summary>
+        /// Whether to allow backend reuse
+        /// </summary>
+        private readonly bool allowBackendReuse;
+        /// <summary>
+        /// Whether any files have been uploaded
+        /// </summary>
+        private bool anyUploaded;
+        /// <summary>
+        /// Whether any files have been downloaded
+        /// </summary>
+        private bool anyDownloaded;
+
+        /// <summary>
+        /// Creates and runs with an instance of the <see cref="Handler"/> class
+        /// </summary>
+        /// <param name="requestChannel">The channel for pending operations</param>
+        /// <param name="backendUrl">The URL of the backend</param>
+        /// <param name="context">The execution context</param>
+        /// <returns>An awaitable task</returns>
+        public static Task RunHandlerAsync(IReadChannel<PendingOperationBase> requestChannel, string backendUrl, ExecuteContext context)
+            => AutomationExtensions.RunTask(new { requestChannel },
+                self => new Handler(backendUrl, context).Run(self.requestChannel)
+            );
+
+        /// <summary>
+        /// Creates a new instance of the <see cref="Handler"/> class
+        /// </summary>
+        /// <param name="backendUrl">The URL of the backend</param>
+        /// <param name="context">The execution context</param>
+        private Handler(string backendUrl, ExecuteContext context)
+        {
+            this.backendUrl = backendUrl;
+            this.context = context;
+
+            maxParallelUploads = Math.Max(1, context.Options.AsynchronousConcurrentUploadLimit);
+            maxRetries = context.Options.NumberOfRetries;
+            retryDelay = context.Options.RetryDelay;
+            retryWithExponentialBackoff = context.Options.RetryWithExponentialBackoff;
+            allowBackendReuse = !context.Options.NoConnectionReuse;
+
+        }
+
+        /// <summary>
+        /// Creates a new backend instance or reuses an existing one
+        /// </summary>
+        /// <returns>The backend instance</returns>
+        private ReclaimableBackend CreateBackend()
+        {
+            backendPool.TryDequeue(out var backend);
+            if (backend == null)
+                backend = DynamicLoader.BackendLoader.GetBackend(backendUrl, context.Options.RawOptions);
+
+            return new ReclaimableBackend(
+                backend,
+                backendPool,
+                allowBackendReuse
+            );
+        }
+
+        /// <summary>
+        /// Reclaims completed uploads
+        /// </summary>
+        /// <returns>An awaitable task</returns>
+        private async Task ReclaimCompletedUploads()
+        {
+            for (int i = activeUploads.Count - 1; i >= 0; i--)
+            {
+                if (activeUploads[i].IsCompleted)
+                {
+                    // Make sure the task is awaited so we capture any exceptions
+                    await activeUploads[i].ConfigureAwait(false);
+                    activeUploads.RemoveAt(i);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Ensures that there are at most N - 1 pending uploads
+        /// </summary>
+        /// <param name="n">The maximum number of pending uploads</param>
+        /// <returns>An awaitable task</returns>
+        private async Task EnsureAtMostNPendingUploads(int n)
+        {
+            while (activeUploads.Count >= n)
+            {
+                await Task.WhenAny(activeUploads).ConfigureAwait(false);
+                await ReclaimCompletedUploads().ConfigureAwait(false);
+            }
+        }
+
+        /// <summary>
+        /// Runs the handler
+        /// </summary>
+        /// <param name="requestChannel">The channel for pending operations</param>
+        /// <returns>An awaitable task</returns>
+        private async Task Run(IReadChannel<PendingOperationBase> requestChannel)
+        {
+            while (true)
+            {
+                // Get next operation
+                var op = await requestChannel.ReadAsync().ConfigureAwait(false);
+
+                try
+                {
+                    // Clean up completed uploads, if any
+                    await ReclaimCompletedUploads().ConfigureAwait(false);
+
+                    // Allow PUT operations to be queued, if requested
+                    if (op is PutOperation putOp && !putOp.WaitForComplete)
+                    {
+                        await EnsureAtMostNPendingUploads(maxParallelUploads).ConfigureAwait(false);
+
+                        // Operation is accepted into queue, so we can signal completion
+                        putOp.SetComplete(true);
+                        activeUploads.Add(ExecuteWithRetry(putOp));
+                    }
+                    else
+                    {
+                        // Wait for all pending uploads to complete
+                        await EnsureAtMostNPendingUploads(1).ConfigureAwait(false);
+                        // Execute the operation
+                        await ExecuteWithRetry(op).ConfigureAwait(false);
+                    }
+                }
+                catch (Exception ex)
+                {
+                    Logging.Log.WriteWarningMessage(LOGTAG, "HandlerError", ex, "Error in handler: {0}", ex.Message);
+                    // If we fail, the task may "hang", so we ensure it is completed here
+                    op.SetFailed(ex);
+                    throw;
+                }
+            }
+        }
+
+        /// <summary>
+        /// Tries to create a folder, handling errors
+        /// </summary>
+        /// <returns><c>true</c> if the folder was created, <c>false</c> otherwise</returns>
+        private async Task<bool> TryCreateFolder()
+        {
+            using var backend = CreateBackend();
+            try
+            {
+                // If we successfully create the folder, we can re-use the connection
+                await backend.Backend.CreateFolderAsync(context.TaskReader.TransferToken).ConfigureAwait(false);
+                return true;
+            }
+            catch (Exception ex)
+            {
+                // Failure should not reuse the backend
+                backend.PreventReuse();
+                Logging.Log.WriteWarningMessage(LOGTAG, "FolderCreateError", ex, "Failed to create folder: {0}", ex.Message);
+            }
+
+            return false;
+        }
+
+        /// <summary>
+        /// Executes an operation with retries and error handling
+        /// </summary>
+        /// <param name="op">The operation to execute</param>
+        /// <returns>An awaitable task</returns>
+        private async Task ExecuteWithRetry(PendingOperationBase op)
+        {
+            // Once in this method, we MUST set the op result,
+            // or the program will hang waiting for the operation to complete
+
+            int retries = 0;
+            Exception? lastException = null;
+
+            do
+            {
+                try
+                {
+                    // Happy case is execute and return
+                    await Execute(op).ConfigureAwait(false);
+                    return;
+                }
+                catch (Exception ex)
+                {
+                    retries++;
+                    lastException = ex;
+                    Logging.Log.WriteRetryMessage(LOGTAG, $"Retry{op.Operation}", ex, "Operation {0} with file {1} attempt {2} of {3} failed with message: {4}", op.Operation, op.RemoteFilename, retries, maxRetries, ex.Message);
+
+                    // If we are cancelled, stop retrying
+                    if (op.CancelToken.IsCancellationRequested || context.TaskReader.ProgressToken.IsCancellationRequested || context.TaskReader.TransferToken.IsCancellationRequested)
+                    {
+                        op.SetCancelled();
+                        return;
+                    }
+
+                    // Refresh DNS name if we fail to connect in order to prevent issues with incorrect DNS entries
+                    var dnsFailure = Library.Utility.Utility.FlattenException(ex).Any(x => x is System.Net.WebException wex && wex.Status == System.Net.WebExceptionStatus.NameResolutionFailure);
+                    if (dnsFailure)
+                    {
+                        try
+                        {
+                            using (var backend = CreateBackend())
+                                foreach (var name in await backend.Backend.GetDNSNamesAsync(context.TaskReader.TransferToken).ConfigureAwait(false) ?? [])
+                                    if (!string.IsNullOrWhiteSpace(name))
+                                        System.Net.Dns.GetHostEntry(name);
+                        }
+                        catch
+                        {
+                        }
+                    }
+
+                    context.Statwriter.SendEvent(op.Operation, retries < maxRetries ? BackendEventType.Retrying : BackendEventType.Failed, op.RemoteFilename, op.Size);
+
+                    // Check if we can recover from the error
+                    var recovered = false;
+
+                    // Check if this was a folder missing exception and we are allowed to autocreate folders
+                    if (!(anyDownloaded || anyUploaded) && context.Options.AutocreateFolders && Library.Utility.Utility.FlattenException(ex).Any(x => x is FolderMissingException))
+                    {
+                        if (await TryCreateFolder().ConfigureAwait(false))
+                            recovered = true;
+                    }
+
+                    // We did not recover, so wait or give up
+                    if (!recovered && retries < maxRetries && retryDelay.Ticks != 0)
+                    {
+                        var delay = Library.Utility.Utility.GetRetryDelay(retryDelay, retries, retryWithExponentialBackoff);
+                        await Task.Delay(delay, context.TaskReader.ProgressToken).ConfigureAwait(false);
+                    }
+                }
+
+            } while (retries < maxRetries);
+
+            // If we have a last exception, we failed
+            if (lastException != null)
+            {
+                op.SetFailed(lastException);
+                (op as IDisposable)?.Dispose();
+
+                // Stop processing tasks if the operation failed and is not being waited for
+                // Delete operations can be retried later, so we don't stop processing
+                if (!op.WaitForComplete && op is not DeleteOperation)
+                    throw lastException;
+            }
+        }
+
+        /// <summary>
+        /// Fan-out for executing operations.
+        /// This method requires manual updates when new operation types are added,
+        /// but avoids a reflection-based dispatch.
+        /// </summary>
+        /// <param name="op">The operation to execute</param>
+        /// <returns>An awaitable task</returns>
+        private async Task Execute(PendingOperationBase op)
+        {
+            await context.TaskReader.ProgressRendevouz().ConfigureAwait(false);
+            using (new Logging.Timer(LOGTAG, $"RemoteOperation{op.Operation}", $"RemoteOperation{op.Operation}"))
+                switch (op)
+                {
+                    case PutOperation putOp:
+                        await Execute(putOp).ConfigureAwait(false);
+                        anyUploaded = true;
+                        return;
+                    case GetOperation getOp:
+                        await Execute(getOp).ConfigureAwait(false);
+                        anyDownloaded = true;
+                        return;
+                    case DeleteOperation deleteOp:
+                        await Execute(deleteOp).ConfigureAwait(false);
+                        return;
+                    case ListOperation listOp:
+                        await Execute(listOp).ConfigureAwait(false);
+                        return;
+                    case QuotaInfoOperation quotaOp:
+                        await Execute(quotaOp).ConfigureAwait(false);
+                        return;
+                    case WaitForEmptyOperation waitOp:
+                        waitOp.SetComplete(true);
+                        return;
+                    default:
+                        throw new NotImplementedException($"Operation type {op.GetType()} is not supported");
+                }
+        }
+
+        /// <summary>
+        /// Executes a specific operation
+        /// </summary>
+        /// <typeparam name="TResult">The return value type of the operation</typeparam>
+        /// <param name="op">The operation to execute</param>
+        /// <returns>An awaitable task</returns>
+        private async Task Execute<TResult>(PendingOperation<TResult> op)
+        {
+            using var backend = CreateBackend();
+            using var token = CancellationTokenSource.CreateLinkedTokenSource(op.CancelToken, context.TaskReader.TransferToken);
+
+            try
+            {
+                // Start processing the operation
+                var task = op.ExecuteAsync(backend.Backend, token.Token);
+
+                if (typeof(TResult) == typeof(bool) && !op.WaitForComplete)
+                {
+                    // Operation is accepted into queue, so we can signal completion
+                    op.SetComplete((TResult)(object)true);
+                    await task.ConfigureAwait(false);
+                }
+                else
+                {
+                    if (!op.WaitForComplete)
+                        throw new NotImplementedException($"WaitForComplete is required for operations returning a value: {op.GetType().FullName}");
+
+                    // Wait for the operation to complete
+                    op.SetComplete(await task.ConfigureAwait(false));
+                }
+            }
+            catch
+            {
+                // If the operation fails, we prevent reuse of the backend
+                backend.PreventReuse();
+                throw;
+            }
+        }
+    }
+}

Duplicati/Library/Main/Backend/BackendManager.ListOperation.cs

@@ -0,0 +1,63 @@
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using Duplicati.Library.Interface;
+
+namespace Duplicati.Library.Main.Backend;
+
+#nullable enable
+
+partial class BackendManager
+{
+    /// <summary>
+    /// Represents a pending LIST operation
+    /// </summary>
+    /// <remarks>
+    /// Creates a new ListOperation
+    /// </remarks>
+    /// <param name="context">The execution context</param>
+    /// <param name="cancelToken">The cancellation token</param>
+    private class ListOperation(ExecuteContext context, CancellationToken cancelToken)
+        : PendingOperation<List<IFileEntry>>(context, true, cancelToken)
+    {
+        /// <summary>
+        /// The operation type
+        /// </summary>
+        public override BackendActionType Operation => BackendActionType.List;
+
+        /// <summary>
+        /// Executes the operation
+        /// </summary>
+        /// <param name="backend">The backend to list</param>
+        /// <param name="cancelToken">The cancellation token</param>
+        /// <returns>An awaitable task</returns>
+        public override Task<List<IFileEntry>> ExecuteAsync(IBackend backend, CancellationToken cancelToken)
+        {
+            Context.Statwriter.SendEvent(BackendActionType.List, BackendEventType.Started, null, -1);
+
+            var r = backend.List().ToList();
+
+            // TODO: Investigate better way to solve this so we do not use memory for large lists
+            var sb = new StringBuilder();
+            sb.AppendLine("[");
+            long count = 0;
+            foreach (var e in r)
+            {
+                if (count != 0)
+                    sb.AppendLine(",");
+                count++;
+                sb.Append(System.Text.Json.JsonSerializer.Serialize(e));
+            }
+
+            sb.AppendLine();
+            sb.Append("]");
+            Context.Database.LogRemoteOperation("list", "", sb.ToString());
+
+            Context.Statwriter.SendEvent(BackendActionType.List, BackendEventType.Completed, null, r.Count);
+
+            return Task.FromResult(r);
+        }
+    }
+}

Duplicati/Library/Main/Backend/BackendManager.PendingOperation.cs

@@ -0,0 +1,138 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Duplicati.Library.Interface;
+using Duplicati.Library.Main.Operation.Common;
+using Duplicati.Library.Utility;
+
+namespace Duplicati.Library.Main.Backend;
+
+#nullable enable
+
+partial class BackendManager
+{
+    /// <summary>
+    /// Execution context for the backend manager
+    /// </summary>
+    /// <param name="HandleProgress">A progress handler</param>
+    /// <param name="Statwriter">The stat writer</param>
+    /// <param name="Database">The database collector</param>
+    /// <param name="TaskReader">The task reader</param>
+    /// <param name="Options">The options</param>
+    private sealed record ExecuteContext(
+        Action<ThrottledStream, long, string> HandleProgress,
+        IBackendWriter Statwriter,
+        DatabaseCollector Database,
+        ITaskReader TaskReader,
+        Options Options);
+
+    /// <summary>
+    /// A base non-generic pending operation
+    /// </summary>
+    private abstract class PendingOperationBase
+    {
+        /// <summary>
+        /// The execution context
+        /// </summary>
+        public ExecuteContext Context { get; }
+        /// <summary>
+        /// Whether to wait for the operation to complete.
+        /// If false, the operation is queued and the task is returned after the operation is queued.
+        /// </summary>
+        public bool WaitForComplete { get; }
+        /// <summary>
+        /// The cancellation token
+        /// </summary>
+        public CancellationToken CancelToken { get; set; }
+        /// <summary>
+        /// The type of operation the pending operation represents
+        /// </summary>
+        public abstract BackendActionType Operation { get; }
+        /// <summary>
+        /// The remote filename, if any
+        /// </summary>
+        public virtual string RemoteFilename => string.Empty;
+        /// <summary>
+        /// The remote size, if any
+        /// </summary>
+        public virtual long Size => -1L;
+
+        /// <summary>
+        /// Sets the operation as cancelled
+        /// </summary>
+        public abstract void SetCancelled();
+        /// <summary>
+        /// Sets the operation as failed
+        /// </summary>
+        /// <param name="ex">The exception</param>
+        public abstract void SetFailed(Exception ex);
+
+        /// <summary>
+        /// Creates a new pending operation
+        /// </summary>
+        /// <param name="context">The execution context</param>
+        /// <param name="waitForComplete">Whether to wait for the operation to complete</param>
+        /// <param name="cancelToken">The cancellation token</param>
+        public PendingOperationBase(ExecuteContext context, bool waitForComplete, CancellationToken cancelToken)
+        {
+            Context = context;
+            WaitForComplete = waitForComplete;
+            CancelToken = cancelToken;
+        }
+    }
+
+    /// <summary>
+    /// The basic queued backend operation
+    /// </summary>
+    /// <remarks>
+    /// Creates a new pending operation
+    /// </remarks>
+    /// <param name="context">The execution context</param>
+    /// <param name="waitForComplete">Whether to wait for the operation to complete</param>
+    /// <param name="cancelToken">The cancellation token</param>
+    private abstract class PendingOperation<TResult>(ExecuteContext context, bool waitForComplete, CancellationToken cancelToken) : PendingOperationBase(context, waitForComplete, cancelToken)
+    {
+        /// <summary>
+        /// A signal for the task completion
+        /// </summary>
+        private readonly TaskCompletionSource<TResult> taskCompleteSignal = new TaskCompletionSource<TResult>(TaskCreationOptions.RunContinuationsAsynchronously);
+        /// <summary>
+        /// The task that is completed when the operation is done
+        /// </summary>
+        public Task<TResult> GetResult() => taskCompleteSignal.Task;
+
+        /// <summary>
+        /// Sets the operation as complete
+        /// </summary>
+        public void SetComplete(TResult result)
+        {
+            taskCompleteSignal.TrySetResult(result);
+        }
+
+        /// <summary>
+        /// Sets the operation as failed
+        /// </summary>
+        /// <param name="ex">The exception</param>
+        public override void SetFailed(Exception ex)
+        {
+            taskCompleteSignal.TrySetException(ex);
+        }
+
+        /// <summary>
+        /// Sets the operation as cancelled
+        /// </summary>
+        public override void SetCancelled()
+        {
+            taskCompleteSignal.TrySetCanceled();
+        }
+
+        /// <summary>
+        /// Executes the operation
+        /// </summary>
+        /// <param name="backend">The backend to execute the operation on</param>
+        /// <param name="cancelToken">The cancellation token</param>
+        /// <returns>An awaitable task</returns>
+        public abstract Task<TResult> ExecuteAsync(IBackend backend, CancellationToken cancelToken);
+    }
+
+}

Duplicati/Library/Main/Backend/BackendManager.ProgressHandler.cs

@@ -0,0 +1,54 @@
+using Duplicati.Library.Main.Operation.Common;
+using Duplicati.Library.Utility;
+
+namespace Duplicati.Library.Main.Backend;
+
+#nullable enable
+
+partial class BackendManager
+{
+    /// <summary>
+    /// Represents a pending QuotaInfo operation
+    /// </summary>
+    private sealed record ProgressHandler(Options Options, IBackendWriter Stats, ITaskReader TaskReader)
+    {
+        /// <summary>
+        /// Stores the last throttle values
+        /// </summary>
+        /// <param name="Upload">The upload throttle value</param>
+        /// <param name="Download">The download throttle value</param>
+        private sealed record ThrottleValues(string? Upload, string? Download);
+
+        /// <summary>
+        /// The last throttle values
+        /// </summary>
+        private ThrottleValues lastThrottleValue = new(string.Empty, string.Empty);
+
+        /// <summary>
+        /// Handles the progress of a throttled stream
+        /// </summary>
+        /// <param name="ts">The throttled stream</param>
+        /// <param name="pg">The progress</param>
+        /// <param name="filename">The filename</param>
+        public void HandleProgress(ThrottledStream ts, long pg, string filename)
+        {
+            // This pauses and throws on cancellation, but ignores stop
+            TaskReader.TransferRendevouz().Await();
+
+            var prev = lastThrottleValue;
+
+            // Update the throttle speeds if they have changed
+            Options.RawOptions.TryGetValue("throttle-upload", out var tmpUpload);
+            Options.RawOptions.TryGetValue("throttle-download", out var tmpDownload);
+
+            if (tmpUpload != prev.Upload || tmpDownload != prev.Download)
+            {
+                ts.WriteSpeed = Options.MaxUploadPrSecond;
+                ts.ReadSpeed = Options.MaxDownloadPrSecond;
+                lastThrottleValue = new(tmpUpload, tmpDownload);
+            }
+
+            Stats.BackendProgressUpdater.UpdateProgress(pg);
+        }
+    }
+}

Duplicati/Library/Main/Backend/BackendManager.PutOperation.cs

@@ -0,0 +1,345 @@
+using System;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Duplicati.Library.Interface;
+using Duplicati.Library.Main.Volumes;
+using Duplicati.Library.Utility;
+using Newtonsoft.Json;
+
+namespace Duplicati.Library.Main.Backend;
+
+#nullable enable
+
+partial class BackendManager
+{
+    /// <summary>
+    /// Represents a pending PUT operation
+    /// </summary>
+    private class PutOperation : PendingOperation<bool>
+    {
+        /// <summary>
+        /// Different states of the operation
+        /// </summary>
+        private enum OperationState
+        {
+            /// <summary>
+            /// The operation is not started
+            /// </summary>
+            None,
+            /// <summary>
+            /// The upload has started
+            /// </summary>
+            Uploading,
+            /// <summary>
+            /// The upload has completed
+            /// </summary>
+            Uploaded,
+            /// <summary>
+            /// The operation has completed
+            /// </summary>
+            Completed
+        }
+
+        /// <summary>
+        /// The log tag for this class
+        /// </summary>
+        private static readonly string LOGTAG = Logging.Log.LogTagFromType<PutOperation>();
+
+        /// <summary>
+        /// The operation type
+        /// </summary>
+        public override BackendActionType Operation => BackendActionType.Put;
+        /// <summary>
+        /// The remote filename backing field
+        /// </summary>
+        private string remoteFilename;
+        /// <summary>
+        /// The remote filename
+        /// </summary>
+        public override string RemoteFilename => remoteFilename;
+        /// <summary>
+        /// The local temporary file
+        /// </summary>
+        public required TempFile? LocalTempfile { get; set; }
+        /// <summary>
+        /// Flag indicating if the file is not to be encrypted
+        /// </summary>
+        public required bool Unencrypted { get; set; }
+        /// <summary>
+        /// The local filename as a string
+        /// </summary>
+        public string LocalFilename => LocalTempfile;
+        /// <summary>
+        /// Flag to indicate if the file is not tracked in the database
+        /// </summary>
+        public required bool TrackedInDb { get; init; }
+        /// <summary>
+        /// The encryption and hashing task, the return value indicates if the hash and size was updated
+        /// </summary>
+        private Task<(string Hash, long Size)>? encryptionAndHashingTask;
+        /// <summary>
+        /// The state of the operation
+        /// </summary>
+        private OperationState operationState = OperationState.None;
+        /// <summary>
+        /// The pending index operation, if any
+        /// </summary>
+        private (IndexVolumeWriter Writer, PutOperation Operation)? indexOperation;
+        /// <summary>
+        /// The original index file, if any
+        /// </summary>
+        public required IndexVolumeWriter? OriginalIndexFile { get; init; }
+        /// <summary>
+        /// A callback that is invoked when the index volume is finished
+        /// </summary>
+        public required Action? IndexVolumeFinishedCallback { get; init; }
+
+        /// <summary>
+        /// Creates a new put operation 
+        /// </summary>
+        /// <param name="context">The execution context</param>
+        /// <param name="waitForComplete">True if the operation should wait for completion</param>
+        /// <param name="cancelToken">The cancellation token</param>
+        public PutOperation(string remotefilename, ExecuteContext context, bool waitForComplete, CancellationToken cancelToken)
+            : base(context, waitForComplete, cancelToken)
+        {
+            remoteFilename = remotefilename;
+        }
+
+        /// <summary>
+        /// Starts the encryption of the file
+        /// </summary>
+        /// <param name="options">The options to use</param>
+        public void StartEncryptionAndHashing()
+        {
+            if (encryptionAndHashingTask != null)
+                throw new Exception("Encryption already started");
+
+            // Run detached to allow encrypting while waiting in upload queue
+            encryptionAndHashingTask = Task.Run(() =>
+            {
+                if (!Context.Options.NoEncryption && !Unencrypted)
+                {
+                    using var enc = DynamicLoader.EncryptionLoader.GetModule(Context.Options.EncryptionModule, Context.Options.Passphrase, Context.Options.RawOptions);
+                    if (enc == null)
+                        throw new Exception(Strings.BackendMananger.EncryptionModuleNotFound(Context.Options.EncryptionModule));
+
+                    var tempfile = new TempFile();
+                    enc.Encrypt(LocalFilename, tempfile);
+                    DeleteLocalFile();
+                    LocalTempfile = tempfile;
+                }
+
+                if (!TrackedInDb)
+                    return (string.Empty, -1L);
+
+                return CalculateHashAndSize();
+
+            });
+        }
+
+        /// <summary>
+        /// Deletes the local temporary file
+        /// </summary>
+        private void DeleteLocalFile()
+        {
+            try { LocalTempfile?.Dispose(); }
+            catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "DeleteTemporaryFileError", ex, $"Failed to dispose temporary file: {LocalTempfile}"); }
+            finally { LocalTempfile = null; }
+        }
+
+        /// <summary>
+        /// Calculates the hash and size of the file
+        /// </summary>
+        /// <returns>The hash and size of the file</returns>
+        private (string Hash, long Size) CalculateHashAndSize()
+        {
+            var hash = CalculateFileHash(LocalFilename, Context.Options);
+            var size = new System.IO.FileInfo(LocalFilename).Length;
+
+            return (hash, size);
+
+        }
+
+        /// <summary>
+        /// Executes the operation
+        /// </summary>
+        /// <param name="backend">The backend to use</param>
+        /// <param name="cancelToken">The cancellation token</param>
+        /// <returns>An awaitable task</returns>
+        public override async Task<bool> ExecuteAsync(IBackend backend, CancellationToken cancelToken)
+        {
+            // This operation is slightly more complex than the others, as it involves two files.
+            // To keep the rest of the code simpler, the upload treats the pair as a single operation
+            // for the caller, but internally it is two separate operations.
+
+            // If there is a retry, this function needs to handle a retry in various states
+            // And, if the block file has been renamed, the index file needs to be rewritten
+
+            if (operationState == OperationState.Completed)
+                throw new Exception("Operation already completed");
+
+            // If this is a retry after the block has uploaded, pass the operating to the index file upload
+            if (operationState == OperationState.Uploaded && indexOperation != null)
+            {
+                await indexOperation.Value.Operation.ExecuteAsync(backend, cancelToken).ConfigureAwait(false);
+                indexOperation.Value.Writer.Dispose();
+                operationState = OperationState.Completed;
+                return true;
+            }
+
+            // Check if the operation is in a valid state
+            if (operationState == OperationState.Uploaded)
+                throw new Exception("Operation already uploaded");
+
+            if (this.encryptionAndHashingTask == null)
+                throw new Exception("Encryption not started");
+
+            // On retry attempts, we get the same value, without recalculating
+            (var hash, var size) = await encryptionAndHashingTask.ConfigureAwait(false);
+
+            // On first upload attempt, calculate the hash and size
+            if (operationState == OperationState.None && TrackedInDb)
+                Context.Database.LogRemoteVolumeUpdated(RemoteFilename, RemoteVolumeState.Uploading, size, hash);
+
+            // First attempt here, finish the index file now that all information is known
+            if (OriginalIndexFile != null && indexOperation == null)
+            {
+                Context.Database.LogRemoteVolumeUpdated(OriginalIndexFile.RemoteFilename, RemoteVolumeState.Uploading, -1, null);
+
+                // Prepare an upload operation for the index file
+                var req2 = new PutOperation(OriginalIndexFile.RemoteFilename, Context, false, cancelToken)
+                {
+                    LocalTempfile = OriginalIndexFile.TempFile,
+                    TrackedInDb = TrackedInDb,
+                    Unencrypted = Unencrypted,
+                    IndexVolumeFinishedCallback = null,
+                    OriginalIndexFile = null
+                };
+
+                OriginalIndexFile.FinishVolume(hash, size);
+                IndexVolumeFinishedCallback?.Invoke();
+                OriginalIndexFile.Close();
+
+                indexOperation = (OriginalIndexFile, req2);
+            }
+
+            // If we have previously attempted to upload, we need to rename the file
+            if (operationState == OperationState.Uploading)
+                RenameFileAfterError(size);
+
+            // Flag for next attempt, if any
+            operationState = OperationState.Uploading;
+
+            await PerformUpload(backend, hash, size, cancelToken).ConfigureAwait(false);
+
+            operationState = OperationState.Uploaded;
+
+            // Upload completed, prepare the index file if any
+            if (indexOperation != null)
+            {
+                // TODO: It would be better if we encrypt the index file while uploading the block file
+                // since most operations work correctly. But if the upload of the block file fails
+                // we need to deal with decryption, or keep the unencrypted file around
+                indexOperation.Value.Operation.StartEncryptionAndHashing();
+                await indexOperation.Value.Operation.ExecuteAsync(backend, cancelToken).ConfigureAwait(false);
+                indexOperation.Value.Writer.Dispose();
+            }
+
+            operationState = OperationState.Completed;
+            return true;
+        }
+
+        /// <summary>
+        /// Performs the actual upload of a file
+        /// </summary>
+        /// <param name="backend">The backend to upload to</param>
+        /// <param name="Hash">The hash of the file</param>
+        /// <param name="Size">The size of the file</param>
+        /// <param name="cancelToken">The cancellation token</param>
+        /// <returns>An awaitable task</returns>
+        private async Task PerformUpload(IBackend backend, string Hash, long Size, CancellationToken cancelToken)
+        {
+            Context.Database.LogRemoteOperation("put", RemoteFilename, JsonConvert.SerializeObject(new { Size = Size, Hash = Hash }));
+            Context.Statwriter.SendEvent(BackendActionType.Put, BackendEventType.Started, RemoteFilename, Size);
+
+            var begin = DateTime.Now;
+
+            if (backend is IStreamingBackend streamingBackend && !Context.Options.DisableStreamingTransfers)
+            {
+                using (var fs = System.IO.File.OpenRead(LocalFilename))
+                using (var act = new StreamUtil.TimeoutObservingStream(fs) { ReadTimeout = backend is ITimeoutExemptBackend ? Timeout.Infinite : Context.Options.ReadWriteTimeout })
+                using (var ts = new ThrottledStream(act, Context.Options.MaxUploadPrSecond, 0))
+                using (var pgs = new ProgressReportingStream(ts, pg => Context.HandleProgress(ts, pg, RemoteFilename)))
+                using (var linkedToken = CancellationTokenSource.CreateLinkedTokenSource(cancelToken, act.TimeoutToken))
+                    await streamingBackend.PutAsync(RemoteFilename, pgs, linkedToken.Token).ConfigureAwait(false);
+            }
+            else
+                await backend.PutAsync(RemoteFilename, LocalFilename, cancelToken).ConfigureAwait(false);
+
+            var duration = DateTime.Now - begin;
+            Logging.Log.WriteProfilingMessage(LOGTAG, "UploadSpeed", "Uploaded {0} in {1}, {2}/s", Library.Utility.Utility.FormatSizeString(Size), duration, Library.Utility.Utility.FormatSizeString((long)(Size / duration.TotalSeconds)));
+
+            if (TrackedInDb)
+                Context.Database.LogRemoteVolumeUpdated(RemoteFilename, RemoteVolumeState.Uploaded, Size, Hash);
+
+            Context.Statwriter.SendEvent(BackendActionType.Put, BackendEventType.Completed, RemoteFilename, Size);
+
+            if (Context.Options.ListVerifyUploads)
+            {
+                var f = backend.List().FirstOrDefault(n => n.Name.Equals(RemoteFilename, StringComparison.OrdinalIgnoreCase));
+                if (f == null)
+                    throw new Exception(string.Format($"List verify failed, file was not found after upload: {RemoteFilename}"));
+                else if (f.Size != Size && f.Size >= 0)
+                    throw new Exception(string.Format($"List verify failed for file: {f.Name}, size was {f.Size} but expected to be {Size}"));
+            }
+
+            DeleteLocalFile();
+        }
+
+        /// <summary>
+        /// Renames the remote target file after an error, and updates the index file, if any
+        /// </summary>
+        /// <param name="Size">The size of the file</param>
+        private void RenameFileAfterError(long Size)
+        {
+            var p = VolumeBase.ParseFilename(RemoteFilename);
+            var guid = VolumeWriterBase.GenerateGuid();
+            var time = p.Time.Ticks == 0 ? p.Time : p.Time.AddSeconds(1);
+            var newname = VolumeBase.GenerateFilename(p.FileType, p.Prefix, guid, time, p.CompressionModule, p.EncryptionModule);
+            var oldname = RemoteFilename;
+
+            Context.Statwriter.SendEvent(BackendActionType.Put, BackendEventType.Rename, oldname, Size);
+            Context.Statwriter.SendEvent(BackendActionType.Put, BackendEventType.Rename, newname, Size);
+            Logging.Log.WriteInformationMessage(LOGTAG, "RenameRemoteTargetFile", "Renaming \"{0}\" to \"{1}\"", oldname, newname);
+            Context.Database.LogRemoteVolumeRenamed(oldname, newname);
+            remoteFilename = newname;
+
+            // If there is an index file attached to the block file, 
+            // it references the block filename, so we create a new index file
+            // which is a copy of the current, but with the new name
+            if (indexOperation != null)
+            {
+                IndexVolumeWriter? wr = null;
+                try
+                {
+                    var hashsize = HashFactory.HashSizeBytes(Context.Options.BlockHashAlgorithm);
+                    wr = new IndexVolumeWriter(Context.Options);
+                    using (var rd = new IndexVolumeReader(p.CompressionModule, indexOperation.Value.Operation.LocalFilename, Context.Options, hashsize))
+                        wr.CopyFrom(rd, x => x == oldname ? newname : x);
+                    indexOperation.Value.Writer.Dispose();
+                    indexOperation = (wr, indexOperation.Value.Operation);
+                    indexOperation.Value.Operation.LocalTempfile?.Dispose();
+                    indexOperation.Value.Operation.LocalTempfile = wr.TempFile;
+                    wr.Close();
+                }
+                catch
+                {
+                    wr?.Dispose();
+                    throw;
+                }
+            }
+        }
+    }
+}

Duplicati/Library/Main/Backend/BackendManager.QuotaInfoOperation.cs

@@ -0,0 +1,46 @@
+using System.Threading;
+using System.Threading.Tasks;
+using Duplicati.Library.Interface;
+
+namespace Duplicati.Library.Main.Backend;
+
+#nullable enable
+
+partial class BackendManager
+{
+    /// <summary>
+    /// Represents a pending QuotaInfo operation
+    /// </summary>
+    /// <remarks>
+    /// Creates a new QuotaInfoOperation
+    /// </remarks>
+    /// <param name="context">The execution context</param>
+    /// <param name="cancelToken">The cancellation token</param>
+    private class QuotaInfoOperation(ExecuteContext context, CancellationToken cancelToken)
+        : PendingOperation<IQuotaInfo?>(context, true, cancelToken)
+    {
+        /// <summary>
+        /// The operation type
+        /// </summary>
+        public override BackendActionType Operation => BackendActionType.QuotaInfo;
+
+        /// <summary>
+        /// Executes the operation
+        /// </summary>
+        /// <param name="backend">The backend to get the quota info from</param>
+        /// <param name="cancelToken">The cancellation token</param>
+        /// <returns>An awaitable task</returns>
+        public override async Task<IQuotaInfo?> ExecuteAsync(IBackend backend, CancellationToken cancelToken)
+        {
+            IQuotaInfo? quotaInfo = null;
+            if (backend is IQuotaEnabledBackend quotaEnabledBackend && !Context.Options.QuotaDisable)
+            {
+                Context.Statwriter.SendEvent(BackendActionType.QuotaInfo, BackendEventType.Started, null, 0);
+                quotaInfo = await quotaEnabledBackend.GetQuotaInfoAsync(cancelToken).ConfigureAwait(false);
+            }
+
+            return quotaInfo;
+        }
+
+    }
+}

Duplicati/Library/Main/Backend/BackendManager.WaitForEmptyOperation.cs

@@ -0,0 +1,37 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Duplicati.Library.Interface;
+
+namespace Duplicati.Library.Main.Backend;
+
+#nullable enable
+
+partial class BackendManager
+{
+    /// <summary>
+    /// Represents a pending QuotaInfo operation
+    /// </summary>
+    /// <remarks>
+    /// Creates a new WaitForEmptyOperation
+    /// </remarks>
+    /// <param name="context">The execution context</param>
+    /// <param name="cancelToken">The cancellation token</param>
+    private class WaitForEmptyOperation(ExecuteContext context, CancellationToken cancelToken)
+        : PendingOperation<bool>(context, true, cancelToken)
+    {
+        /// <summary>
+        /// The operation type
+        /// </summary>
+        public override BackendActionType Operation => BackendActionType.WaitForEmpty;
+
+        /// <summary>
+        /// Executes the operation
+        /// </summary>
+        /// <param name="backend">The backend to get the quota info from</param>
+        /// <param name="cancelToken">The cancellation token</param>
+        /// <returns>An awaitable task</returns>
+        public override Task<bool> ExecuteAsync(IBackend backend, CancellationToken cancelToken)
+            => Task.FromResult(true);
+    }
+}

Duplicati/Library/Main/Backend/BackendManager.cs

@@ -0,0 +1,363 @@
+using System;
+using System.Collections.Generic;
+using System.Data;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Threading;
+using System.Threading.Tasks;
+using CoCoL;
+using Duplicati.Library.Interface;
+using Duplicati.Library.Main.Database;
+using Duplicati.Library.Main.Operation.Common;
+using Duplicati.Library.Main.Volumes;
+using Duplicati.Library.Utility;
+
+namespace Duplicati.Library.Main.Backend;
+
+#nullable enable
+
+/// <summary>
+/// The backend manager
+/// </summary>
+internal partial class BackendManager : IBackendManager
+{
+    /// <summary>
+    /// The log tag for the class
+    /// </summary>
+    private static readonly string LOGTAG = Logging.Log.LogTagFromType<BackendManager>();
+    /// <summary>
+    /// List of backend instances that are currently in use
+    /// </summary>
+    private readonly List<IBackend> backendPool = [];
+
+    /// <summary>
+    /// The channel for issuing and handling requests
+    /// </summary>
+    private readonly IChannel<PendingOperationBase> requestChannel = ChannelManager.CreateChannel<PendingOperationBase>(name: "BackendManager");
+
+    /// <summary>
+    /// The queue runner task
+    /// </summary>
+    private readonly Task queueRunner;
+
+    /// <summary>
+    /// The last file read size
+    /// </summary>
+    public long LastReadSize { get; private set; }
+    /// <summary>
+    /// The last file write size
+    /// </summary>
+    public long LastWriteSize { get; private set; }
+
+    /// <summary>
+    /// The execution context
+    /// </summary>
+    private readonly ExecuteContext context;
+
+    /// <summary>
+    /// Flag keeping track of whether the object has been disposed
+    /// </summary>
+    private bool isDisposed = false;
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="BackendManager"/> class.
+    /// </summary>
+    /// <param name="backendUrl">The backend URL</param>
+    /// <param name="options">The options</param>
+    /// <param name="backendWriter">The backend writer</param>
+    /// <param name="taskReader">The task reader</param>
+    public BackendManager(string backendUrl, Options options, IBackendWriter backendWriter, ITaskReader taskReader)
+    {
+        if (string.IsNullOrWhiteSpace(backendUrl))
+            throw new ArgumentNullException(nameof(backendUrl));
+
+        // To avoid excessive parameter passing, the context is captured here
+        context = new ExecuteContext(
+            new ProgressHandler(options, backendWriter, taskReader).HandleProgress,
+            backendWriter ?? throw new ArgumentNullException(nameof(backendWriter)),
+            new DatabaseCollector(),
+            taskReader ?? throw new ArgumentNullException(nameof(taskReader)),
+            options ?? throw new ArgumentNullException(nameof(options))
+        );
+
+        // The BackendManager class is a wrapper that essentially sends
+        // requests into a queue and processes them in order.
+        // The Handler class is the one that actually processes the requests.
+        queueRunner = Handler.RunHandlerAsync(
+            requestChannel,
+            backendUrl,
+            context);
+    }
+
+    /// <summary>
+    /// Enters a task into the queue for processing.
+    /// </summary>
+    /// <param name="op">The operation to queue</param>
+    /// <returns>An awaitable task</returns>
+    private async Task QueueTask(PendingOperationBase op)
+    {
+        if (queueRunner.IsCompleted)
+        {
+            if (queueRunner.IsFaulted)
+                await queueRunner.ConfigureAwait(false);
+            if (queueRunner.IsCanceled)
+                throw new OperationCanceledException("Backend manager is stopped", queueRunner.Exception);
+            throw new InvalidOperationException("Backend manager is stopped");
+        }
+
+        try
+        {
+            await requestChannel.WriteAsync(op).ConfigureAwait(false);
+        }
+        catch (RetiredException ex)
+        {
+            // Try to get a better error message
+            if (queueRunner.IsFaulted)
+                await queueRunner.ConfigureAwait(false);
+
+            throw new InvalidOperationException("Backend manager is stopped", ex);
+        }
+    }
+
+    /// <summary>
+    /// Calculates the hash of a file
+    /// </summary>
+    /// <param name="filename">The filename</param>
+    /// <param name="options">The options</param>
+    /// <returns>The hash</returns>
+    protected static string CalculateFileHash(string filename, Options options)
+    {
+        using (var fs = System.IO.File.OpenRead(filename))
+        using (var hasher = HashFactory.CreateHasher(options.FileHashAlgorithm))
+            return Convert.ToBase64String(hasher.ComputeHash(fs));
+    }
+
+    /// <summary>
+    /// Deletes a remote file
+    /// </summary>
+    /// <param name="remotename">The name of the remote file</param>
+    /// <param name="size">The size of the remote file, for statistics</param>
+    /// <param name="waitForComplete">True if the operation should wait for the file to actually be deleted. If this argument is false, the task will complete once the operation is queued</param>
+    /// <param name="cancelToken">The cancellation token</param>
+    /// <returns>An awaitable task</returns>
+    public async Task DeleteAsync(string remotename, long size, bool waitForComplete, CancellationToken cancelToken)
+    {
+        var op = new DeleteOperation(remotename, size, context, waitForComplete, cancelToken);
+        await QueueTask(op).ConfigureAwait(false);
+        await op.GetResult().ConfigureAwait(false);
+    }
+
+    /// <summary>
+    /// Gets a file from the remote location
+    /// </summary>
+    /// <param name="remotename">The name of the remote file</param>
+    /// <param name="hash">The hash of the remote file, for verification</param>
+    /// <param name="size">The size of the remote file, for verification</param>
+    /// <param name="cancelToken">The cancellation token</param>
+    /// <returns>A temporary file with the contents of the remote file</returns>
+    public async Task<TempFile> GetAsync(string remotename, string hash, long size, CancellationToken cancelToken)
+    {
+        var op = new GetOperation(remotename, size, context, cancelToken)
+        {
+            Hash = hash
+        };
+        await QueueTask(op).ConfigureAwait(false);
+        (var file, var _, var downloadSize) = await op.GetResult().ConfigureAwait(false);
+        LastReadSize = downloadSize;
+        return file;
+    }
+
+    /// <summary>
+    /// Gets quota information from the backend
+    /// </summary>
+    /// <param name="cancelToken">The cancellation token</param>
+    /// <returns>The quota information</returns>
+    public async Task<IQuotaInfo?> GetQuotaInfoAsync(CancellationToken cancelToken)
+    {
+        var op = new QuotaInfoOperation(context, cancelToken);
+        await QueueTask(op).ConfigureAwait(false);
+        return await op.GetResult().ConfigureAwait(false);
+    }
+
+    /// <summary>
+    /// Gets a file from the remote location, along with the hash and size of the file
+    /// </summary>
+    /// <param name="remotename">The name of the remote file</param>
+    /// <param name="hash">The hash of the remote file, or null if not known</param>
+    /// <param name="size">The size of the remote file, or -1 if not known</param>
+    /// <param name="cancelToken">The cancellation token</param>
+    /// <returns>A tuple containing the temporary file, the hash of the file, and the size of the file</returns>
+    public async Task<(TempFile File, string Hash, long Size)> GetWithInfoAsync(string remotename, string hash, long size, CancellationToken cancelToken)
+    {
+        var op = new GetOperation(remotename, size, context, cancelToken)
+        {
+            Hash = hash
+        };
+        await QueueTask(op).ConfigureAwait(false);
+        (var file, var downloadHash, var downloadSize) = await op.GetResult().ConfigureAwait(false);
+        LastReadSize = downloadSize;
+        return (file, downloadHash, downloadSize);
+    }
+
+    /// <summary>
+    /// Lists files on the remote destination
+    /// </summary>
+    /// <param name="cancelToken">The cancellation token</param>
+    /// <returns>The list of files</returns>
+    public async Task<IEnumerable<Interface.IFileEntry>> ListAsync(CancellationToken cancelToken)
+    {
+        var op = new ListOperation(context, cancelToken);
+        await QueueTask(op).ConfigureAwait(false);
+        return await op.GetResult().ConfigureAwait(false);
+    }
+
+    /// <summary>
+    /// Uploads a volume to the remote location
+    /// </summary>
+    /// <param name="volume">The volume to upload</param>
+    /// <param name="indexVolume">The index volume to upload, if any</param>
+    /// <param name="indexVolumeFinished">The callback to call when the index volume is finished</param>
+    /// <param name="waitForComplete">True if the operation should wait for the file to actually be uploaded. If this argument is false, the task will complete once the operation is queued</param>
+    /// <param name="cancelToken">The cancellation token</param>
+    /// <returns>An awaitable task</returns>
+    public async Task PutAsync(VolumeWriterBase volume, IndexVolumeWriter? indexVolume, Action? indexVolumeFinished, bool waitForComplete, CancellationToken cancelToken)
+    {
+        volume.Close();
+
+        var op = new PutOperation(volume.RemoteFilename, context, waitForComplete, cancelToken)
+        {
+            LocalTempfile = volume.TempFile,
+            OriginalIndexFile = indexVolume,
+            Unencrypted = false,
+            TrackedInDb = true,
+            IndexVolumeFinishedCallback = indexVolumeFinished
+        };
+
+        // Prepare encryption
+        op.StartEncryptionAndHashing();
+        await QueueTask(op).ConfigureAwait(false);
+        await op.GetResult().ConfigureAwait(false);
+    }
+
+    /// <summary>
+    /// Uploads a verification file to the remote location without encryption
+    /// </summary>
+    /// <param name="remotename">The name of the remote file</param>
+    /// <param name="tempFile">The temporary file to upload</param>
+    /// <param name="cancelToken">The cancellation token</param>
+    /// <returns>An awaitable task</returns>
+    public async Task PutVerificationFileAsync(string remotename, TempFile tempFile, CancellationToken cancelToken)
+    {
+        var op = new PutOperation(remotename, context, true, cancelToken)
+        {
+            LocalTempfile = tempFile,
+            Unencrypted = true, // Avoid encrypting
+            TrackedInDb = false, // Not tracked
+            OriginalIndexFile = null,
+            IndexVolumeFinishedCallback = null
+        };
+
+        // Sets the task as already completed
+        op.StartEncryptionAndHashing();
+        await QueueTask(op).ConfigureAwait(false);
+        await op.GetResult().ConfigureAwait(false);
+    }
+
+    /// <summary>
+    /// Waits for the backend queue to be empty and flushes the database messages
+    /// </summary>
+    /// <param name="cancellationToken">The cancellation token</param>
+    /// <returns>An awaitable task</returns>
+    public async Task WaitForEmptyAsync(LocalDatabase database, IDbTransaction? transaction, CancellationToken cancellationToken)
+    {
+        context.Database.FlushPendingMessages(database, transaction);
+
+        var op = new WaitForEmptyOperation(context, cancellationToken);
+        await QueueTask(op).ConfigureAwait(false);
+        await op.GetResult().ConfigureAwait(false);
+
+        context.Database.FlushPendingMessages(database, transaction);
+    }
+
+    /// <summary>
+    /// Stops the backend manager and flushes any pending messages to the database
+    /// </summary>
+    /// <param name="database">The database to write pending messages to</param>
+    /// <param name="transaction">The transaction to use, if any</param>
+    public async Task StopRunnerAndFlushMessages(LocalDatabase database, IDbTransaction? transaction)
+    {
+        await requestChannel.RetireAsync().ConfigureAwait(false);
+        context.Database.FlushPendingMessages(database, transaction);
+        if (queueRunner.IsFaulted)
+            Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerShutdown", queueRunner.Exception, "Backend manager queue runner crashed");
+    }
+
+    /// <summary>
+    /// Stops the backend manager and discards any pending messages
+    /// </summary>
+    public void StopRunnerAndDiscardMessages()
+    {
+        requestChannel.RetireAsync().Await();
+        if (queueRunner.IsFaulted)
+            Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerShutdown", queueRunner.Exception, "Backend manager queue runner crashed");
+        context.Database.ClearPendingMessages();
+    }
+
+    /// <summary>
+    /// Performs a download of the files specified, with pre-fetch to overlap the download and processing
+    /// </summary>
+    /// <param name="volumes">The volumes to download</param>
+    /// <param name="cancelToken">The cancellation token</param>
+    /// <returns>The downloaded files and the volume they came from</returns>
+    public async IAsyncEnumerable<(TempFile File, string Hash, long Size, string Name)> GetFilesOverlappedAsync(IEnumerable<IRemoteVolume> volumes, [EnumeratorCancellation] CancellationToken cancelToken)
+    {
+        var prevVolume = volumes.FirstOrDefault();
+        if (prevVolume == null)
+            yield break;
+
+        // Get the first volume, so we do not have pending parallel transfers
+        var prevResult = await GetWithInfoAsync(prevVolume.Name, prevVolume.Hash, prevVolume.Size, cancelToken);
+
+        foreach (var volume in volumes.Skip(1))
+        {
+            // Prepare the next volume, while processing the previous one
+            var nextTask = GetWithInfoAsync(volume.Name, volume.Hash, volume.Size, cancelToken);
+
+            // Assuming we do not throw while yielding, otherwise we would need to dispose nextTask
+            yield return (prevResult.File, prevResult.Hash, prevResult.Size, prevVolume.Name);
+            prevResult.File.Dispose();
+
+            // Set up for next iteration
+            prevVolume = volume;
+            prevResult = await nextTask;
+        }
+
+        // Return the last result
+        yield return (prevResult.File, prevResult.Hash, prevResult.Size, prevVolume.Name);
+        prevResult.File.Dispose();
+    }
+
+    /// <summary>
+    /// Disposes the backend manager
+    /// </summary>
+    public void Dispose()
+    {
+        if (isDisposed)
+            return;
+
+        isDisposed = true;
+        requestChannel.RetireAsync().Await();
+        context.Database.FlushMessagesToLog();
+
+        if (!queueRunner.IsCompleted)
+        {
+            Task.WhenAny(queueRunner, Task.Delay(1000)).Await();
+
+            if (!queueRunner.IsCompleted)
+                Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerShutdown", null, "Backend manager queue runner did not stop");
+            if (queueRunner.IsFaulted)
+                Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerShutdown", queueRunner.Exception, "Backend manager queue runner crashed");
+
+        }
+    }
+}

Duplicati/Library/Main/Controller.cs

@@ -29,6 +29,7 @@ using Duplicati.Library.Interface;
 using Duplicati.Library.Main.Database;
 using System.Threading.Tasks;
 using Duplicati.Library.Main.Operation.Common;
+using System.IO;
 
 namespace Duplicati.Library.Main
 {
@@ -41,7 +42,7 @@ namespace Duplicati.Library.Main
         /// <summary>
         /// The backend url
         /// </summary>
-        private string m_backend;
+        private string m_backendUrl;
         /// <summary>
         /// The parsed type-safe version of the commandline options
         /// </summary>
@@ -83,11 +84,11 @@ namespace Duplicati.Library.Main
         /// <summary>
         /// Constructs a new interface for performing backup and restore operations
         /// </summary>
-        /// <param name="backend">The url for the backend to use</param>
+        /// <param name="backendUrl">The url for the backend to use</param>
         /// <param name="options">All required options</param>
-        public Controller(string backend, Dictionary<string, string> options, IMessageSink messageSink)
+        public Controller(string backendUrl, Dictionary<string, string> options, IMessageSink messageSink)
         {
-            m_backend = backend;
+            m_backendUrl = backendUrl;
             m_options = new Options(options);
             m_messageSink = messageSink;
         }
@@ -115,19 +116,19 @@ namespace Duplicati.Library.Main
 
         public Duplicati.Library.Interface.IBackupResults Backup(string[] inputsources, IFilter filter = null)
         {
-            Library.UsageReporter.Reporter.Report("USE_BACKEND", new Library.Utility.Uri(m_backend).Scheme);
+            Library.UsageReporter.Reporter.Report("USE_BACKEND", new Library.Utility.Uri(m_backendUrl).Scheme);
             Library.UsageReporter.Reporter.Report("USE_COMPRESSION", m_options.CompressionModule);
             Library.UsageReporter.Reporter.Report("USE_ENCRYPTION", m_options.EncryptionModule);
 
             CheckAutoCompactInterval();
             CheckAutoVacuumInterval();
 
-            return RunAction(new BackupResults(), ref inputsources, ref filter, (result) =>
+            return RunAction(new BackupResults(), ref inputsources, ref filter, (result, backendManager) =>
             {
 
-                using (var h = new Operation.BackupHandler(m_backend, m_options, result))
+                using (var h = new Operation.BackupHandler(m_backendUrl, m_options, result))
                 {
-                    h.RunAsync(ExpandInputSources(inputsources, filter), filter).Await();
+                    h.RunAsync(ExpandInputSources(inputsources, filter), backendManager, filter).Await();
                 }
 
                 Library.UsageReporter.Reporter.Report("BACKUP_FILECOUNT", result.ExaminedFiles);
@@ -138,9 +139,9 @@ namespace Duplicati.Library.Main
 
         public Library.Interface.IRestoreResults Restore(string[] paths, Library.Utility.IFilter filter = null)
         {
-            return RunAction(new RestoreResults(), ref paths, ref filter, (result) =>
+            return RunAction(new RestoreResults(), ref paths, ref filter, (result, backendManager) =>
             {
-                new Operation.RestoreHandler(m_backend, m_options, result).Run(paths, filter);
+                new Operation.RestoreHandler(m_options, result).Run(paths, backendManager, filter);
 
                 Library.UsageReporter.Reporter.Report("RESTORE_FILECOUNT", result.RestoredFiles);
                 Library.UsageReporter.Reporter.Report("RESTORE_FILESIZE", result.SizeOfRestoredFiles);
@@ -150,25 +151,25 @@ namespace Duplicati.Library.Main
 
         public Duplicati.Library.Interface.IRestoreControlFilesResults RestoreControlFiles(IEnumerable<string> files = null, Library.Utility.IFilter filter = null)
         {
-            return RunAction(new RestoreControlFilesResults(), ref filter, (result) =>
+            return RunAction(new RestoreControlFilesResults(), ref filter, (result, backendManager) =>
             {
-                new Operation.RestoreControlFilesHandler(m_backend, m_options, result).Run(files, filter);
+                new Operation.RestoreControlFilesHandler(m_options, result).Run(files, backendManager, filter);
             });
         }
 
         public Duplicati.Library.Interface.IDeleteResults Delete()
         {
-            return RunAction(new DeleteResults(), (result) =>
+            return RunAction(new DeleteResults(), (result, backendManager) =>
             {
-                new Operation.DeleteHandler(m_backend, m_options, result).Run();
+                new Operation.DeleteHandler(m_options, result).Run(backendManager);
             });
         }
 
         public Duplicati.Library.Interface.IRepairResults Repair(Library.Utility.IFilter filter = null)
         {
-            return RunAction(new RepairResults(), ref filter, (result) =>
+            return RunAction(new RepairResults(), ref filter, (result, backendManager) =>
             {
-                new Operation.RepairHandler(m_backend, m_options, result).Run(filter);
+                new Operation.RepairHandler(m_options, result).Run(backendManager, filter);
             });
         }
 
@@ -184,40 +185,39 @@ namespace Duplicati.Library.Main
 
         public Duplicati.Library.Interface.IListResults List(IEnumerable<string> filterstrings, Library.Utility.IFilter filter)
         {
-            return RunAction(new ListResults(), ref filter, (result) =>
+            return RunAction(new ListResults(), ref filter, (result, backendManager) =>
             {
-                new Operation.ListFilesHandler(m_backend, m_options, result).Run(filterstrings, filter);
+                new Operation.ListFilesHandler(m_options, result).Run(backendManager, filterstrings, filter).Await();
             });
         }
 
         public Duplicati.Library.Interface.IListResults ListControlFiles(IEnumerable<string> filterstrings, Library.Utility.IFilter filter)
         {
-            return RunAction(new ListResults(), ref filter, (result) =>
+            return RunAction(new ListResults(), ref filter, (result, backendManager) =>
             {
-                new Operation.ListControlFilesHandler(m_backend, m_options, result).Run(filterstrings, filter);
+                new Operation.ListControlFilesHandler(m_options, result).Run(backendManager, filterstrings, filter);
             });
         }
 
         public Duplicati.Library.Interface.IListRemoteResults ListRemote()
         {
-            return RunAction(new ListRemoteResults(), (result) =>
+            return RunAction(new ListRemoteResults(), (result, backendManager) =>
             {
                 using (var tf = System.IO.File.Exists(m_options.Dbpath) ? null : new Library.Utility.TempFile())
                 using (var db = new Database.LocalDatabase(((string)tf) ?? m_options.Dbpath, "list-remote", true))
-                using (var bk = new BackendManager(m_backend, m_options, result.BackendWriter, null))
-                    result.SetResult(bk.List());
+                    result.SetResult(backendManager.ListAsync(CancellationToken.None).Await());
             });
         }
 
         public Duplicati.Library.Interface.IListRemoteResults DeleteAllRemoteFiles()
         {
-            return RunAction(new ListRemoteResults(), (result) =>
+            return RunAction(new ListRemoteResults(), (result, backendManager) =>
             {
+                var cancelToken = CancellationToken.None;
                 result.OperationProgressUpdater.UpdatePhase(OperationPhase.Delete_Listing);
-                using (var bk = new BackendManager(m_backend, m_options, result.BackendWriter, null))
                 {
                     // Only delete files that match the expected pattern and prefix
-                    var list = bk.List()
+                    var list = backendManager.ListAsync(cancelToken).Await()
                         .Select(x => Volumes.VolumeBase.ParseFilename(x))
                         .Where(x => x != null)
                         .Where(x => x.Prefix == m_options.Prefix)
@@ -243,7 +243,7 @@ namespace Duplicati.Library.Main
                     {
                         try
                         {
-                            bk.Delete(list[i].File.Name, list[i].File.Size, true);
+                            backendManager.DeleteAsync(list[i].File.Name, list[i].File.Size, true, cancelToken).Await();
                         }
                         catch (Exception ex)
                         {
@@ -260,9 +260,9 @@ namespace Duplicati.Library.Main
         {
             CheckAutoVacuumInterval();
 
-            return RunAction(new CompactResults(), (result) =>
+            return RunAction(new CompactResults(), (result, backendManager) =>
             {
-                new Operation.CompactHandler(m_backend, m_options, result).Run();
+                new Operation.CompactHandler(m_options, result).Run(backendManager).Await();
             });
         }
 
@@ -270,10 +270,10 @@ namespace Duplicati.Library.Main
         {
             var filelistfilter = Operation.RestoreHandler.FilterNumberedFilelist(m_options.Time, m_options.Version, singleTimeMatch: true);
 
-            return RunAction(new RecreateDatabaseResults(), ref filter, (result) =>
+            return RunAction(new RecreateDatabaseResults(), ref filter, (result, backendManager) =>
             {
-                using (var h = new Operation.RecreateDatabaseHandler(m_backend, m_options, result))
-                    h.RunUpdate(filter, filelistfilter);
+                using (var h = new Operation.RecreateDatabaseHandler(m_options, result))
+                    h.RunUpdate(backendManager, filter, filelistfilter, null);
             });
         }
 
@@ -281,7 +281,7 @@ namespace Duplicati.Library.Main
         {
             var t = new string[] { targetpath };
 
-            return RunAction(new CreateLogDatabaseResults(), ref t, (result) =>
+            return RunAction(new CreateLogDatabaseResults(), ref t, (result, backendManager) =>
             {
                 new Operation.CreateBugReportHandler(t[0], m_options, result).Run();
             });
@@ -291,15 +291,15 @@ namespace Duplicati.Library.Main
         {
             var t = new string[] { baseVersion, targetVersion };
 
-            return RunAction(new ListChangesResults(), ref t, ref filter, (result) =>
+            return RunAction(new ListChangesResults(), ref t, ref filter, (result, backendManager) =>
             {
-                new Operation.ListChangesHandler(m_backend, m_options, result).Run(t[0], t[1], filterstrings, filter, callback);
+                new Operation.ListChangesHandler(m_options, result).Run(t[0], t[1], backendManager, filterstrings, filter, callback);
             });
         }
 
         public Duplicati.Library.Interface.IListAffectedResults ListAffected(List<string> args, Action<Duplicati.Library.Interface.IListAffectedResults> callback = null)
         {
-            return RunAction(new ListAffectedResults(), (result) =>
+            return RunAction(new ListAffectedResults(), (result, backendManager) =>
             {
                 new Operation.ListAffected(m_options, result).Run(args, callback);
             });
@@ -310,9 +310,9 @@ namespace Duplicati.Library.Main
             if (!m_options.RawOptions.ContainsKey("full-remote-verification"))
                 m_options.RawOptions["full-remote-verification"] = "true";
 
-            return RunAction(new TestResults(), (result) =>
+            return RunAction(new TestResults(), (result, backendManager) =>
             {
-                new Operation.TestHandler(m_backend, m_options, result).Run(samples);
+                new Operation.TestHandler(m_options, result).Run(samples, backendManager);
             });
         }
 
@@ -325,7 +325,7 @@ namespace Duplicati.Library.Main
             var filtertag = Logging.Log.LogTagFromType(typeof(Operation.Backup.FileEnumerationProcess));
             using (Logging.Log.StartScope(m_messageSink.WriteMessage, x => x.FilterTag.Contains(filtertag)))
             {
-                return RunAction(new TestFilterResults(), ref paths, ref filter, (result) =>
+                return RunAction(new TestFilterResults(), ref paths, ref filter, (result, backendManager) =>
                 {
                     new Operation.TestFilterHandler(m_options, result).RunAsync(ExpandInputSources(paths, filter), filter).Await();
                 });
@@ -334,7 +334,7 @@ namespace Duplicati.Library.Main
 
         public Library.Interface.ISystemInfoResults SystemInfo()
         {
-            return RunAction(new SystemInfoResults(), result =>
+            return RunAction(new SystemInfoResults(), (result, backendManager) =>
             {
                 Operation.SystemInfoHandler.Run(result);
             });
@@ -342,25 +342,25 @@ namespace Duplicati.Library.Main
 
         public Library.Interface.IPurgeFilesResults PurgeFiles(Library.Utility.IFilter filter)
         {
-            return RunAction(new PurgeFilesResults(), result =>
+            return RunAction(new PurgeFilesResults(), (result, backendManager) =>
             {
-                new Operation.PurgeFilesHandler(m_backend, m_options, result).Run(filter);
+                new Operation.PurgeFilesHandler(m_options, result).Run(backendManager, filter);
             });
         }
 
         public Library.Interface.IListBrokenFilesResults ListBrokenFiles(Library.Utility.IFilter filter, Func<long, DateTime, long, string, long, bool> callbackhandler = null)
         {
-            return RunAction(new ListBrokenFilesResults(), result =>
+            return RunAction(new ListBrokenFilesResults(), (result, backendManager) =>
             {
-                new Operation.ListBrokenFilesHandler(m_backend, m_options, result).Run(filter, callbackhandler);
+                new Operation.ListBrokenFilesHandler(m_options, result).Run(backendManager, filter, callbackhandler);
             });
         }
 
         public Library.Interface.IPurgeBrokenFilesResults PurgeBrokenFiles(Library.Utility.IFilter filter)
         {
-            return RunAction(new PurgeBrokenFilesResults(), result =>
+            return RunAction(new PurgeBrokenFilesResults(), (result, backendManager) =>
             {
-                new Operation.PurgeBrokenFilesHandler(m_backend, m_options, result).Run(filter);
+                new Operation.PurgeBrokenFilesHandler(m_options, result).Run(backendManager, filter);
             });
         }
 
@@ -386,7 +386,7 @@ namespace Duplicati.Library.Main
             var filtertag = Logging.Log.LogTagFromType<Modules.Builtin.SendMail>();
             using (Logging.Log.StartScope(m_messageSink.WriteMessage, x => x.FilterTag.Contains(filtertag)))
             {
-                return RunAction(new SendMailResults(), result =>
+                return RunAction(new SendMailResults(), (result, backendManager) =>
                 {
                     result.Lines = new string[0];
                     System.Threading.Thread.Sleep(5);
@@ -396,36 +396,36 @@ namespace Duplicati.Library.Main
 
         public Library.Interface.IVacuumResults Vacuum()
         {
-            return RunAction(new VacuumResults(), result =>
+            return RunAction(new VacuumResults(), (result, backendManager) =>
             {
                 new Operation.VacuumHandler(m_options, result).Run();
             });
         }
 
-        private T RunAction<T>(T result, Action<T> method)
-            where T : ISetCommonOptions, ITaskControlProvider, Logging.ILogDestination, IBasicResults
+        private T RunAction<T>(T result, Action<T, IBackendManager> method)
+            where T : ISetCommonOptions, ITaskControlProvider, Logging.ILogDestination, IBasicResults, IBackendWriterProvider
         {
             var tmp = new string[0];
             IFilter tempfilter = null;
             return RunAction<T>(result, ref tmp, ref tempfilter, method);
         }
 
-        private T RunAction<T>(T result, ref string[] paths, Action<T> method)
-            where T : ISetCommonOptions, ITaskControlProvider, Logging.ILogDestination, IBasicResults
+        private T RunAction<T>(T result, ref string[] paths, Action<T, IBackendManager> method)
+            where T : ISetCommonOptions, ITaskControlProvider, Logging.ILogDestination, IBasicResults, IBackendWriterProvider
         {
             IFilter tempfilter = null;
             return RunAction<T>(result, ref paths, ref tempfilter, method);
         }
 
-        private T RunAction<T>(T result, ref IFilter filter, Action<T> method)
-            where T : ISetCommonOptions, ITaskControlProvider, Logging.ILogDestination, IBasicResults
+        private T RunAction<T>(T result, ref IFilter filter, Action<T, IBackendManager> method)
+            where T : ISetCommonOptions, ITaskControlProvider, Logging.ILogDestination, IBasicResults, IBackendWriterProvider
         {
             var tmp = new string[0];
             return RunAction<T>(result, ref tmp, ref filter, method);
         }
 
-        private T RunAction<T>(T result, ref string[] paths, ref IFilter filter, Action<T> method)
-            where T : ISetCommonOptions, ITaskControlProvider, Logging.ILogDestination, IBasicResults
+        private T RunAction<T>(T result, ref string[] paths, ref IFilter filter, Action<T, IBackendManager> method)
+            where T : ISetCommonOptions, ITaskControlProvider, Logging.ILogDestination, IBasicResults, IBackendWriterProvider
         {
             OnOperationStarted?.Invoke(result);
             var resultSetter = result as ISetCommonOptions;
@@ -447,7 +447,25 @@ namespace Duplicati.Library.Main
                     using (new Logging.Timer(LOGTAG, string.Format("Run{0}", result.MainOperation), string.Format("Running {0}", result.MainOperation)))
                     using (new CoCoL.IsolatedChannelScope())
                     using (m_options.ConcurrencyMaxThreads <= 0 ? null : new CoCoL.CappedThreadedThreadPool(m_options.ConcurrencyMaxThreads))
-                        method(result);
+                    using (var backend = new Backend.BackendManager(m_backendUrl, m_options, result.BackendWriter, result.TaskControl))
+                    {
+                        method(result, backend);
+
+                        // TODO: Should also have a single shared database connection for all operations
+                        // The transactions should be managed inside the connection, and not passed around
+
+                        // This would allow us to pass the database instance to the backend manager
+                        // And safeguard against remote operations not being logged in the database
+                        if (File.Exists(m_options.Dbpath))
+                        {
+                            using (var db = new LocalDatabase(m_options.Dbpath, result.MainOperation.ToString(), true))
+                                backend.StopRunnerAndFlushMessages(db, null).Await();
+                        }
+                        else
+                        {
+                            backend.StopRunnerAndDiscardMessages();
+                        }
+                    }
 
                     if (resultSetter.EndTime.Ticks == 0)
                         resultSetter.EndTime = DateTime.UtcNow;
@@ -621,7 +639,7 @@ namespace Duplicati.Library.Main
 
             // Store the URL connection options separately, as these should only be visible to modules implementing IConnectionModule
             var conopts = new Dictionary<string, string>(m_options.RawOptions);
-            var qp = new Library.Utility.Uri(m_backend).QueryParameters;
+            var qp = new Library.Utility.Uri(m_backendUrl).QueryParameters;
             foreach (var k in qp.Keys)
                 conopts[(string)k] = qp[(string)k];
 
@@ -657,7 +675,7 @@ namespace Duplicati.Library.Main
                     }
 
                     if (mx.Value is IGenericCallbackModule module)
-                        module.OnStart(result.MainOperation.ToString(), ref m_backend, ref paths);
+                        module.OnStart(result.MainOperation.ToString(), ref m_backendUrl, ref paths);
                 }
 
             // If the filters were changed by a module, read them back in
@@ -700,17 +718,17 @@ namespace Duplicati.Library.Main
             }
 
             if (string.IsNullOrEmpty(m_options.Dbpath))
-                m_options.Dbpath = CLIDatabaseLocator.GetDatabasePathForCLI(m_backend, m_options);
+                m_options.Dbpath = CLIDatabaseLocator.GetDatabasePathForCLI(m_backendUrl, m_options);
 
             ValidateOptions();
         }
 
         private async Task ApplySecretProvider(CancellationToken cancellationToken)
         {
-            var args = new[] { new Library.Utility.Uri(m_backend) };
+            var args = new[] { new Library.Utility.Uri(m_backendUrl) };
             await SecretProviderHelper.ApplySecretProviderAsync([], args, m_options.RawOptions, TempFolder.SystemTempPath, SecretProvider, cancellationToken);
             // Write back the backend argument, if it was modified by the secret provider
-            m_backend = args[0].ToString();
+            m_backendUrl = args[0].ToString();
         }
 
         /// <summary>
@@ -794,14 +812,14 @@ namespace Duplicati.Library.Main
 
             // Throw url-encoded options into the mix
             //TODO: This can hide values if both commandline and url-parameters supply the same key
-            var ext = new Library.Utility.Uri(m_backend).QueryParameters;
+            var ext = new Library.Utility.Uri(m_backendUrl).QueryParameters;
             foreach (var k in ext.AllKeys)
                 ropts[k] = ext[k];
 
             //Now run through all supported options, and look for deprecated options
             foreach (var l in new IEnumerable<ICommandLineArgument>[] {
                 m_options.SupportedCommands,
-                DynamicLoader.BackendLoader.GetSupportedCommands(m_backend),
+                DynamicLoader.BackendLoader.GetSupportedCommands(m_backendUrl),
                 m_options.NoEncryption ? null : DynamicLoader.EncryptionLoader.GetSupportedCommands(m_options.EncryptionModule),
                 moduleOptions,
                 DynamicLoader.CompressionLoader.GetSupportedCommands(m_options.CompressionModule) })
@@ -865,7 +883,7 @@ namespace Duplicati.Library.Main
             }
 
             //Inform the user about the deprecated Tardigrade-Backend. They should switch to Storj DCS instead.
-            if (string.Equals(new Library.Utility.Uri(m_backend).Scheme, "tardigrade", StringComparison.OrdinalIgnoreCase))
+            if (string.Equals(new Library.Utility.Uri(m_backendUrl).Scheme, "tardigrade", StringComparison.OrdinalIgnoreCase))
                 Logging.Log.WriteWarningMessage(LOGTAG, "TardigradeRename", null, "The Tardigrade-backend got renamed to Storj DCS - please migrate your backups to the new configuration by changing the destination storage type to Storj DCS.");
 
             //TODO: Based on the action, see if all options are relevant

Duplicati/Library/Main/Database/IRemoteVolume.cs

@@ -18,17 +18,12 @@
 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
 // DEALINGS IN THE SOFTWARE.
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-
-namespace Duplicati.Library.Main.Database
-{
-    public interface IRemoteVolume
-    {
-        string Name { get; }
-        string Hash { get; }
-        long Size { get; }
-    }
-}
+namespace Duplicati.Library.Main.Database
+{
+    public interface IRemoteVolume
+    {
+        string Name { get; }
+        string Hash { get; }
+        long Size { get; }
+    }
+}

Duplicati/Library/Main/Database/LocalDatabase.cs

@@ -1467,14 +1467,20 @@ ORDER BY
             {
                 if (m_connection.State == System.Data.ConnectionState.Open && !m_hasExecutedVacuum)
                 {
-                    using (IDbTransaction transaction = m_connection.BeginTransaction())
+                    using (var transaction = m_connection.BeginTransaction())
+                    using (var command = m_connection.CreateCommand(transaction))
                     {
-                        using (IDbCommand command = m_connection.CreateCommand(transaction))
+                        // SQLite recommends that PRAGMA optimize is run just before closing each database connection.
+                        command.ExecuteNonQuery("PRAGMA optimize");
+
+                        try
                         {
-                            // SQLite recommends that PRAGMA optimize is run just before closing each database connection.
-                            command.ExecuteNonQuery("PRAGMA optimize");
                             transaction.Commit();
                         }
+                        catch (System.Data.SQLite.SQLiteException ex)
+                        {
+                            Logging.Log.WriteVerboseMessage(LOGTAG, "FailedToCommitTransaction", ex, "Failed to commit transaction after pragma optimize, usually caused by the a no-op transaction");
+                        }
                     }
 
                     m_connection.Close();

Duplicati/Library/Main/Database/LocalRestoreDatabase.cs

@@ -1,22 +1,22 @@
-// Copyright (C) 2025, The Duplicati Team
-// https://duplicati.com, hello@duplicati.com
-// 
-// Permission is hereby granted, free of charge, to any person obtaining a 
-// copy of this software and associated documentation files (the "Software"), 
-// to deal in the Software without restriction, including without limitation 
-// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
-// and/or sell copies of the Software, and to permit persons to whom the 
-// Software is furnished to do so, subject to the following conditions:
-// 
-// The above copyright notice and this permission notice shall be included in 
-// all copies or substantial portions of the Software.
-// 
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+// Copyright (C) 2025, The Duplicati Team
+// https://duplicati.com, hello@duplicati.com
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a 
+// copy of this software and associated documentation files (the "Software"), 
+// to deal in the Software without restriction, including without limitation 
+// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+// and/or sell copies of the Software, and to permit persons to whom the 
+// Software is furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in 
+// all copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
 // DEALINGS IN THE SOFTWARE.
 using System;
 using System.Collections.Generic;
@@ -198,7 +198,7 @@ namespace Duplicati.Library.Main.Database
                 {
                     m_fileprogtable = null;
                     m_totalprogtable = null;
-					Logging.Log.WriteWarningMessage(LOGTAG, "ProgressTrackerSetupError", ex, "Failed to set up progress tracking tables");
+                    Logging.Log.WriteWarningMessage(LOGTAG, "ProgressTrackerSetupError", ex, "Failed to set up progress tracking tables");
                     throw;
                 }
             }
@@ -211,10 +211,10 @@ namespace Duplicati.Library.Main.Database
             m_tempfiletable = "Fileset-" + m_temptabsetguid;
             m_tempblocktable = "Blocks-" + m_temptabsetguid;
 
-            using(var cmd = m_connection.CreateCommand())
+            using (var cmd = m_connection.CreateCommand())
             {
                 var filesetIds = GetFilesetIDs(Library.Utility.Utility.NormalizeDateTime(restoretime), versions).ToList();
-                while(filesetIds.Count > 0)
+                while (filesetIds.Count > 0)
                 {
                     var filesetId = filesetIds[0];
                     filesetIds.RemoveAt(0);
@@ -249,7 +249,7 @@ namespace Duplicati.Library.Main.Database
                         // If we get a list of filenames, the lookup table is faster
                         // unfortunately we cannot do this if the filesystem is case sensitive as
                         // SQLite only supports ASCII compares
-                        using(var tr = m_connection.BeginTransaction())
+                        using (var tr = m_connection.BeginTransaction())
                         {
                             var p = expression.GetSimpleList();
                             var m_filenamestable = "Filenames-" + m_temptabsetguid;
@@ -258,7 +258,7 @@ namespace Duplicati.Library.Main.Database
                             cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"") VALUES (?)", m_filenamestable);
                             cmd.AddParameter();
 
-                            foreach(var s in p)
+                            foreach (var s in p)
                             {
                                 cmd.SetParameterValue(0, s);
                                 cmd.ExecuteNonQuery();
@@ -275,7 +275,7 @@ namespace Duplicati.Library.Main.Database
                                 var sb = new StringBuilder();
                                 sb.AppendLine();
 
-                                using(var rd = cmd.ExecuteReader(string.Format(@"SELECT ""Path"" FROM ""{0}"" WHERE ""Path"" NOT IN (SELECT ""Path"" FROM ""{1}"")", m_filenamestable, m_tempfiletable)))
+                                using (var rd = cmd.ExecuteReader(string.Format(@"SELECT ""Path"" FROM ""{0}"" WHERE ""Path"" NOT IN (SELECT ""Path"" FROM ""{1}"")", m_filenamestable, m_tempfiletable)))
                                     while (rd.Read())
                                         sb.AppendLine(rd.GetValue(0).ToString());
 
@@ -286,7 +286,7 @@ namespace Duplicati.Library.Main.Database
 
                             cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", m_filenamestable));
 
-                            using(new Logging.Timer(LOGTAG, "CommitPrepareFileset", "CommitPrepareFileset"))
+                            using (new Logging.Timer(LOGTAG, "CommitPrepareFileset", "CommitPrepareFileset"))
                                 tr.Commit();
                         }
                     }
@@ -298,14 +298,14 @@ namespace Duplicati.Library.Main.Database
                         cmd.AddParameter(filesetId);
 
                         object[] values = new object[3];
-                        using(var cmd2 = m_connection.CreateCommand())
+                        using (var cmd2 = m_connection.CreateCommand())
                         {
                             cmd2.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") VALUES (?,?,?,0)", m_tempfiletable);
                             cmd2.AddParameter();
                             cmd2.AddParameter();
                             cmd2.AddParameter();
 
-                            using(var rd = cmd.ExecuteReader())
+                            using (var rd = cmd.ExecuteReader())
                                 while (rd.Read())
                                 {
                                     rd.GetValues(values);
@@ -407,7 +407,7 @@ namespace Duplicati.Library.Main.Database
         {
             var dirsep = Util.GuessDirSeparator(string.IsNullOrWhiteSpace(largest_prefix) ? GetFirstPath() : largest_prefix);
 
-            using(var cmd = m_connection.CreateCommand())
+            using (var cmd = m_connection.CreateCommand())
             {
                 if (string.IsNullOrEmpty(destination))
                 {
@@ -474,7 +474,7 @@ namespace Duplicati.Library.Main.Database
 
         public void FindMissingBlocks(bool skipMetadata)
         {
-            using(var cmd = m_connection.CreateCommand())
+            using (var cmd = m_connection.CreateCommand())
             {
                 cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""FileID"", ""Index"", ""Hash"", ""Size"", ""Restored"", ""Metadata"", ""VolumeId"", ""BlockId"") SELECT DISTINCT ""{1}"".""ID"", ""BlocksetEntry"".""Index"", ""Block"".""Hash"", ""Block"".""Size"", 0, 0, ""Block"".""VolumeID"", ""Block"".""ID"" FROM ""{1}"", ""BlocksetEntry"", ""Block"" WHERE ""{1}"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" ", m_tempblocktable, m_tempfiletable);
                 var p1 = cmd.ExecuteNonQuery();
@@ -576,7 +576,7 @@ namespace Duplicati.Library.Main.Database
 
             public string TargetPath { get { return m_reader.ConvertValueToString(0); } }
             public string TargetHash { get { return m_reader.ConvertValueToString(1); } }
-            public long TargetFileID { get { return  m_reader.ConvertValueToInt64(2); } }
+            public long TargetFileID { get { return m_reader.ConvertValueToInt64(2); } }
             public long Length { get { return m_reader.ConvertValueToInt64(3); } }
 
             public bool HasMore { get; private set; }
@@ -607,14 +607,14 @@ namespace Duplicati.Library.Main.Database
 
             public static IEnumerable<IExistingFile> GetExistingFilesWithBlocks(System.Data.IDbConnection connection, string tablename)
             {
-                using(var cmd = connection.CreateCommand())
+                using (var cmd = connection.CreateCommand())
                 {
                     cmd.CommandText = string.Format(@"SELECT ""{0}"".""TargetPath"", ""Blockset"".""FullHash"", ""{0}"".""ID"", ""Blockset"".""Length"", ""Block"".""Hash"", ""BlocksetEntry"".""Index"", ""Block"".""Size"" FROM ""{0}"", ""Blockset"", ""BlocksetEntry"", ""Block"" WHERE ""{0}"".""BlocksetID"" = ""Blockset"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""{0}"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" ORDER BY ""{0}"".""TargetPath"", ""BlocksetEntry"".""Index""", tablename);
-                    using(var rd = cmd.ExecuteReader())
+                    using (var rd = cmd.ExecuteReader())
                         if (rd.Read())
                         {
                             var more = true;
-                            while(more)
+                            while (more)
                             {
                                 var f = new ExistingFile(rd);
                                 string current = f.TargetPath;
@@ -712,12 +712,12 @@ namespace Duplicati.Library.Main.Database
                 {
                     // TODO: Skip metadata as required
                     cmd.CommandText = string.Format(@"SELECT DISTINCT ""A"".""TargetPath"", ""A"".""ID"", ""B"".""Hash"", (""B"".""Index"" * {2}), ""B"".""Index"", ""B"".""Size"", ""C"".""Path"", (""D"".""Index"" * {2}), ""E"".""Size"", ""B"".""Metadata"" FROM ""{0}"" ""A"", ""{1}"" ""B"", ""File"" ""C"", ""BlocksetEntry"" ""D"", ""Block"" E WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""C"".""BlocksetID"" = ""D"".""BlocksetID"" AND ""D"".""BlockID"" = ""E"".""ID"" AND ""B"".""Hash"" = ""E"".""Hash"" AND ""B"".""Size"" = ""E"".""Size"" AND ""B"".""Restored"" = 0", filetablename, blocktablename, blocksize);
-                    using(var rd = cmd.ExecuteReader())
+                    using (var rd = cmd.ExecuteReader())
                     {
                         if (rd.Read())
                         {
                             var more = true;
-                            while(more)
+                            while (more)
                             {
                                 var f = new LocalBlockSource(rd);
                                 string current = f.TargetPath;
@@ -869,7 +869,7 @@ namespace Duplicati.Library.Main.Database
             {
                 get
                 {
-                    using(var cmd = m_connection.CreateCommand())
+                    using (var cmd = m_connection.CreateCommand())
                     {
                         // The IN-clause with subquery enables SQLite to use indexes better. Three way join (A,B,C) is slow here!
                         cmd.CommandText = string.Format(
@@ -879,12 +879,12 @@ namespace Duplicati.Library.Main.Database
                             + @"   AND ""BB"".""ID"" IN  (SELECT ""B"".""ID"" FROM ""{1}"" ""B"", ""{2}"" ""C"" WHERE ""B"".""Hash"" = ""C"".""Hash"" AND ""B"".""Size"" = ""C"".""Size"") "
                             + @" ORDER BY ""A"".""TargetPath"", ""BB"".""Index"""
                             , m_filetablename, m_blocktablename, m_tmptable, m_blocksize, "0");
-                        using(var rd = cmd.ExecuteReader())
+                        using (var rd = cmd.ExecuteReader())
                         {
                             if (rd.Read())
                             {
                                 var more = true;
-                                while(more)
+                                while (more)
                                 {
                                     var f = new VolumePatch(rd);
                                     string current = f.Path;
@@ -904,7 +904,7 @@ namespace Duplicati.Library.Main.Database
             {
                 get
                 {
-                    using(var cmd = m_connection.CreateCommand())
+                    using (var cmd = m_connection.CreateCommand())
                     {
                         // The IN-clause with subquery enables SQLite to use indexes better. Three way join (A,B,C) is slow here!
                         cmd.CommandText = string.Format(
@@ -914,7 +914,7 @@ namespace Duplicati.Library.Main.Database
                             + @"   AND ""BB"".""ID"" IN  (SELECT ""B"".""ID"" FROM ""{1}"" ""B"", ""{2}"" ""C"" WHERE ""B"".""Hash"" = ""C"".""Hash"" AND ""B"".""Size"" = ""C"".""Size"") "
                             + @" ORDER BY ""A"".""TargetPath"", ""BB"".""Index"""
                             , m_filetablename, m_blocktablename, m_tmptable, m_blocksize, "1");
-                        using(var rd = cmd.ExecuteReader())
+                        using (var rd = cmd.ExecuteReader())
                         {
                             if (rd.Read())
                             {
@@ -1029,7 +1029,7 @@ namespace Duplicati.Library.Main.Database
             using var reader = cmd.ExecuteReader();
             for (long i = 0; reader.Read(); i++)
             {
-                yield return new BlockRequest(reader.GetInt64(0), i, reader.GetString(1), reader.GetInt64(2), reader.GetInt64(3), false);
+                yield return new BlockRequest(reader.ConvertValueToInt64(0), i, reader.ConvertValueToString(1), reader.ConvertValueToInt64(2), reader.ConvertValueToInt64(3), false);
             }
         }
 
@@ -1054,7 +1054,7 @@ namespace Duplicati.Library.Main.Database
             using var reader = cmd.ExecuteReader();
             for (long i = 0; reader.Read(); i++)
             {
-                yield return new BlockRequest(reader.GetInt64(0), i, reader.GetString(1), reader.GetInt64(2), reader.GetInt64(3), false);
+                yield return new BlockRequest(reader.ConvertValueToInt64(0), i, reader.ConvertValueToString(1), reader.ConvertValueToInt64(2), reader.ConvertValueToInt64(3), false);
             }
         }
 
@@ -1063,7 +1063,7 @@ namespace Duplicati.Library.Main.Database
         /// </summary>
         /// <param name="VolumeID">The ID of the volume.</param>
         /// <returns>A tuple containing the name, size, and hash of the volume.</returns>
-        public IEnumerable<(string,long,string)> GetVolumeInfo(long VolumeID)
+        public IEnumerable<(string, long, string)> GetVolumeInfo(long VolumeID)
         {
             using var cmd = m_connection.CreateCommand();
             cmd.CommandText = "SELECT Name, Size, Hash FROM RemoteVolume WHERE ID = ?";
@@ -1071,7 +1071,7 @@ namespace Duplicati.Library.Main.Database
             cmd.SetParameterValue(0, VolumeID);
             using var reader = cmd.ExecuteReader();
             while (reader.Read())
-                yield return (reader.GetString(0), reader.GetInt64(1), reader.GetString(2));
+                yield return (reader.ConvertValueToString(0), reader.ConvertValueToInt64(1), reader.ConvertValueToString(2));
         }
 
         public void DropRestoreTable()
@@ -1084,7 +1084,7 @@ namespace Duplicati.Library.Main.Database
                         cmd.CommandText = string.Format(@"DROP TABLE IF EXISTS ""{0}""", m_tempfiletable);
                         cmd.ExecuteNonQuery();
                     }
-                    catch(Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); }
+                    catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); }
                     finally { m_tempfiletable = null; }
 
                 if (m_tempblocktable != null)
@@ -1328,8 +1328,8 @@ namespace Duplicati.Library.Main.Database
         {
             using (var cmd = m_connection.CreateCommand())
             using (var rd = cmd.ExecuteReader(string.Format(@"SELECT ""TargetPath"" FROM ""{0}"" WHERE ""BlocksetID"" == ?", m_tempfiletable), FOLDER_BLOCKSET_ID))
-                while(rd.Read())
-                    yield return rd.GetValue(0).ToString();
+                while (rd.Read())
+                    yield return rd.ConvertValueToString(0);
         }
 
         public interface IFastSource
@@ -1355,19 +1355,19 @@ namespace Duplicati.Library.Main.Database
                 private readonly System.Data.IDataReader m_rd;
                 private readonly long m_blocksize;
                 public BlockEntry(System.Data.IDataReader rd, long blocksize) { m_rd = rd; m_blocksize = blocksize; }
-                public long Offset { get { return m_rd.GetInt64(3) * m_blocksize; } }
-                public long Index { get { return m_rd.GetInt64(3); } }
-                public long Size { get { return m_rd.GetInt64(5); } }
-                public string Hash { get { return m_rd.GetString(4); } }
+                public long Offset { get { return m_rd.ConvertValueToInt64(3) * m_blocksize; } }
+                public long Index { get { return m_rd.ConvertValueToInt64(3); } }
+                public long Size { get { return m_rd.ConvertValueToInt64(5); } }
+                public string Hash { get { return m_rd.ConvertValueToString(4); } }
             }
 
             private readonly System.Data.IDataReader m_rd;
             private readonly long m_blocksize;
             public FastSource(System.Data.IDataReader rd, long blocksize) { m_rd = rd; m_blocksize = blocksize; MoreData = true; }
             public bool MoreData { get; private set; }
-            public string TargetPath { get { return m_rd.GetValue(0).ToString(); } }
-            public long TargetFileID { get { return m_rd.GetInt64(2); } }
-            public string SourcePath { get { return m_rd.GetValue(1).ToString(); } }
+            public string TargetPath { get { return m_rd.ConvertValueToString(0); } }
+            public long TargetFileID { get { return m_rd.ConvertValueToInt64(2); } }
+            public string SourcePath { get { return m_rd.ConvertValueToString(1); } }
 
             public IEnumerable<IBlockEntry> Blocks
             {
@@ -1378,7 +1378,7 @@ namespace Duplicati.Library.Main.Database
                     do
                     {
                         yield return new BlockEntry(m_rd, m_blocksize);
-                    } while((MoreData = m_rd.Read()) && tid == this.TargetFileID);
+                    } while ((MoreData = m_rd.Read()) && tid == this.TargetFileID);
 
                 }
             }
@@ -1483,7 +1483,7 @@ namespace Duplicati.Library.Main.Database
                 @"  ORDER BY ""{0}"".""ID"", ""{1}"".""Index"" ", m_tempfiletable, m_tempblocktable);
 
             using (var cmd = m_connection.CreateCommand())
-            using(var rd = cmd.ExecuteReader(sources))
+            using (var rd = cmd.ExecuteReader(sources))
             {
                 if (rd.Read())
                 {
@@ -1495,7 +1495,7 @@ namespace Duplicati.Library.Main.Database
                         yield return n;
 
                         more = n.MoreData;
-                        while(more && n.TargetFileID == tid)
+                        while (more && n.TargetFileID == tid)
                             more = rd.Read();
 
                     } while (more);

Duplicati/Library/Main/Duplicati.Library.Main.csproj

@@ -4,6 +4,7 @@
     <TargetFramework>net8.0</TargetFramework>
     <OutputType>Library</OutputType>
     <Copyright>Copyright © 2025 Team Duplicati, MIT license</Copyright>
+    <RootNamespace>Duplicati.Library.Main</RootNamespace>
   </PropertyGroup>
 
   <ItemGroup>

Duplicati/Library/Main/Enums.cs

@@ -1,27 +1,23 @@
-// Copyright (C) 2025, The Duplicati Team
-// https://duplicati.com, hello@duplicati.com
-// 
-// Permission is hereby granted, free of charge, to any person obtaining a 
-// copy of this software and associated documentation files (the "Software"), 
-// to deal in the Software without restriction, including without limitation 
-// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
-// and/or sell copies of the Software, and to permit persons to whom the 
-// Software is furnished to do so, subject to the following conditions:
-// 
-// The above copyright notice and this permission notice shall be included in 
-// all copies or substantial portions of the Software.
-// 
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
-// DEALINGS IN THE SOFTWARE.
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
+// Copyright (C) 2025, The Duplicati Team
+// https://duplicati.com, hello@duplicati.com
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a 
+// copy of this software and associated documentation files (the "Software"), 
+// to deal in the Software without restriction, including without limitation 
+// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+// and/or sell copies of the Software, and to permit persons to whom the 
+// Software is furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in 
+// all copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+// DEALINGS IN THE SOFTWARE.
 
 namespace Duplicati.Library.Main
 {
@@ -32,9 +28,11 @@ namespace Duplicati.Library.Main
         Get,
         Put,
         Delete,
-        CreateFolder
+        CreateFolder,
+        QuotaInfo,
+        WaitForEmpty
     }
-    
+
     public enum BackendEventType
     {
         Started,
@@ -150,5 +148,5 @@ namespace Duplicati.Library.Main
         /// The entry is a symbolic link
         /// </summary>
         Symlink
-    }   
+    }
 }

Duplicati/Library/Main/IBackendManager.cs

@@ -0,0 +1,112 @@
+using System;
+using System.Collections.Generic;
+using System.Data;
+using System.Threading;
+using System.Threading.Tasks;
+using Duplicati.Library.Interface;
+using Duplicati.Library.Main.Database;
+using Duplicati.Library.Main.Volumes;
+using Duplicati.Library.Utility;
+using IFileEntry = Duplicati.Library.Interface.IFileEntry;
+
+#nullable enable
+
+namespace Duplicati.Library.Main;
+
+/// <summary>
+/// Interface for the backend manager
+/// </summary>
+internal interface IBackendManager : IDisposable
+{
+    /// <summary>
+    /// Uploads a block volume to the backend, including an optional index volume
+    /// </summary>
+    /// <param name="blockVolume">The block volume to upload</param>
+    /// <param name="indexVolume">The index volume to upload, if any</param>
+    /// <param name="indexVolumeFinished">The action to call when the index volume is finished</param>
+    /// <param name="waitForComplete">Whether to wait for the upload to complete</param>
+    /// <param name="cancelToken">The cancellation token</param>
+    /// <returns>An awaitable task</returns>
+    Task PutAsync(VolumeWriterBase blockVolume, IndexVolumeWriter? indexVolume, Action? indexVolumeFinished, bool waitForComplete, CancellationToken cancelToken);
+
+    /// <summary>
+    /// Uploads a file to the backend without encryption
+    /// </summary>
+    /// <param name="remotename">The name of the file to upload to</param>
+    /// <param name="tempFile">The file to upload</param>
+    /// <param name="cancelToken">The cancellation token</param>
+    /// <returns>An awaitable task</returns>
+    Task PutVerificationFileAsync(string remotename, TempFile tempFile, CancellationToken cancelToken);
+
+    /// <summary>
+    /// Waits for the backend queue to be empty and flushes any pending messages to the database
+    /// </summary>
+    /// <param name="database">The database to write pending messages to</param>
+    /// <param name="transaction">The transaction to use, if any</param>
+    /// <param name="cancellationToken">The cancellation token</param>
+    /// <returns>An awaitable task</returns>
+    Task WaitForEmptyAsync(LocalDatabase database, IDbTransaction? transaction, CancellationToken cancellationToken);
+
+    /// <summary>
+    /// Lists the files on the backend
+    /// </summary>
+    /// <param name="cancelToken">The cancellation token</param>
+    /// <returns>An enumerable of file entries</returns>
+    Task<IEnumerable<IFileEntry>> ListAsync(CancellationToken cancelToken);
+
+    /// <summary>
+    /// Deletes a file on the backend
+    /// </summary>
+    /// <param name="remotename">The name of the file to delete</param>
+    /// <param name="size">The size of the file to delete, or -1 if not known</param>
+    /// <param name="waitForComplete">Whether to wait for the delete to complete</param>
+    /// <param name="cancelToken">The cancellation token</param>
+    /// <returns>An awaitable task</returns>
+    Task DeleteAsync(string remotename, long size, bool waitForComplete, CancellationToken cancelToken);
+
+    /// <summary>
+    /// Gets the quota information for the backend
+    /// </summary>
+    /// <param name="cancelToken">The cancellation token</param>
+    /// <returns>The quota information, or null if not available or disabled</returns>
+    Task<IQuotaInfo?> GetQuotaInfoAsync(CancellationToken cancelToken);
+
+    /// <summary>
+    /// Gets a file with the hash and size
+    /// </summary>
+    /// <param name="remotename">The name of the file to get</param>
+    /// <param name="hash">The hash of the file to get, or <c>null</c> if not known</param>
+    /// <param name="size">The size of the file to get, or -1 if not known</param>
+    /// <param name="cancelToken">The cancellation token</param>
+    /// <returns>The file, hash, and size</returns>
+    Task<(TempFile File, string Hash, long Size)> GetWithInfoAsync(string remotename, string hash, long size, CancellationToken cancelToken);
+
+    /// <summary>
+    /// Gets a file from the backend
+    /// </summary>
+    /// <param name="remotename">The name of the file to get</param>
+    /// <param name="hash">The hash of the file to get, or <c>null</c> if not known</param>
+    /// <param name="size">The size of the file to get, or -1 if not known</param>
+    /// <param name="cancelToken">The cancellation token</param>
+    /// <returns></returns>
+    Task<TempFile> GetAsync(string remotename, string hash, long size, CancellationToken cancelToken);
+
+    /// <summary>
+    /// Performs a download of the files specified, with pre-fetch to overlap the download and processing
+    /// </summary>
+    /// <param name="volumes">The volumes to download</param>
+    /// <param name="cancelToken">The cancellation token</param>
+    /// <returns>The downloaded files, hash, size, and name</returns>
+    IAsyncEnumerable<(TempFile File, string Hash, long Size, string Name)> GetFilesOverlappedAsync(IEnumerable<IRemoteVolume> volumes, CancellationToken cancelToken);
+
+    /// <summary>
+    /// Gets the size of the last read operation
+    /// </summary>
+    long LastReadSize { get; }
+
+    /// <summary>
+    /// Gets the size of the last write operation
+    /// </summary>
+    long LastWriteSize { get; }
+
+}

Duplicati/Library/Main/Operation/Backup/BackendUploader.cs

@@ -1,551 +0,0 @@
-// Copyright (C) 2025, The Duplicati Team
-// https://duplicati.com, hello@duplicati.com
-// 
-// Permission is hereby granted, free of charge, to any person obtaining a 
-// copy of this software and associated documentation files (the "Software"), 
-// to deal in the Software without restriction, including without limitation 
-// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
-// and/or sell copies of the Software, and to permit persons to whom the 
-// Software is furnished to do so, subject to the following conditions:
-// 
-// The above copyright notice and this permission notice shall be included in 
-// all copies or substantial portions of the Software.
-// 
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
-// DEALINGS IN THE SOFTWARE.
-
-using CoCoL;
-using Duplicati.Library.Interface;
-using Duplicati.Library.Main.Operation.Common;
-using Duplicati.Library.Main.Volumes;
-using Duplicati.Library.Utility;
-using Newtonsoft.Json;
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Runtime.ExceptionServices;
-using System.Threading;
-using System.Threading.Tasks;
-using static Duplicati.Library.Main.Operation.Common.BackendHandler;
-
-namespace Duplicati.Library.Main.Operation.Backup
-{
-    internal interface IUploadRequest
-    {
-    }
-
-    internal class FlushRequest : IUploadRequest
-    {
-        public Task<long> LastWriteSizeAsync { get { return m_tcs.Task; } }
-        private readonly TaskCompletionSource<long> m_tcs = new TaskCompletionSource<long>();
-
-        public void SetFlushed(long size)
-        {
-            m_tcs.TrySetResult(size);
-        }
-
-        public void TrySetCanceled()
-        {
-            m_tcs.TrySetCanceled();
-        }
-    }
-
-    internal class IndexVolumeUploadRequest : IUploadRequest
-    {
-        public IndexVolumeWriter IndexVolume { get; private set; }
-
-        public IndexVolumeUploadRequest(IndexVolumeWriter indexVolume)
-        {
-            IndexVolume = indexVolume;
-        }
-    }
-
-    internal class FilesetUploadRequest : IUploadRequest
-    {
-        public FilesetVolumeWriter Fileset { get; private set; }
-
-        public FilesetUploadRequest(FilesetVolumeWriter fileset)
-        {
-            Fileset = fileset;
-        }
-    }
-
-    internal class VolumeUploadRequest : IUploadRequest
-    {
-        public FileEntryItem BlockEntry { get; }
-        public BlockVolumeWriter BlockVolume { get; }
-        public TemporaryIndexVolume IndexVolume { get; }
-        public Options Options { get; }
-        public BackupDatabase Database { get; }
-
-        public VolumeUploadRequest(BlockVolumeWriter blockVolume, FileEntryItem blockEntry, TemporaryIndexVolume indexVolume, Options options, BackupDatabase database)
-        {
-            BlockVolume = blockVolume;
-            BlockEntry = blockEntry;
-            IndexVolume = indexVolume;
-            Options = options;
-            Database = database;
-        }
-    }
-
-    /// <summary>
-    /// This class encapsulates all requests to the backend
-    /// and ensures that the <code>AsynchronousUploadLimit</code> is honored
-    /// </summary>
-    internal class BackendUploader
-    {
-        private static readonly string LOGTAG = Logging.Log.LogTagFromType<BackendUploader>();
-        private readonly Func<IBackend> m_backendFactory;
-        private readonly IBackend m_backend;
-        private int m_backendTaken = 0;
-        private bool m_backendDisposed = false;
-        private CancellationTokenSource m_cancelTokenSource;
-        private readonly DatabaseCommon m_database;
-        private long m_initialUploadThrottleSpeed;
-        private int m_lastThrottleCheckTime;
-        private long m_lastUploadThrottleSpeed;
-        private int m_maxConcurrentUploads;
-        private readonly Options m_options;
-        private readonly FileProgressThrottler m_progressUpdater;
-        private readonly StatsCollector m_stats;
-        private readonly ITaskReader m_taskReader;
-
-        public IBackend Backend { get { return m_backendDisposed ? null : m_backend; } }
-
-        public BackendUploader(IBackend backend, Func<IBackend> backendFactory, Options options, DatabaseCommon database, ITaskReader taskReader, StatsCollector stats)
-        {
-            m_backend = backend;
-            m_backendFactory = () =>
-            {
-                if (Interlocked.Exchange(ref m_backendTaken, 1) == 0)
-                    return m_backend;
-
-                return backendFactory();
-            };
-            m_options = options;
-            m_taskReader = taskReader;
-            m_stats = stats;
-            m_database = database;
-            m_progressUpdater = new FileProgressThrottler(stats, options.MaxUploadPrSecond);
-        }
-
-        public Task Run(Channels channels)
-        {
-            return AutomationExtensions.RunTask(new
-            {
-                Input = channels.BackendRequest.AsRead()
-            },
-
-            async self =>
-            {
-                var workers = new List<Worker>();
-                m_maxConcurrentUploads = m_options.AsynchronousConcurrentUploadLimit <= 0 ? int.MaxValue : m_options.AsynchronousConcurrentUploadLimit;
-                m_initialUploadThrottleSpeed = m_options.AsynchronousConcurrentUploadLimit <= 0 ? int.MaxValue : m_options.MaxUploadPrSecond / m_maxConcurrentUploads;
-                var lastSize = -1L;
-                var uploadsInProgress = 0;
-                m_cancelTokenSource = new CancellationTokenSource();
-                m_progressUpdater.Run(m_cancelTokenSource.Token);
-                var filesetsToUpload = new List<FilesetUploadRequest>();
-
-                try
-                {
-                    while (true)
-                    {
-                        var req = await self.Input.ReadAsync();
-
-                        // Listen to pause/resume and abort, but ignore stop requests
-                        await m_taskReader.ProgressRendevouz().ConfigureAwait(false);
-
-                        Worker worker = GetWorker(workers);
-
-                        if (req is VolumeUploadRequest volumeUpload)
-                        {
-                            if (volumeUpload.IndexVolume == null)
-                                worker.Task = Task.Run(() => UploadFileAsync(volumeUpload.BlockEntry, worker, m_cancelTokenSource.Token));
-                            else
-                                worker.Task = Task.Run(() => UploadBlockAndIndexAsync(volumeUpload, worker, m_cancelTokenSource.Token));
-
-                            lastSize = volumeUpload.BlockVolume.SourceSize;
-                            uploadsInProgress++;
-                        }
-                        else if (req is FilesetUploadRequest filesetUpload)
-                        {
-                            filesetsToUpload.Add(filesetUpload);
-                        }
-                        else if (req is IndexVolumeUploadRequest indexUpload)
-                        {
-                            worker.Task = Task.Run(() => UploadVolumeWriter(indexUpload.IndexVolume, worker, m_cancelTokenSource.Token));
-                            uploadsInProgress++;
-                        }
-                        else if (req is FlushRequest flush)
-                        {
-                            await WaitForWorkers(workers, flush).ConfigureAwait(false);
-                            UploadFilesets(workers, filesetsToUpload);
-                            await WaitForWorkers(workers, flush).ConfigureAwait(false);
-
-                            flush.SetFlushed(lastSize);
-                            uploadsInProgress = 0;
-                            break;
-                        }
-
-                        if (uploadsInProgress >= m_maxConcurrentUploads)
-                        {
-                            await Task.WhenAny(workers.Select(w => w.Task)).ConfigureAwait(false);
-                            uploadsInProgress--;
-
-                            var failedUploads = workers.Where(w => w.Task.IsFaulted).Select(w => GetInnerMostException(w.Task.Exception)).ToList();
-                            if (failedUploads.Any())
-                            {
-                                if (failedUploads.Count == 1)
-                                    ExceptionDispatchInfo.Capture(failedUploads.First()).Throw();
-                                else
-                                    throw new AggregateException(failedUploads);
-                            }
-                        }
-                    }
-                }
-                catch (Exception ex) when (!ex.IsRetiredException())
-                {
-                    m_cancelTokenSource.Cancel();
-                    try
-                    {
-                        await Task.WhenAll(workers.Select(w => w.Task));
-                    }
-                    catch
-                    {
-                        // As we are cancelling all threads we do not need to alert the user to any of these exceptions.
-                    }
-                    finally
-                    {
-                        workers.ForEach(w =>
-                        {
-                            if (w.Backend == m_backend)
-                                m_backendDisposed = true;
-                            w.Dispose();
-                        });
-                    }
-                    throw;
-                }
-
-                try
-                {
-                    m_stats.SetBlocking(true);
-                    await Task.WhenAll(workers.Select(w => w.Task));
-                }
-                finally
-                {
-                    m_stats.SetBlocking(false);
-                    workers.ForEach(w =>
-                    {
-                        if (w.Backend == m_backend)
-                        {
-                            Interlocked.Exchange(ref m_backendTaken, 0);
-                            w.Backend = null;
-                        }
-                        w.Dispose();
-                    });
-                }
-            });
-        }
-
-        private Worker GetWorker(List<Worker> workers)
-        {
-            var worker = workers.FirstOrDefault(w => w.Task.IsCompleted && !w.Task.IsFaulted);
-            if (worker == null)
-            {
-                worker = new Worker(m_backendFactory());
-                workers.Add(worker);
-            }
-
-            return worker;
-        }
-
-        private async Task WaitForWorkers(List<Worker> workers, FlushRequest flush)
-        {
-            while (workers.Any())
-            {
-                Task finishedTask = await Task.WhenAny(workers.Select(w => w.Task)).ConfigureAwait(false);
-                if (finishedTask.IsFaulted)
-                {
-                    flush.TrySetCanceled();
-                    ExceptionDispatchInfo.Capture(finishedTask.Exception).Throw();
-                }
-
-                Worker finishedWorker = workers.Single(w => w.Task == finishedTask);
-                workers.Remove(finishedWorker);
-                if (finishedWorker.Backend == m_backend && !m_backendDisposed)
-                {
-                    finishedWorker.Backend = null;
-                    Interlocked.Exchange(ref m_backendTaken, 0);
-                }
-
-                finishedWorker.Dispose();
-            }
-        }
-
-        private void UploadFilesets(List<Worker> workers, List<FilesetUploadRequest> filesets)
-        {
-            foreach (var fileset in filesets)
-            {
-                Worker worker = GetWorker(workers);
-                worker.Task = Task.Run(() => UploadVolumeWriter(fileset.Fileset, worker, m_cancelTokenSource.Token));
-            }
-
-            filesets.Clear();
-        }
-
-        private static Exception GetInnerMostException(Exception ex)
-        {
-            while (ex.InnerException != null)
-                ex = ex.InnerException;
-            return ex;
-        }
-
-        private async Task UploadBlockAndIndexAsync(VolumeUploadRequest upload, Worker worker, CancellationToken cancelToken)
-        {
-            if (await UploadFileAsync(upload.BlockEntry, worker, cancelToken).ConfigureAwait(false))
-            {
-                // We must use the BlockEntry's RemoteFilename and not the BlockVolume's since the
-                // BlockEntry's' RemoteFilename reflects renamed files due to retries after errors.
-                IndexVolumeWriter indexVolumeWriter = await upload.IndexVolume.CreateVolume(upload.BlockEntry.RemoteFilename, upload.BlockEntry.Hash, upload.BlockEntry.Size, upload.Options, upload.Database);
-                // Create link before upload is started, it will be removed later if upload fails
-                await m_database.AddIndexBlockLinkAsync(indexVolumeWriter.VolumeID, upload.BlockVolume.VolumeID).ConfigureAwait(false);
-
-                FileEntryItem indexEntry = indexVolumeWriter.CreateFileEntryForUpload(upload.Options);
-                await UploadFileAsync(indexEntry, worker, cancelToken).ConfigureAwait(false);
-            }
-        }
-
-        private async Task UploadVolumeWriter(VolumeWriterBase volumeWriter, Worker worker, CancellationToken cancelToken)
-        {
-            var fileEntry = new FileEntryItem(BackendActionType.Put, volumeWriter.RemoteFilename);
-            fileEntry.SetLocalfilename(volumeWriter.LocalFilename);
-            fileEntry.Encrypt(m_options);
-            fileEntry.UpdateHashAndSize(m_options);
-
-            await UploadFileAsync(fileEntry, worker, cancelToken).ConfigureAwait(false);
-        }
-
-        private async Task<bool> UploadFileAsync(FileEntryItem item, Worker worker, CancellationToken cancelToken)
-        {
-            if (cancelToken.IsCancellationRequested)
-                return false;
-
-            return await DoWithRetry(async () =>
-            {
-                if (item.IsRetry)
-                    await RenameFileAfterErrorAsync(item).ConfigureAwait(false);
-                await DoPut(item, worker.Backend, cancelToken).ConfigureAwait(false);
-            },
-            item, worker, cancelToken).ConfigureAwait(false);
-        }
-
-        private async Task<bool> DoWithRetry(Func<Task> method, FileEntryItem item, Worker worker, CancellationToken cancelToken)
-        {
-            item.IsRetry = false;
-            var retryCount = 0;
-
-            for (retryCount = 0; retryCount <= m_options.NumberOfRetries; retryCount++)
-            {
-                if (m_options.RetryDelay.Ticks != 0 && retryCount != 0)
-                {
-                    var delay = Library.Utility.Utility.GetRetryDelay(m_options.RetryDelay, retryCount, m_options.RetryWithExponentialBackoff);
-                    await Task.Delay(delay).ConfigureAwait(false);
-                }
-
-                if (cancelToken.IsCancellationRequested)
-                    return false;
-
-                try
-                {
-                    if (worker.Backend == null)
-                        worker.Backend = m_backendFactory();
-                    await method().ConfigureAwait(false);
-                    return true;
-                }
-                catch (Exception ex)
-                {
-                    item.IsRetry = true;
-                    Logging.Log.WriteRetryMessage(LOGTAG, $"Retry{item.Operation}", ex, "Operation {0} with file {1} attempt {2} of {3} failed with message: {4}", item.Operation, item.RemoteFilename, retryCount + 1, m_options.NumberOfRetries + 1, ex.Message);
-                    if (ex is ThreadAbortException || ex is OperationCanceledException)
-                        break;
-
-                    await m_stats.SendEventAsync(item.Operation, retryCount < m_options.NumberOfRetries ? BackendEventType.Retrying : BackendEventType.Failed, item.RemoteFilename, item.Size);
-
-                    bool recovered = false;
-                    if (m_options.AutocreateFolders && ex is FolderMissingException)
-                    {
-                        try
-                        {
-                            // If we successfully create the folder, we can re-use the connection
-                            await worker.Backend.CreateFolderAsync(cancelToken).ConfigureAwait(false);
-                            recovered = true;
-                        }
-                        catch (Exception dex)
-                        {
-                            Logging.Log.WriteWarningMessage(LOGTAG, "FolderCreateError", dex, "Failed to create folder: {0}", ex.Message);
-                        }
-                    }
-
-                    if (!recovered)
-                        ResetBackend(ex, worker);
-
-                    if (retryCount == m_options.NumberOfRetries)
-                        throw;
-                }
-                finally
-                {
-                    if (m_options.NoConnectionReuse)
-                        ResetBackend(null, worker);
-                }
-            }
-
-            return false;
-        }
-
-        private void ResetBackend(Exception ex, Worker worker)
-        {
-            try
-            {
-                if (worker.Backend == m_backend)
-                    m_backendDisposed = true;
-                worker.Backend?.Dispose();
-            }
-            catch (Exception dex)
-            {
-                Logging.Log.WriteWarningMessage(LOGTAG, "BackendDisposeError", dex, "Failed to dispose backend instance: {0}", ex?.Message);
-            }
-            finally
-            {
-                worker.Backend = null;
-            }
-        }
-
-        private async Task RenameFileAfterErrorAsync(FileEntryItem item)
-        {
-            var p = VolumeBase.ParseFilename(item.RemoteFilename);
-            var guid = VolumeWriterBase.GenerateGuid();
-            var time = p.Time.Ticks == 0 ? p.Time : p.Time.AddSeconds(1);
-            var newname = VolumeBase.GenerateFilename(p.FileType, p.Prefix, guid, time, p.CompressionModule, p.EncryptionModule);
-            var oldname = item.RemoteFilename;
-
-            await m_stats.SendEventAsync(item.Operation, BackendEventType.Rename, oldname, item.Size);
-            await m_stats.SendEventAsync(item.Operation, BackendEventType.Rename, newname, item.Size);
-            Logging.Log.WriteInformationMessage(LOGTAG, "RenameRemoteTargetFile", "Renaming \"{0}\" to \"{1}\"", oldname, newname);
-            await m_database.RenameRemoteFileAsync(oldname, newname);
-            item.RemoteFilename = newname;
-        }
-
-        private async Task DoPut(FileEntryItem item, IBackend backend, CancellationToken cancelToken)
-        {
-            if (cancelToken.IsCancellationRequested)
-                return;
-
-            if (item.TrackedInDb)
-                await m_database.UpdateRemoteVolumeAsync(item.RemoteFilename, RemoteVolumeState.Uploading, item.Size, item.Hash);
-
-            if (m_options.Dryrun)
-            {
-                Logging.Log.WriteDryrunMessage(LOGTAG, "WouldUploadVolume", "Would upload volume: {0}, size: {1}", item.RemoteFilename, Library.Utility.Utility.FormatSizeString(new FileInfo(item.LocalFilename).Length));
-                item.DeleteLocalFile();
-                return;
-            }
-
-            await m_database.LogRemoteOperationAsync("put", item.RemoteFilename, JsonConvert.SerializeObject(new { Size = item.Size, Hash = item.Hash }));
-            await m_stats.SendEventAsync(BackendActionType.Put, BackendEventType.Started, item.RemoteFilename, item.Size, updateProgress: false);
-            m_progressUpdater.StartFileProgress(item.RemoteFilename, item.Size);
-
-            var begin = DateTime.Now;
-
-            if (!m_options.DisableStreamingTransfers && backend is IStreamingBackend streamingBackend)
-            {
-                // A download throttle speed is not given to the ThrottledStream as we are only uploading data here
-                using (var fs = File.OpenRead(item.LocalFilename))
-                using (var act = new StreamUtil.TimeoutObservingStream(fs) { ReadTimeout = backend is ITimeoutExemptBackend ? System.Threading.Timeout.Infinite : m_options.ReadWriteTimeout })
-                using (var ts = new ThrottledStream(act, m_initialUploadThrottleSpeed, 0))
-                using (var pgs = new ProgressReportingStream(ts, pg => HandleProgress(ts, pg, item.RemoteFilename)))
-                using (var linkedToken = CancellationTokenSource.CreateLinkedTokenSource(m_taskReader.TransferToken, act.TimeoutToken))
-                    await streamingBackend.PutAsync(item.RemoteFilename, pgs, linkedToken.Token).ConfigureAwait(false);
-            }
-            else
-                await backend.PutAsync(item.RemoteFilename, item.LocalFilename, cancelToken).ConfigureAwait(false);
-
-            var duration = DateTime.Now - begin;
-            m_progressUpdater.EndFileProgress(item.RemoteFilename);
-            Logging.Log.WriteProfilingMessage(LOGTAG, "UploadSpeed", "Uploaded {0} in {1}, {2}/s", Library.Utility.Utility.FormatSizeString(item.Size), duration, Library.Utility.Utility.FormatSizeString((long)(item.Size / duration.TotalSeconds)));
-
-            if (item.TrackedInDb)
-                await m_database.UpdateRemoteVolumeAsync(item.RemoteFilename, RemoteVolumeState.Uploaded, item.Size, item.Hash);
-
-            await m_stats.SendEventAsync(BackendActionType.Put, BackendEventType.Completed, item.RemoteFilename, item.Size);
-
-            if (m_options.ListVerifyUploads)
-            {
-                var f = backend.List().FirstOrDefault(n => n.Name.Equals(item.RemoteFilename, StringComparison.OrdinalIgnoreCase));
-                if (f == null)
-                    throw new Exception(string.Format("List verify failed, file was not found after upload: {0}", item.RemoteFilename));
-                else if (f.Size != item.Size && f.Size >= 0)
-                    throw new Exception(string.Format("List verify failed for file: {0}, size was {1} but expected to be {2}", f.Name, f.Size, item.Size));
-            }
-
-            item.DeleteLocalFile();
-            await m_database.CommitTransactionAsync("CommitAfterUpload");
-        }
-
-        private void HandleProgress(ThrottledStream stream, long progress, string path)
-        {
-            UpdateThrottleSpeed();
-            m_progressUpdater.UpdateFileProgress(path, progress, stream);
-        }
-
-        private void UpdateThrottleSpeed()
-        {
-            var updateThrottleSpeed = false;
-
-            lock (m_progressUpdater)
-            {
-                var currentTick = Environment.TickCount;
-                if (currentTick - m_lastThrottleCheckTime > 2000)
-                {
-                    m_lastThrottleCheckTime = currentTick;
-                    updateThrottleSpeed = true;
-                }
-            }
-
-            if (!updateThrottleSpeed)
-                return;
-
-            var maxUploadPerSecond = m_options.MaxUploadPrSecond;
-            if (maxUploadPerSecond != m_lastUploadThrottleSpeed)
-            {
-                m_lastUploadThrottleSpeed = maxUploadPerSecond;
-                m_initialUploadThrottleSpeed = maxUploadPerSecond / m_maxConcurrentUploads;
-                m_progressUpdater.UpdateThrottleSpeeds(maxUploadPerSecond);
-            }
-        }
-
-        private class Worker : IDisposable
-        {
-            public Task Task;
-            public IBackend Backend;
-
-            public Worker(IBackend backend)
-            {
-                Backend = backend;
-                Task = Task.FromResult(true);
-            }
-
-            public void Dispose()
-            {
-                this.Task?.Dispose();
-                this.Backend?.Dispose();
-            }
-        }
-    }
-}

Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs

@@ -260,6 +260,7 @@ namespace Duplicati.Library.Main.Operation.Backup
             return RunOnMain(() => m_database.UpdateFullBackupStateInFileset(filesetid, true, GetTransaction()));
         }
 
+        // TODO: Make IAsyncEnumerable or get rid of the SingleRunner setup
         public Task<string[]> GetMissingIndexFilesAsync()
         {
             return RunOnMain(() => m_database.GetMissingIndexFiles(GetTransaction()).ToArray());

Duplicati/Library/Main/Operation/Backup/Channels.cs

@@ -1,22 +1,22 @@
-// Copyright (C) 2025, The Duplicati Team
-// https://duplicati.com, hello@duplicati.com
-// 
-// Permission is hereby granted, free of charge, to any person obtaining a 
-// copy of this software and associated documentation files (the "Software"), 
-// to deal in the Software without restriction, including without limitation 
-// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
-// and/or sell copies of the Software, and to permit persons to whom the 
-// Software is furnished to do so, subject to the following conditions:
-// 
-// The above copyright notice and this permission notice shall be included in 
-// all copies or substantial portions of the Software.
-// 
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+// Copyright (C) 2025, The Duplicati Team
+// https://duplicati.com, hello@duplicati.com
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a 
+// copy of this software and associated documentation files (the "Software"), 
+// to deal in the Software without restriction, including without limitation 
+// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+// and/or sell copies of the Software, and to permit persons to whom the 
+// Software is furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in 
+// all copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
 // DEALINGS IN THE SOFTWARE.
 
 using CoCoL;
@@ -28,10 +28,6 @@ namespace Duplicati.Library.Main.Operation.Backup
     /// </summary>
     internal class Channels
     {
-        /// <summary>
-        /// Requests to the backend are send over this channel and picked up by the <see cref="BackendUploader" /> class
-        /// </summary>
-        public readonly IChannel<IUploadRequest> BackendRequest = ChannelManager.CreateChannel<IUploadRequest>();
         /// <summary>
         /// When the backup completes, all in-progress archives are sent from the <see cref="DataBlockProcessor"/> to the <see cref="SpillCollectorProcess"/>
         /// </summary>

Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs

@@ -1,22 +1,22 @@
-// Copyright (C) 2025, The Duplicati Team
-// https://duplicati.com, hello@duplicati.com
-// 
-// Permission is hereby granted, free of charge, to any person obtaining a 
-// copy of this software and associated documentation files (the "Software"), 
-// to deal in the Software without restriction, including without limitation 
-// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
-// and/or sell copies of the Software, and to permit persons to whom the 
-// Software is furnished to do so, subject to the following conditions:
-// 
-// The above copyright notice and this permission notice shall be included in 
-// all copies or substantial portions of the Software.
-// 
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+// Copyright (C) 2025, The Duplicati Team
+// https://duplicati.com, hello@duplicati.com
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a 
+// copy of this software and associated documentation files (the "Software"), 
+// to deal in the Software without restriction, including without limitation 
+// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+// and/or sell copies of the Software, and to permit persons to whom the 
+// Software is furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in 
+// all copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
 // DEALINGS IN THE SOFTWARE.
 
 using CoCoL;
@@ -24,7 +24,6 @@ using Duplicati.Library.Main.Operation.Common;
 using Duplicati.Library.Main.Volumes;
 using System;
 using System.Threading.Tasks;
-using static Duplicati.Library.Main.Operation.Common.BackendHandler;
 
 namespace Duplicati.Library.Main.Operation.Backup
 {
@@ -35,13 +34,12 @@ namespace Duplicati.Library.Main.Operation.Backup
     /// </summary>
     internal static class DataBlockProcessor
     {
-        public static Task Run(Channels channels, BackupDatabase database, Options options, ITaskReader taskreader)
+        public static Task Run(Channels channels, BackupDatabase database, IBackendManager backendManager, Options options, ITaskReader taskreader)
         {
             return AutomationExtensions.RunTask(
             new
             {
                 Input = channels.OutputBlocks.AsRead(),
-                Output = channels.BackendRequest.AsWrite(),
                 SpillPickup = channels.SpillPickup.AsWrite(),
             },
 
@@ -131,22 +129,22 @@ namespace Duplicati.Library.Main.Operation.Backup
 
                                 await database.CommitTransactionAsync("CommitAddBlockToOutputFlush");
 
-                                FileEntryItem blockEntry = blockvolume.CreateFileEntryForUpload(options);
-
-                                TemporaryIndexVolume indexVolumeCopy = null;
+                                IndexVolumeWriter indexVolumeCopy = null;
                                 if (indexvolume != null)
                                 {
-                                    indexVolumeCopy = new TemporaryIndexVolume(options);
-                                    indexvolume.CopyTo(indexVolumeCopy, false);
+                                    // TODO: It is much easier to let the BackendManager deal with index files,
+                                    // but it adds a bit of strain to the database
+                                    indexVolumeCopy = await indexvolume.CreateVolume(blockvolume.RemoteFilename, options, database);
+                                    // Create link before upload is started, it will be removed later if upload fails
+                                    await database.AddIndexBlockLinkAsync(indexVolumeCopy.VolumeID, blockvolume.VolumeID).ConfigureAwait(false);
                                 }
 
-                                var uploadRequest = new VolumeUploadRequest(blockvolume, blockEntry, indexVolumeCopy, options, database);
-
+                                var blockVolumeCopy = blockvolume;
                                 blockvolume = null;
                                 indexvolume = null;
 
-                                // Write to output at the end here to prevent sending a full volume to the SpillCollector
-                                await self.Output.WriteAsync(uploadRequest);
+                                await backendManager.PutAsync(blockVolumeCopy, indexVolumeCopy, null, false, taskreader.ProgressToken);
+
                             }
 
                         }

Duplicati/Library/Main/Operation/Backup/RecreateMissingIndexFiles.cs

@@ -1,66 +1,56 @@
-// Copyright (C) 2025, The Duplicati Team
-// https://duplicati.com, hello@duplicati.com
-// 
-// Permission is hereby granted, free of charge, to any person obtaining a 
-// copy of this software and associated documentation files (the "Software"), 
-// to deal in the Software without restriction, including without limitation 
-// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
-// and/or sell copies of the Software, and to permit persons to whom the 
-// Software is furnished to do so, subject to the following conditions:
-// 
-// The above copyright notice and this permission notice shall be included in 
-// all copies or substantial portions of the Software.
-// 
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+// Copyright (C) 2025, The Duplicati Team
+// https://duplicati.com, hello@duplicati.com
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a 
+// copy of this software and associated documentation files (the "Software"), 
+// to deal in the Software without restriction, including without limitation 
+// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+// and/or sell copies of the Software, and to permit persons to whom the 
+// Software is furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in 
+// all copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
 // DEALINGS IN THE SOFTWARE.
 
 using System.Threading.Tasks;
-using CoCoL;
 using Duplicati.Library.Main.Operation.Common;
 
-namespace Duplicati.Library.Main.Operation.Backup
+namespace Duplicati.Library.Main.Operation.Backup;
+
+internal static class RecreateMissingIndexFiles
 {
-	internal static class RecreateMissingIndexFiles
+	/// <summary>
+	/// The tag used for log messages
+	/// </summary>
+	private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(RecreateMissingIndexFiles));
+
+	public static async Task Run(BackupDatabase database, IBackendManager backendManager, Options options, ITaskReader taskreader)
 	{
-		/// <summary>
-		/// The tag used for log messages
-		/// </summary>
-		private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(RecreateMissingIndexFiles));
-
-		public static Task Run(Channels channels, BackupDatabase database, Options options, ITaskReader taskreader)
+		if (options.IndexfilePolicy != Options.IndexFileStrategy.None)
 		{
-			return AutomationExtensions.RunTask(new
+			foreach (var blockfile in await database.GetMissingIndexFilesAsync().ConfigureAwait(false))
 			{
-				UploadChannel = channels.BackendRequest.AsWrite()
-			},
+				if (!await taskreader.ProgressRendevouz().ConfigureAwait(false))
+					return;
 
-			async self =>
-			{
-				if (options.IndexfilePolicy != Options.IndexFileStrategy.None)
-				{
-					foreach (var blockfile in await database.GetMissingIndexFilesAsync())
-					{
-						if (!await taskreader.ProgressRendevouz().ConfigureAwait(false))
-							return;
+				Logging.Log.WriteInformationMessage(LOGTAG, "RecreateMissingIndexFile", "Re-creating missing index file for {0}", blockfile);
+				var w = await Common.IndexVolumeCreator.CreateIndexVolume(blockfile, options, database).ConfigureAwait(false);
 
-						Logging.Log.WriteInformationMessage(LOGTAG, "RecreateMissingIndexFile", "Re-creating missing index file for {0}", blockfile);
-						var w = await Common.IndexVolumeCreator.CreateIndexVolume(blockfile, options, database);
-
-						if (!await taskreader.ProgressRendevouz().ConfigureAwait(false))
-							return;
-
-						await database.UpdateRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null);
-						await self.UploadChannel.WriteAsync(new IndexVolumeUploadRequest(w));
-					}
-				}
-			});
+				if (!await taskreader.ProgressRendevouz().ConfigureAwait(false))
+					return;
 
+				await database.UpdateRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null).ConfigureAwait(false);
+				await backendManager.PutAsync(w, null, null, false, taskreader.ProgressToken).ConfigureAwait(false);
+			}
 		}
-
 	}
+
 }
+

Duplicati/Library/Main/Operation/Backup/SpillCollectorProcess.cs

@@ -1,22 +1,22 @@
-// Copyright (C) 2025, The Duplicati Team
-// https://duplicati.com, hello@duplicati.com
-// 
-// Permission is hereby granted, free of charge, to any person obtaining a 
-// copy of this software and associated documentation files (the "Software"), 
-// to deal in the Software without restriction, including without limitation 
-// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
-// and/or sell copies of the Software, and to permit persons to whom the 
-// Software is furnished to do so, subject to the following conditions:
-// 
-// The above copyright notice and this permission notice shall be included in 
-// all copies or substantial portions of the Software.
-// 
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+// Copyright (C) 2025, The Duplicati Team
+// https://duplicati.com, hello@duplicati.com
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a 
+// copy of this software and associated documentation files (the "Software"), 
+// to deal in the Software without restriction, including without limitation 
+// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+// and/or sell copies of the Software, and to permit persons to whom the 
+// Software is furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in 
+// all copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
 // DEALINGS IN THE SOFTWARE.
 
 using CoCoL;
@@ -25,7 +25,6 @@ using Duplicati.Library.Main.Volumes;
 using System;
 using System.Collections.Generic;
 using System.Threading.Tasks;
-using static Duplicati.Library.Main.Operation.Common.BackendHandler;
 
 namespace Duplicati.Library.Main.Operation.Backup
 {
@@ -49,13 +48,12 @@ namespace Duplicati.Library.Main.Operation.Backup
     /// </summary>
     internal static class SpillCollectorProcess
     {
-        public static Task Run(Channels channels, Options options, BackupDatabase database, ITaskReader taskreader)
+        public static Task Run(Channels channels, Options options, BackupDatabase database, IBackendManager backendManager, ITaskReader taskreader)
         {
             return AutomationExtensions.RunTask(
             new
             {
-                Input = channels.SpillPickup.AsRead(),
-                Output = channels.BackendRequest.AsWrite(),
+                Input = channels.SpillPickup.AsRead()
             },
 
             async self =>
@@ -135,7 +133,7 @@ namespace Duplicati.Library.Main.Operation.Backup
                             if (target.BlockVolume.Filesize > options.VolumeSize - options.Blocksize)
                             {
                                 target.BlockVolume.Close();
-                                await UploadVolumeAndIndex(target, self.Output, options, database).ConfigureAwait(false);
+                                await UploadVolumeAndIndex(target, options, database, backendManager, taskreader).ConfigureAwait(false);
                                 target = null;
                             }
                         }
@@ -156,25 +154,25 @@ namespace Duplicati.Library.Main.Operation.Backup
                     await taskreader.ProgressRendevouz().ConfigureAwait(false);
 
                     n.BlockVolume.Close();
-                    await UploadVolumeAndIndex(n, self.Output, options, database).ConfigureAwait(false);
+                    await UploadVolumeAndIndex(n, options, database, backendManager, taskreader).ConfigureAwait(false);
                 }
 
             });
         }
 
-        private static async Task UploadVolumeAndIndex(SpillVolumeRequest target, IWriteChannel<IUploadRequest> outputChannel, Options options, BackupDatabase database)
+        private static async Task UploadVolumeAndIndex(SpillVolumeRequest target, Options options, BackupDatabase database, IBackendManager backendManager, ITaskReader taskreader)
         {
-            var blockEntry = target.BlockVolume.CreateFileEntryForUpload(options);
-
-            TemporaryIndexVolume indexVolumeCopy = null;
+            IndexVolumeWriter indexVolumeCopy = null;
             if (target.IndexVolume != null)
             {
-                indexVolumeCopy = new TemporaryIndexVolume(options);
-                target.IndexVolume.CopyTo(indexVolumeCopy, false);
+                // TODO: It is much easier to let the BackendManager deal with index files,
+                // but it adds a bit of strain to the database
+                indexVolumeCopy = await target.IndexVolume.CreateVolume(target.BlockVolume.RemoteFilename, options, database).ConfigureAwait(false);
+                // Create link before upload is started, it will be removed later if upload fails
+                await database.AddIndexBlockLinkAsync(indexVolumeCopy.VolumeID, target.BlockVolume.VolumeID).ConfigureAwait(false);
             }
 
-            var uploadRequest = new VolumeUploadRequest(target.BlockVolume, blockEntry, indexVolumeCopy, options, database);
-            await outputChannel.WriteAsync(uploadRequest).ConfigureAwait(false);
+            await backendManager.PutAsync(target.BlockVolume, indexVolumeCopy, null, false, taskreader.ProgressToken).ConfigureAwait(false);
         }
     }
 }

Duplicati/Library/Main/Operation/Backup/UploadRealFilelist.cs

@@ -1,91 +1,83 @@
-// Copyright (C) 2025, The Duplicati Team
-// https://duplicati.com, hello@duplicati.com
-// 
-// Permission is hereby granted, free of charge, to any person obtaining a 
-// copy of this software and associated documentation files (the "Software"), 
-// to deal in the Software without restriction, including without limitation 
-// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
-// and/or sell copies of the Software, and to permit persons to whom the 
-// Software is furnished to do so, subject to the following conditions:
-// 
-// The above copyright notice and this permission notice shall be included in 
-// all copies or substantial portions of the Software.
-// 
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+// Copyright (C) 2025, The Duplicati Team
+// https://duplicati.com, hello@duplicati.com
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a 
+// copy of this software and associated documentation files (the "Software"), 
+// to deal in the Software without restriction, including without limitation 
+// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+// and/or sell copies of the Software, and to permit persons to whom the 
+// Software is furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in 
+// all copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
 // DEALINGS IN THE SOFTWARE.
 
 using System;
 using System.Threading.Tasks;
-using CoCoL;
 using Duplicati.Library.Main.Volumes;
 
-namespace Duplicati.Library.Main.Operation.Backup
+namespace Duplicati.Library.Main.Operation.Backup;
+
+internal static class UploadRealFilelist
 {
-    internal static class UploadRealFilelist
+    /// <summary>
+    /// The tag used for log messages
+    /// </summary>
+    private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(UploadRealFilelist));
+
+    public static async Task Run(BackupResults result, BackupDatabase db, IBackendManager backendManager, Options options, FilesetVolumeWriter filesetvolume, long filesetid, Common.ITaskReader taskreader)
     {
-        /// <summary>
-        /// The tag used for log messages
-        /// </summary>
-        private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(UploadRealFilelist));
+        // We ignore the stop signal, but not the pause and terminate
+        await taskreader.ProgressRendevouz().ConfigureAwait(false);
 
-        public static Task Run(Channels channels, BackupResults result, BackupDatabase db, Options options, FilesetVolumeWriter filesetvolume, long filesetid, Common.ITaskReader taskreader)
+        // Update the reported source and backend changes
+        using (new Logging.Timer(LOGTAG, "UpdateChangeStatistics", "UpdateChangeStatistics"))
+            await db.UpdateChangeStatisticsAsync(result);
+
+        var changeCount =
+            result.AddedFiles + result.ModifiedFiles + result.DeletedFiles +
+            result.AddedFolders + result.ModifiedFolders + result.DeletedFolders +
+            result.AddedSymlinks + result.ModifiedSymlinks + result.DeletedSymlinks;
+
+        //Changes in the filelist triggers a filelist upload
+        if (options.UploadUnchangedBackups || changeCount > 0)
         {
-            return AutomationExtensions.RunTask(new
+            using (new Logging.Timer(LOGTAG, "UploadNewFileset", "Uploading a new fileset"))
             {
-                Output = channels.BackendRequest.AsWrite()
-            },
+                if (!string.IsNullOrEmpty(options.ControlFiles))
+                    foreach (var p in options.ControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries))
+                        filesetvolume.AddControlFile(p, options.GetCompressionHintFromFilename(p));
 
-            async self =>
-            {
                 // We ignore the stop signal, but not the pause and terminate
                 await taskreader.ProgressRendevouz().ConfigureAwait(false);
 
-                // Update the reported source and backend changes
-                using (new Logging.Timer(LOGTAG, "UpdateChangeStatistics", "UpdateChangeStatistics"))
-                    await db.UpdateChangeStatisticsAsync(result);
+                await db.WriteFilesetAsync(filesetvolume, filesetid).ConfigureAwait(false);
+                filesetvolume.Close();
 
-                var changeCount =
-                    result.AddedFiles + result.ModifiedFiles + result.DeletedFiles +
-                    result.AddedFolders + result.ModifiedFolders + result.DeletedFolders +
-                    result.AddedSymlinks + result.ModifiedSymlinks + result.DeletedSymlinks;
+                // We ignore the stop signal, but not the pause and terminate
+                await taskreader.ProgressRendevouz().ConfigureAwait(false);
 
-                //Changes in the filelist triggers a filelist upload
-                if (options.UploadUnchangedBackups || changeCount > 0)
-                {
-                    using (new Logging.Timer(LOGTAG, "UploadNewFileset", "Uploading a new fileset"))
-                    {
-                        if (!string.IsNullOrEmpty(options.ControlFiles))
-                            foreach (var p in options.ControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries))
-                                filesetvolume.AddControlFile(p, options.GetCompressionHintFromFilename(p));
+                await db.UpdateRemoteVolumeAsync(filesetvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null).ConfigureAwait(false);
+                await db.CommitTransactionAsync("CommitUpdateRemoteVolume").ConfigureAwait(false);
 
-                        // We ignore the stop signal, but not the pause and terminate
-                        await taskreader.ProgressRendevouz().ConfigureAwait(false);
-
-                        await db.WriteFilesetAsync(filesetvolume, filesetid);
-                        filesetvolume.Close();
-
-                        // We ignore the stop signal, but not the pause and terminate
-                        await taskreader.ProgressRendevouz().ConfigureAwait(false);
-
-                        await db.UpdateRemoteVolumeAsync(filesetvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null);
-                        await db.CommitTransactionAsync("CommitUpdateRemoteVolume");
-                        await self.Output.WriteAsync(new FilesetUploadRequest(filesetvolume));
-                    }
-                }
-                else
-                {
-                    Logging.Log.WriteVerboseMessage(LOGTAG, "RemovingLeftoverTempFile", "removing temp files, as no data needs to be uploaded");
-                    await db.RemoveRemoteVolumeAsync(filesetvolume.RemoteFilename);
-                }
-
-                await db.CommitTransactionAsync("CommitUpdateRemoteVolume");
-            });
+                await backendManager.PutAsync(filesetvolume, null, null, false, taskreader.ProgressToken).ConfigureAwait(false);
+            }
         }
+        else
+        {
+            Logging.Log.WriteVerboseMessage(LOGTAG, "RemovingLeftoverTempFile", "removing temp files, as no data needs to be uploaded");
+            await db.RemoveRemoteVolumeAsync(filesetvolume.RemoteFilename);
+        }
+
+        await db.CommitTransactionAsync("CommitUpdateRemoteVolume");
     }
 }
 
+

Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs

@@ -1,25 +1,24 @@
-// Copyright (C) 2025, The Duplicati Team
-// https://duplicati.com, hello@duplicati.com
-// 
-// Permission is hereby granted, free of charge, to any person obtaining a 
-// copy of this software and associated documentation files (the "Software"), 
-// to deal in the Software without restriction, including without limitation 
-// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
-// and/or sell copies of the Software, and to permit persons to whom the 
-// Software is furnished to do so, subject to the following conditions:
-// 
-// The above copyright notice and this permission notice shall be included in 
-// all copies or substantial portions of the Software.
-// 
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+// Copyright (C) 2025, The Duplicati Team
+// https://duplicati.com, hello@duplicati.com
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a 
+// copy of this software and associated documentation files (the "Software"), 
+// to deal in the Software without restriction, including without limitation 
+// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+// and/or sell copies of the Software, and to permit persons to whom the 
+// Software is furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in 
+// all copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
 // DEALINGS IN THE SOFTWARE.
 
-using CoCoL;
 using Duplicati.Library.Main.Operation.Common;
 using Duplicati.Library.Main.Volumes;
 using System;
@@ -38,114 +37,105 @@ namespace Duplicati.Library.Main.Operation.Backup
         /// </summary>
         private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(UploadSyntheticFilelist));
 
-        public static Task Run(Channels channels, BackupDatabase database, Options options, BackupResults result, ITaskReader taskreader, string lastTempFilelist, long lastTempFilesetId)
+        public static async Task Run(BackupDatabase database, Options options, BackupResults result, ITaskReader taskreader, IBackendManager backendManager, string lastTempFilelist, long lastTempFilesetId)
         {
-            return AutomationExtensions.RunTask(new
+            // Check if we should upload a synthetic filelist
+            if (options.DisableSyntheticFilelist || string.IsNullOrWhiteSpace(lastTempFilelist) || lastTempFilesetId <= 0)
+                return;
+
+            // Check that we still need to process this after the cleanup has performed its duties
+            var syntbase = await database.GetRemoteVolumeFromFilesetIDAsync(lastTempFilesetId);
+
+            // If we do not have a valid entry, warn and quit
+            if (syntbase.Name == null)
             {
-                UploadChannel = channels.BackendRequest.AsWrite()
-            },
+                // TODO: If the repair succeeds, this could give a false warning?
+                Logging.Log.WriteWarningMessage(LOGTAG, "MissingTemporaryFilelist", null, "Expected there to be a temporary fileset for synthetic filelist ({0}, {1}), but none was found?", lastTempFilesetId, lastTempFilelist);
+                return;
+            }
 
-            async self =>
+            // Files is missing or repaired
+            if (syntbase.State != RemoteVolumeState.Uploading && syntbase.State != RemoteVolumeState.Temporary)
             {
-                // Check if we should upload a synthetic filelist
-                if (options.DisableSyntheticFilelist || string.IsNullOrWhiteSpace(lastTempFilelist) || lastTempFilesetId <= 0)
-                    return;
+                Logging.Log.WriteInformationMessage(LOGTAG, "SkippingSyntheticListUpload", "Skipping synthetic upload because temporary fileset appers to be complete: ({0}, {1}, {2})", lastTempFilesetId, lastTempFilelist, syntbase.State);
+                return;
+            }
 
-                // Check that we still need to process this after the cleanup has performed its duties
-                var syntbase = await database.GetRemoteVolumeFromFilesetIDAsync(lastTempFilesetId);
+            // Ready to build and upload the synthetic list
+            await database.CommitTransactionAsync("PreSyntheticFilelist");
+            var incompleteFilesets = (await database.GetIncompleteFilesetsAsync()).OrderBy(x => x.Value).ToList();
 
-                // If we do not have a valid entry, warn and quit
-                if (syntbase.Name == null)
+            if (!incompleteFilesets.Any())
+            {
+                return;
+            }
+
+            if (!await taskreader.ProgressRendevouz().ConfigureAwait(false))
+                return;
+
+            result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PreviousBackupFinalize);
+            Logging.Log.WriteInformationMessage(LOGTAG, "PreviousBackupFilelistUpload", "Uploading filelist from previous interrupted backup");
+
+            var incompleteSet = incompleteFilesets.Last();
+            var badIds = from n in incompleteFilesets select n.Key;
+
+            var prevs = (from n in await database.GetFilesetTimesAsync()
+                         where
+                         n.Key < incompleteSet.Key
+                         &&
+                         !badIds.Contains(n.Key)
+                         orderby n.Key
+                         select n.Key).ToArray();
+
+            var prevId = prevs.Length == 0 ? -1 : prevs.Last();
+
+            FilesetVolumeWriter fsw = null;
+            try
+            {
+                var s = 1;
+                var fileTime = incompleteSet.Value + TimeSpan.FromSeconds(s);
+
+                // Probe for an unused filename
+                while (s < 60)
                 {
-                    // TODO: If the repair succeeds, this could give a false warning?
-                    Logging.Log.WriteWarningMessage(LOGTAG, "MissingTemporaryFilelist", null, "Expected there to be a temporary fileset for synthetic filelist ({0}, {1}), but none was found?", lastTempFilesetId, lastTempFilelist);
-                    return;
+                    var id = await database.GetRemoteVolumeIDAsync(VolumeBase.GenerateFilename(RemoteVolumeType.Files, options, null, fileTime));
+                    if (id < 0)
+                        break;
+
+                    fileTime = incompleteSet.Value + TimeSpan.FromSeconds(++s);
                 }
 
-                // Files is missing or repaired
-                if (syntbase.State != RemoteVolumeState.Uploading && syntbase.State != RemoteVolumeState.Temporary)
-                {
-                    Logging.Log.WriteInformationMessage(LOGTAG, "SkippingSyntheticListUpload", "Skipping synthetic upload because temporary fileset appers to be complete: ({0}, {1}, {2})", lastTempFilesetId, lastTempFilelist, syntbase.State);
-                    return;
-                }
+                fsw = new FilesetVolumeWriter(options, fileTime);
+                fsw.VolumeID = await database.RegisterRemoteVolumeAsync(fsw.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary);
 
-                // Ready to build and upload the synthetic list
-                await database.CommitTransactionAsync("PreSyntheticFilelist");
-                var incompleteFilesets = (await database.GetIncompleteFilesetsAsync()).OrderBy(x => x.Value).ToList();
+                if (!string.IsNullOrEmpty(options.ControlFiles))
+                    foreach (var p in options.ControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries))
+                        fsw.AddControlFile(p, options.GetCompressionHintFromFilename(p));
 
-                if (!incompleteFilesets.Any())
-                {
-                    return;
-                }
+                // We declare this to be a partial backup since the synthetic filelist is only created
+                // when a backup is interrupted.
+                fsw.CreateFilesetFile(false);
+                var newFilesetID = await database.CreateFilesetAsync(fsw.VolumeID, fileTime);
+                await database.LinkFilesetToVolumeAsync(newFilesetID, fsw.VolumeID);
+                await database.AppendFilesFromPreviousSetAsync(null, newFilesetID, prevId, fileTime);
+
+                await database.WriteFilesetAsync(fsw, newFilesetID);
+                fsw.Close();
 
                 if (!await taskreader.ProgressRendevouz().ConfigureAwait(false))
                     return;
 
-                result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PreviousBackupFinalize);
-                Logging.Log.WriteInformationMessage(LOGTAG, "PreviousBackupFilelistUpload", "Uploading filelist from previous interrupted backup");
+                await database.UpdateRemoteVolumeAsync(fsw.RemoteFilename, RemoteVolumeState.Uploading, -1, null);
+                await database.CommitTransactionAsync("CommitUpdateFilelistVolume");
 
-                var incompleteSet = incompleteFilesets.Last();
-                var badIds = from n in incompleteFilesets select n.Key;
-
-                var prevs = (from n in await database.GetFilesetTimesAsync()
-                             where
-                             n.Key < incompleteSet.Key
-                             &&
-                             !badIds.Contains(n.Key)
-                             orderby n.Key
-                             select n.Key).ToArray();
-
-                var prevId = prevs.Length == 0 ? -1 : prevs.Last();
-
-                FilesetVolumeWriter fsw = null;
-                try
-                {
-                    var s = 1;
-                    var fileTime = incompleteSet.Value + TimeSpan.FromSeconds(s);
-
-                    // Probe for an unused filename
-                    while (s < 60)
-                    {
-                        var id = await database.GetRemoteVolumeIDAsync(VolumeBase.GenerateFilename(RemoteVolumeType.Files, options, null, fileTime));
-                        if (id < 0)
-                            break;
-
-                        fileTime = incompleteSet.Value + TimeSpan.FromSeconds(++s);
-                    }
-
-                    fsw = new FilesetVolumeWriter(options, fileTime);
-                    fsw.VolumeID = await database.RegisterRemoteVolumeAsync(fsw.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary);
-
-                    if (!string.IsNullOrEmpty(options.ControlFiles))
-                        foreach (var p in options.ControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries))
-                            fsw.AddControlFile(p, options.GetCompressionHintFromFilename(p));
-
-                    // We declare this to be a partial backup since the synthetic filelist is only created
-                    // when a backup is interrupted.
-                    fsw.CreateFilesetFile(false);
-                    var newFilesetID = await database.CreateFilesetAsync(fsw.VolumeID, fileTime);
-                    await database.LinkFilesetToVolumeAsync(newFilesetID, fsw.VolumeID);
-                    await database.AppendFilesFromPreviousSetAsync(null, newFilesetID, prevId, fileTime);
-
-                    await database.WriteFilesetAsync(fsw, newFilesetID);
-                    fsw.Close();
-
-                    if (!await taskreader.ProgressRendevouz().ConfigureAwait(false))
-                        return;
-
-                    await database.UpdateRemoteVolumeAsync(fsw.RemoteFilename, RemoteVolumeState.Uploading, -1, null);
-                    await database.CommitTransactionAsync("CommitUpdateFilelistVolume");
-
-                    await self.UploadChannel.WriteAsync(new FilesetUploadRequest(fsw));
-                }
-                catch
-                {
-                    await database.RollbackTransactionAsync();
-                    fsw?.Dispose();
-                    throw;
-                }
+                await backendManager.PutAsync(fsw, null, null, false, taskreader.ProgressToken);
+            }
+            catch
+            {
+                await database.RollbackTransactionAsync();
+                fsw?.Dispose();
+                throw;
             }
-            );
         }
     }
 }

Duplicati/Library/Main/Operation/BackupHandler.cs

@@ -33,6 +33,7 @@ using Duplicati.Library.Utility;
 using Duplicati.Library.Common.IO;
 using Duplicati.Library.Logging;
 using Duplicati.Library.Main.Operation.Common;
+using System.Data;
 
 namespace Duplicati.Library.Main.Operation
 {
@@ -173,7 +174,6 @@ namespace Duplicati.Library.Main.Operation
 
         private sealed record PreBackupVerifyResult(
             LocalBackupDatabase Database,
-            BackendManager BackendManager,
             string LastTempFilelist,
             long LastTempFilesetId
         );
@@ -192,13 +192,12 @@ namespace Duplicati.Library.Main.Operation
         /// and returned to the caller in an open state in either case.
         /// </summary>
         /// <returns>Results from the pre-backup verification</returns>
-        private static async Task<PreBackupVerifyResult> PreBackupVerify(string backendurl, Options options, BackupResults result)
+        private static async Task<PreBackupVerifyResult> PreBackupVerify(Options options, BackupResults result, IBackendManager backendManager)
         {
             result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PreBackupVerify);
 
             // Setup variables
             LocalBackupDatabase database = null;
-            BackendManager backendManager = null;
 
             // If we have an interrupted backup, grab the fileset
             string lastTempFilelist = null;
@@ -209,8 +208,6 @@ namespace Duplicati.Library.Main.Operation
                 try
                 {
                     database = new LocalBackupDatabase(options.Dbpath, options);
-                    backendManager = new BackendManager(backendurl, options, result.BackendWriter, database);
-
                     result.SetDatabase(database);
                     result.Dryrun = options.Dryrun;
 
@@ -248,41 +245,36 @@ namespace Duplicati.Library.Main.Operation
                     {
                         if (options.NoBackendverification)
                         {
-                            FilelistProcessor.VerifyLocalList(backendManager, database);
+                            await FilelistProcessor.VerifyLocalList(backendManager, database, result.TaskControl.ProgressToken).ConfigureAwait(false);
                             await UpdateStorageStatsFromDatabase(result, database, options, backendManager, result.TaskControl.ProgressToken).ConfigureAwait(false);
                         }
                         else
-                            FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, new string[] { lastTempFilelist }, logErrors: false);
+                            await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, new string[] { lastTempFilelist }, logErrors: false).ConfigureAwait(false);
                     }
                     catch (RemoteListVerificationException ex)
                     {
                         if (options.AutoCleanup)
                         {
-                            Logging.Log.WriteWarningMessage(LOGTAG, "BackendVerifyFailedAttemptingCleanup", ex, "Backend verification failed, attempting automatic cleanup");
+                            Log.WriteWarningMessage(LOGTAG, "BackendVerifyFailedAttemptingCleanup", ex, "Backend verification failed, attempting automatic cleanup");
                             result.RepairResults = new RepairResults(result);
-
-                            // Close the database to allow the repair to run, it may create a new database
-                            backendManager.Dispose();
                             database.Dispose();
 
                             database = null;
-                            backendManager = null;
                             result.SetDatabase(null);
-                            new RepairHandler(backendurl, options, (RepairResults)result.RepairResults).Run();
+                            new RepairHandler(options, (RepairResults)result.RepairResults).Run(backendManager, null);
 
                             // Re-open the database and backend manager
                             database = new LocalBackupDatabase(options.Dbpath, options);
-                            backendManager = new BackendManager(backendurl, options, result.BackendWriter, database);
                             result.SetDatabase(database);
 
-                            Logging.Log.WriteInformationMessage(LOGTAG, "BackendCleanupFinished", "Backend cleanup finished, retrying verification");
-                            FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, new string[] { lastTempFilelist });
+                            Log.WriteInformationMessage(LOGTAG, "BackendCleanupFinished", "Backend cleanup finished, retrying verification");
+                            await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, new string[] { lastTempFilelist }).ConfigureAwait(false);
                         }
                         else
                             throw;
                     }
 
-                    return new PreBackupVerifyResult(database, backendManager, lastTempFilelist, lastTempFilesetId);
+                    return new PreBackupVerifyResult(database, lastTempFilelist, lastTempFilesetId);
                 }
                 catch
                 {
@@ -296,7 +288,7 @@ namespace Duplicati.Library.Main.Operation
         /// <summary>
         /// Performs the bulk of work by starting all relevant processes
         /// </summary>
-        private static async Task RunMainOperation(Backup.Channels channels, IEnumerable<string> sources, Snapshots.ISnapshotService snapshot, UsnJournalService journalService, Backup.BackupDatabase database, Backup.BackupStatsCollector stats, Options options, IFilter sourcefilter, IFilter filter, BackupResults result, Common.ITaskReader taskreader, long filesetid, long lastfilesetid)
+        private static async Task RunMainOperation(Backup.Channels channels, IEnumerable<string> sources, Snapshots.ISnapshotService snapshot, UsnJournalService journalService, Backup.BackupDatabase database, IBackendManager backendManager, Backup.BackupStatsCollector stats, Options options, IFilter sourcefilter, IFilter filter, BackupResults result, ITaskReader taskreader, long filesetid, long lastfilesetid)
         {
             using (new Logging.Timer(LOGTAG, "BackupMainOperation", "BackupMainOperation"))
             {
@@ -306,7 +298,7 @@ namespace Duplicati.Library.Main.Operation
                 Task all = Task.WhenAll(
                     new[]
                         {
-                                Backup.DataBlockProcessor.Run(channels, database, options, taskreader),
+                                Backup.DataBlockProcessor.Run(channels, database, backendManager, options, taskreader),
                                 Backup.FileBlockProcessor.Run(channels, snapshot, options, database, stats, taskreader),
                                 Backup.StreamBlockSplitter.Run(channels, options, database, taskreader),
                                 Backup.FileEnumerationProcess.Run(channels, sources, snapshot, journalService,
@@ -316,21 +308,21 @@ namespace Duplicati.Library.Main.Operation
                                     () => result.PartialBackup = true, CancellationToken.None),
                                 Backup.FilePreFilterProcess.Run(channels, snapshot, options, stats, database),
                                 Backup.MetadataPreProcess.Run(channels, snapshot, options, database, lastfilesetid, taskreader),
-                                Backup.SpillCollectorProcess.Run(channels, options, database, taskreader),
+                                Backup.SpillCollectorProcess.Run(channels, options, database, backendManager, taskreader),
                                 Backup.ProgressHandler.Run(channels, result)
                         }
                         // Spawn additional block hashers
-                        .Union(
+                        .Concat(
                             Enumerable.Range(0, options.ConcurrencyBlockHashers - 1).Select(x =>
                                 Backup.StreamBlockSplitter.Run(channels, options, database, taskreader))
                         )
                         // Spawn additional compressors
-                        .Union(
+                        .Concat(
                             Enumerable.Range(0, options.ConcurrencyCompressors - 1).Select(x =>
-                                Backup.DataBlockProcessor.Run(channels, database, options, taskreader))
+                                Backup.DataBlockProcessor.Run(channels, database, backendManager, options, taskreader))
                         )
                         // Spawn additional file processors
-                        .Union(
+                        .Concat(
                             Enumerable.Range(0, options.ConcurrencyFileprocessors - 1).Select(x =>
                                 Backup.FileBlockProcessor.Run(channels, snapshot, options, database, stats, taskreader)
                         )
@@ -393,7 +385,7 @@ namespace Duplicati.Library.Main.Operation
             }
         }
 
-        private void CompactIfRequired(BackendManager backend, long lastVolumeSize)
+        private async Task CompactIfRequired(IBackendManager backendManager, long lastVolumeSize)
         {
             var currentIsSmall = lastVolumeSize != -1 && lastVolumeSize <= m_options.SmallFileSize;
 
@@ -402,7 +394,7 @@ namespace Duplicati.Library.Main.Operation
                 m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Delete);
                 m_result.DeleteResults = new DeleteResults(m_result);
                 using (var db = new LocalDeleteDatabase(m_database))
-                    new DeleteHandler(backend.BackendUrl, m_options, (DeleteResults)m_result.DeleteResults).DoRun(db, ref m_transaction, true, currentIsSmall, backend);
+                    new DeleteHandler(m_options, (DeleteResults)m_result.DeleteResults).DoRun(db, ref m_transaction, true, currentIsSmall, backendManager);
 
             }
             else if (currentIsSmall && !m_options.NoAutoCompact)
@@ -410,19 +402,19 @@ namespace Duplicati.Library.Main.Operation
                 m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Compact);
                 m_result.CompactResults = new CompactResults(m_result);
                 using (var db = new LocalDeleteDatabase(m_database))
-                    new CompactHandler(backend.BackendUrl, m_options, (CompactResults)m_result.CompactResults).DoCompact(db, true, ref m_transaction, backend);
+                {
+                    (var _, var tr) = await new CompactHandler(m_options, (CompactResults)m_result.CompactResults).DoCompact(db, true, m_transaction, backendManager);
+                    m_transaction = tr;
+                }
             }
         }
 
-        private void PostBackupVerification(string currentFilelistVolume)
+        private async Task PostBackupVerification(string currentFilelistVolume, IBackendManager backendManager)
         {
             m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PostBackupVerify);
-            using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, m_database))
-            {
-                using (new Logging.Timer(LOGTAG, "AfterBackupVerify", "AfterBackupVerify"))
-                    FilelistProcessor.VerifyRemoteList(backend, m_options, m_database, m_result.BackendWriter, new string[] { currentFilelistVolume });
-                backend.WaitForComplete(m_database, null);
-            }
+            using (new Logging.Timer(LOGTAG, "AfterBackupVerify", "AfterBackupVerify"))
+                await FilelistProcessor.VerifyRemoteList(backendManager, m_options, m_database, m_result.BackendWriter, new string[] { currentFilelistVolume }).ConfigureAwait(false);
+            await backendManager.WaitForEmptyAsync(m_database, m_transaction, m_taskReader.ProgressToken);
 
             long remoteVolumeCount = m_database.GetRemoteVolumes().LongCount(x => x.State == RemoteVolumeState.Verified);
             long samplesToTest = Math.Max(m_options.BackupTestSampleCount, (long)Math.Round(remoteVolumeCount * (m_options.BackupTestPercentage / 100m), MidpointRounding.AwayFromZero));
@@ -432,16 +424,15 @@ namespace Duplicati.Library.Main.Operation
                 m_result.TestResults = new TestResults(m_result);
 
                 using (var testdb = new LocalTestDatabase(m_database))
-                using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, testdb))
-                    new TestHandler(m_backendurl, m_options, (TestResults)m_result.TestResults)
-                        .DoRun(samplesToTest, testdb, backend);
+                    await new TestHandler(m_options, (TestResults)m_result.TestResults)
+                        .DoRun(samplesToTest, testdb, backendManager).ConfigureAwait(false);
             }
         }
 
         /// <summary>
         /// Handler for computing backend statistics, without relying on a remote folder listing
         /// </summary>
-        private static async Task UpdateStorageStatsFromDatabase(BackupResults result, LocalBackupDatabase database, Options options, BackendManager backendManager, CancellationToken cancelToken)
+        private static async Task UpdateStorageStatsFromDatabase(BackupResults result, LocalBackupDatabase database, Options options, IBackendManager backendManager, CancellationToken cancelToken)
         {
             if (result.BackendWriter != null)
             {
@@ -473,8 +464,10 @@ namespace Duplicati.Library.Main.Operation
             if (tasks == null || tasks.Length == 0)
                 return source;
 
-            var ex = new List<Exception>();
-            ex.Add(source);
+            var ex = new List<Exception>
+            {
+                source
+            };
 
             foreach (var t in tasks)
                 if (t != null)
@@ -492,36 +485,30 @@ namespace Duplicati.Library.Main.Operation
                 return new AggregateException(ex.First().Message, ex);
         }
 
-        private static async Task<long> FlushBackend(BackupResults result, IWriteChannel<Backup.IUploadRequest> uploadtarget, Task uploader)
+        private static async Task<long> FlushBackend(LocalDatabase database, IDbTransaction transaction, BackupResults result, IBackendManager backendManager)
         {
             // Wait for upload completion
             result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_WaitForUpload);
 
-            if (!await uploadtarget.IsRetiredAsync)
+            try
             {
-                try
-                {
-                    var flushReq = new Backup.FlushRequest();
-                    await uploadtarget.WriteAsync(flushReq).ConfigureAwait(false);
-                    await uploader.ConfigureAwait(false);
-                    // Grab the size of the last uploaded volume
-                    return await flushReq.LastWriteSizeAsync;
-                }
-                catch (RetiredException)
-                {
-                    // Retired check is not atomic, so this exception can still happen
-                }
+                await backendManager.WaitForEmptyAsync(database, transaction, result.TaskControl.ProgressToken).ConfigureAwait(false);
+                // Grab the size of the last uploaded volume
+                return backendManager.LastWriteSize;
             }
-            await uploader.ConfigureAwait(false);
+            catch (RetiredException)
+            {
+            }
+
             return -1;
         }
 
-        public async Task RunAsync(string[] sources, Library.Utility.IFilter filter)
+        public async Task RunAsync(string[] sources, IBackendManager backendManager, Library.Utility.IFilter filter)
         {
             m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Begin);
 
             // Do a remote verification, unless disabled
-            var (database, backendManager, lastTempFilelist, lastTempFilesetId) = await PreBackupVerify(m_backendurl, m_options, m_result);
+            var (database, lastTempFilelist, lastTempFilesetId) = await PreBackupVerify(m_options, m_result, backendManager);
 
             Backup.Channels channels = new();
 
@@ -531,29 +518,22 @@ namespace Duplicati.Library.Main.Operation
             m_sourceFilter = new Library.Utility.FilterExpression(sources, true);
 
             Task parallelScanner = null;
-            Task uploaderTask = null;
             try
             {
                 using (m_database = database)
-                using (backendManager)
                 using (var db = new Backup.BackupDatabase(m_database, m_options))
                 // Setup runners and instances here
                 using (var filesetvolume = new FilesetVolumeWriter(m_options, m_database.OperationTimestamp))
                 using (var stats = new Backup.BackupStatsCollector(m_result))
-                using (var uploadtarget = ChannelManager.GetChannel(channels.BackendRequest.AsWrite()))
                 {
                     long filesetid;
                     using var counterToken = CancellationTokenSource.CreateLinkedTokenSource(m_taskReader.ProgressToken);
-                    var uploader = new Backup.BackendUploader(backendManager.BorrowBackend(), () => DynamicLoader.BackendLoader.GetBackend(m_backendurl, m_options.RawOptions), m_options, db, m_result.TaskControl, stats);
                     using (var snapshot = GetSnapshot(sources, m_options))
                     {
                         try
                         {
-                            // Start the uploader process
-                            uploaderTask = uploader.Run(channels);
-
                             // If the previous backup was interrupted, send a synthetic list
-                            await Backup.UploadSyntheticFilelist.Run(channels, db, m_options, m_result, m_result.TaskControl, lastTempFilelist, lastTempFilesetId);
+                            await Backup.UploadSyntheticFilelist.Run(db, m_options, m_result, m_result.TaskControl, backendManager, lastTempFilelist, lastTempFilesetId);
 
                             // Grab the previous backup ID, if any
                             var prevfileset = m_database.FilesetTimes.FirstOrDefault();
@@ -563,7 +543,7 @@ namespace Duplicati.Library.Main.Operation
                             var lastfilesetid = prevfileset.Value.Ticks == 0 ? -1 : prevfileset.Key;
 
                             // Rebuild any index files that are missing
-                            await Backup.RecreateMissingIndexFiles.Run(channels, db, m_options, m_result.TaskControl);
+                            await Backup.RecreateMissingIndexFiles.Run(db, backendManager, m_options, m_result.TaskControl);
 
                             // Prepare the operation by registering the filelist
                             m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_ProcessingFiles);
@@ -595,7 +575,7 @@ namespace Duplicati.Library.Main.Operation
                             // Run the backup operation
                             if (await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
                             {
-                                await RunMainOperation(channels, sources, snapshot, journalService, db, stats, m_options, m_sourceFilter, m_filter, m_result, m_result.TaskControl, filesetid, lastfilesetid).ConfigureAwait(false);
+                                await RunMainOperation(channels, sources, snapshot, journalService, db, backendManager, stats, m_options, m_sourceFilter, m_filter, m_result, m_result.TaskControl, filesetid, lastfilesetid).ConfigureAwait(false);
                             }
                         }
                         finally
@@ -613,18 +593,15 @@ namespace Duplicati.Library.Main.Operation
                         await db.VerifyConsistencyAsync(m_options.Blocksize, m_options.BlockhashSize, false);
 
                     // Send the actual filelist
-                    await Backup.UploadRealFilelist.Run(channels, m_result, db, m_options, filesetvolume, filesetid, m_result.TaskControl);
+                    await Backup.UploadRealFilelist.Run(m_result, db, backendManager, m_options, filesetvolume, filesetid, m_result.TaskControl);
 
                     // Wait for upload completion
                     m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_WaitForUpload);
-                    var lastVolumeSize = await FlushBackend(m_result, uploadtarget, uploaderTask).ConfigureAwait(false);
+                    var lastVolumeSize = await FlushBackend(m_database, m_transaction, m_result, backendManager).ConfigureAwait(false);
 
                     // Make sure we have the database up-to-date
                     await db.CommitTransactionAsync("CommitAfterUpload", false);
 
-                    // Return the backend instance to the manager, if possible
-                    backendManager.ResetBackend(uploader.Backend);
-
                     // TODO: Remove this later
                     m_transaction = m_database.BeginTransaction();
 
@@ -632,12 +609,12 @@ namespace Duplicati.Library.Main.Operation
                     {
                         // If this throws, we should roll back the transaction
                         if (await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
-                            CompactIfRequired(backendManager, lastVolumeSize);
+                            await CompactIfRequired(backendManager, lastVolumeSize);
 
                         if (m_options.UploadVerificationFile && await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
                         {
                             m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_VerificationUpload);
-                            FilelistProcessor.UploadVerificationFile(backendManager.BackendUrl, m_options, m_result.BackendWriter, m_database, m_transaction);
+                            await FilelistProcessor.UploadVerificationFile(backendManager, m_options, m_database, m_transaction);
                         }
                     }
                     catch
@@ -674,7 +651,7 @@ namespace Duplicati.Library.Main.Operation
                             if (m_options.NoBackendverification)
                                 await UpdateStorageStatsFromDatabase(m_result, m_database, m_options, backendManager, m_taskReader.ProgressToken).ConfigureAwait(false);
                             else
-                                PostBackupVerification(filesetvolume.RemoteFilename);
+                                await PostBackupVerification(filesetvolume.RemoteFilename, backendManager).ConfigureAwait(false);
                         }
                     }
 
@@ -693,7 +670,7 @@ namespace Duplicati.Library.Main.Operation
             }
             catch (Exception ex)
             {
-                var aex = BuildException(ex, uploaderTask, parallelScanner);
+                var aex = BuildException(ex, parallelScanner);
                 Logging.Log.WriteErrorMessage(LOGTAG, "FatalError", ex, "Fatal error");
                 if (aex == ex)
                     throw;

Duplicati/Library/Main/Operation/Common/BackendHandler.cs

@@ -1,147 +0,0 @@
-// Copyright (C) 2025, The Duplicati Team
-// https://duplicati.com, hello@duplicati.com
-// 
-// Permission is hereby granted, free of charge, to any person obtaining a 
-// copy of this software and associated documentation files (the "Software"), 
-// to deal in the Software without restriction, including without limitation 
-// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
-// and/or sell copies of the Software, and to permit persons to whom the 
-// Software is furnished to do so, subject to the following conditions:
-// 
-// The above copyright notice and this permission notice shall be included in 
-// all copies or substantial portions of the Software.
-// 
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
-// DEALINGS IN THE SOFTWARE.
-
-using System;
-using Duplicati.Library.Utility;
-
-namespace Duplicati.Library.Main.Operation.Common
-{
-    /// <summary>
-    /// This class encapsulates access to the backend and ensures at most one connection is active at a time
-    /// </summary>
-    internal class BackendHandler : SingleRunner
-    {
-        /// <summary>
-        /// The tag used for logging
-        /// </summary>
-        private static readonly string LOGTAG = Logging.Log.LogTagFromType<BackendHandler>();
-
-        /// <summary>
-        /// Data storage for an ongoing backend operation
-        /// </summary>
-        public class FileEntryItem
-        {
-            /// <summary>
-            /// The current operation this entry represents
-            /// </summary>
-            public readonly BackendActionType Operation;
-            /// <summary>
-            /// The name of the remote file
-            /// </summary>
-            public string RemoteFilename;
-            /// <summary>
-            /// The name of the local file
-            /// </summary>
-            public string LocalFilename { get { return LocalTempfile; } }
-            /// <summary>
-            /// A reference to a temporary file that is disposed upon
-            /// failure or completion of the item
-            /// </summary>
-            public TempFile LocalTempfile;
-            /// <summary>
-            /// True if the item has been encrypted
-            /// </summary>
-            public bool Encrypted;
-            /// <summary>
-            /// The expected hash value of the file
-            /// </summary>
-            public string Hash;
-            /// <summary>
-            /// The expected size of the file
-            /// </summary>
-            public long Size;
-            /// <summary>
-            /// A flag indicating if the file is a extra metadata file
-            /// that has no entry in the database
-            /// </summary>
-            public bool TrackedInDb = true;
-            /// <summary>
-            /// A flag indicating if the operation is a retry run
-            /// </summary>
-            public bool IsRetry;
-
-            public FileEntryItem(BackendActionType operation, string remotefilename)
-            {
-                Operation = operation;
-                RemoteFilename = remotefilename;
-                Size = -1;
-            }
-
-            public void SetLocalfilename(string name)
-            {
-                this.LocalTempfile = Library.Utility.TempFile.WrapExistingFile(name);
-                this.LocalTempfile.Protected = true;
-            }
-
-            public void Encrypt(Options options)
-            {
-                if (!this.Encrypted && !options.NoEncryption)
-                {
-                    var tempfile = new Library.Utility.TempFile();
-                    using (var enc = DynamicLoader.EncryptionLoader.GetModule(options.EncryptionModule, options.Passphrase, options.RawOptions))
-                        enc.Encrypt(this.LocalFilename, tempfile);
-
-                    this.DeleteLocalFile();
-
-                    this.LocalTempfile = tempfile;
-                    this.Hash = null;
-                    this.Size = 0;
-                    this.Encrypted = true;
-                }
-            }
-
-            public static string CalculateFileHash(string filename, Options options)
-            {
-                using (System.IO.FileStream fs = System.IO.File.OpenRead(filename))
-                using (var hasher = HashFactory.CreateHasher(options.FileHashAlgorithm))
-                    return Convert.ToBase64String(hasher.ComputeHash(fs));
-            }
-
-
-            public bool UpdateHashAndSize(Options options)
-            {
-                if (Hash == null || Size < 0)
-                {
-                    Hash = CalculateFileHash(this.LocalFilename, options);
-                    Size = new System.IO.FileInfo(this.LocalFilename).Length;
-                    return true;
-                }
-
-                return false;
-            }
-
-            public void DeleteLocalFile()
-            {
-                if (this.LocalTempfile != null)
-                {
-                    try
-                    {
-                        this.LocalTempfile.Protected = false;
-                        this.LocalTempfile.Dispose();
-                    }
-                    catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "DeleteTemporaryFileError", ex, "Failed to dispose temporary file: {0}", this.LocalTempfile); }
-                    finally { this.LocalTempfile = null; }
-                }
-            }
-        }
-    }
-}
-

Duplicati/Library/Main/Operation/Common/IndexVolumeCreator.cs

@@ -27,6 +27,8 @@ using Duplicati.Library.Utility;
 
 namespace Duplicati.Library.Main.Operation.Common
 {
+    // TODO: Remove this class and simply rely on the database to create the index files
+
     /// <summary>
     /// A collection class for keeping the temporary data required to build an index file
     /// </summary>
@@ -67,22 +69,13 @@ namespace Duplicati.Library.Main.Operation.Common
         /// </summary>
         /// <returns>The index volume.</returns>
         /// <param name="blockfilename">The name of the block file.</param>
-        /// <param name="blockHash">Hash of the volume</param>
-        /// <param name="blockSize">Size of the volume</param>
-        public async Task<IndexVolumeWriter> CreateVolume(string blockfilename, string blockHash, long blockSize, Options options, DatabaseCommon database)
+        /// <param name="options">The options used in this run.</param>
+        /// <param name="database">The database to use.</param>
+        public async Task<IndexVolumeWriter> CreateVolume(string blockfilename, Options options, DatabaseCommon database)
         {
             var w = new IndexVolumeWriter(options);
             w.VolumeID = await database.RegisterRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary);
 
-            w.StartVolume(blockfilename);
-            foreach (var n in blockHashes)
-            {
-                var args = n.Split(new char[] { ':' }, 2);
-                w.AddBlock(args[1], long.Parse(args[0]));
-            }
-
-            w.FinishVolume(blockHash, blockSize);
-
             var enumerator = blockListHashes.GetEnumerator();
             while (enumerator.MoveNext())
             {
@@ -93,7 +86,12 @@ namespace Duplicati.Library.Main.Operation.Common
                 w.WriteBlocklist(hash, data, 0, data.Length);
             }
 
-            w.Close();
+            w.StartVolume(blockfilename);
+            foreach (var n in blockHashes)
+            {
+                var args = n.Split(new char[] { ':' }, 2);
+                w.AddBlock(args[1], long.Parse(args[0]));
+            }
 
             return w;
         }
@@ -192,39 +190,6 @@ namespace Duplicati.Library.Main.Operation.Common
                 return w;
             }
         }
-
-        /*public static async Task<IndexVolumeWriter> ReCreateIndexVolume(string selfname, Options options, Repair.RepairDatabase database)
-        {
-            using(var h = HashFactory.CreateHasher(options.BlockHashAlgorithm))
-            {
-                var w = new IndexVolumeWriter(options);
-                w.SetRemoteFilename(selfname);
-
-                foreach(var blockvolume in await database.GetBlockVolumesFromIndexNameAsync(selfname))
-                {                               
-                    w.StartVolume(blockvolume.Name);
-                    var volumeid = await database.GetRemoteVolumeIDAsync(blockvolume.Name);
-
-                    foreach(var b in await database.GetBlocksAsync(volumeid))
-                        w.AddBlock(b.Hash, b.Size);
-
-                    w.FinishVolume(blockvolume.Hash, blockvolume.Size);
-
-                    if (options.IndexfilePolicy == Options.IndexFileStrategy.Full)
-                        foreach(var b in await database.GetBlocklistsAsync(volumeid, options.Blocksize, options.BlockhashSize))
-                        {
-                            var bh = Convert.ToBase64String(h.ComputeHash(b.Item2, 0, b.Item3));
-                            if (bh != b.Item1)
-                                throw new Exception(string.Format("Internal consistency check failed, generated index block has wrong hash, {0} vs {1}", bh, b.Item1));
-                            w.WriteBlocklist(b.Item1, b.Item2, 0, b.Item3);
-                        }
-                }
-
-                w.Close();
-
-                return w;
-            }
-        }*/
     }
 }
 

Duplicati/Library/Main/Operation/CompactHandler.cs

@@ -1,22 +1,22 @@
-// Copyright (C) 2025, The Duplicati Team
-// https://duplicati.com, hello@duplicati.com
-// 
-// Permission is hereby granted, free of charge, to any person obtaining a 
-// copy of this software and associated documentation files (the "Software"), 
-// to deal in the Software without restriction, including without limitation 
-// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
-// and/or sell copies of the Software, and to permit persons to whom the 
-// Software is furnished to do so, subject to the following conditions:
-// 
-// The above copyright notice and this permission notice shall be included in 
-// all copies or substantial portions of the Software.
-// 
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+// Copyright (C) 2025, The Duplicati Team
+// https://duplicati.com, hello@duplicati.com
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a 
+// copy of this software and associated documentation files (the "Software"), 
+// to deal in the Software without restriction, including without limitation 
+// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+// and/or sell copies of the Software, and to permit persons to whom the 
+// Software is furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in 
+// all copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
 // DEALINGS IN THE SOFTWARE.
 
 using Duplicati.Library.Main.Database;
@@ -25,6 +25,8 @@ using Duplicati.Library.Utility;
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
 
 namespace Duplicati.Library.Main.Operation
 {
@@ -34,18 +36,16 @@ namespace Duplicati.Library.Main.Operation
         /// The tag used for logging
         /// </summary>
         private static readonly string LOGTAG = Logging.Log.LogTagFromType<CompactHandler>();
-        protected readonly string m_backendurl;
         protected readonly Options m_options;
         protected readonly CompactResults m_result;
 
-        public CompactHandler(string backend, Options options, CompactResults result)
+        public CompactHandler(Options options, CompactResults result)
         {
-            m_backendurl = backend;
             m_options = options;
             m_result = result;
         }
 
-        public virtual void Run()
+        public virtual async Task Run(IBackendManager backendManager)
         {
             if (!System.IO.File.Exists(m_options.Dbpath))
                 throw new Exception(string.Format("Database file does not exist: {0}", m_options.Dbpath));
@@ -59,10 +59,12 @@ namespace Duplicati.Library.Main.Operation
                     Utility.UpdateOptionsFromDb(db, m_options);
                     Utility.VerifyOptionsAndUpdateDatabase(db, m_options);
 
-                    var changed = DoCompact(db, false, ref tr, null);
+                    // TODO: Handle transaction scopes better
+                    var (changed, tr2) = DoCompact(db, false, tr, backendManager).Await();
+                    tr = tr2;
 
                     if (changed && m_options.UploadVerificationFile)
-                        FilelistProcessor.UploadVerificationFile(m_backendurl, m_options, m_result.BackendWriter, db, null);
+                        await FilelistProcessor.UploadVerificationFile(backendManager, m_options, db, null);
 
                     if (!m_options.Dryrun)
                     {
@@ -92,220 +94,216 @@ namespace Duplicati.Library.Main.Operation
             }
         }
 
-        internal bool DoCompact(LocalDeleteDatabase db, bool hasVerifiedBackend, ref System.Data.IDbTransaction transaction, BackendManager sharedBackend)
+        internal async Task<(bool, System.Data.IDbTransaction)> DoCompact(LocalDeleteDatabase db, bool hasVerifiedBackend, System.Data.IDbTransaction transaction, IBackendManager backendManager)
         {
+            var cancellationToken = CancellationToken.None;
             var report = db.GetCompactReport(m_options.VolumeSize, m_options.Threshold, m_options.SmallFileSize, m_options.SmallFileMaxCount, transaction);
             report.ReportCompactData();
 
             if (report.ShouldReclaim || report.ShouldCompact)
             {
                 // Workaround where we allow a running backendmanager to be used
-                using (var bk = sharedBackend == null ? new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db) : null)
+                if (!hasVerifiedBackend)
+                    await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, true, transaction).ConfigureAwait(false);
+
+                var newvol = new BlockVolumeWriter(m_options);
+                newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, transaction);
+
+                IndexVolumeWriter newvolindex = null;
+                if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None)
                 {
-                    var backend = bk ?? sharedBackend;
-                    if (!hasVerifiedBackend)
+                    newvolindex = new IndexVolumeWriter(m_options);
+                    newvolindex.VolumeID = db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, transaction);
+                    db.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, transaction);
+                }
+
+                var blocksInVolume = 0L;
+                var buffer = new byte[m_options.Blocksize];
+                var remoteList = db.GetRemoteVolumes().Where(n => n.State == RemoteVolumeState.Uploaded || n.State == RemoteVolumeState.Verified).ToArray();
+
+                //These are for bookkeeping
+                var uploadedVolumes = new List<KeyValuePair<string, long>>();
+                var deletedVolumes = new List<KeyValuePair<string, long>>();
+                var downloadedVolumes = new List<KeyValuePair<string, long>>();
+
+                //We start by deleting unused volumes to save space before uploading new stuff
+                List<IRemoteVolume> fullyDeleteable = [];
+                if (report.DeleteableVolumes.Any())
+                {
+                    var deleteableVolumesAsHashSet = new HashSet<string>(report.DeleteableVolumes);
+                    fullyDeleteable =
+                        remoteList
+                            .Where(n => deleteableVolumesAsHashSet.Contains(n.Name))
+                            .Cast<IRemoteVolume>()
+                            .ToList();
+                }
+                deletedVolumes.AddRange(DoDelete(db, backendManager, fullyDeleteable, ref transaction, cancellationToken));
+
+                // This list is used to pick up unused volumes,
+                // so they can be deleted once the upload of the
+                // required fragments is complete
+                var deleteableVolumes = new List<IRemoteVolume>();
+
+                if (report.ShouldCompact)
+                {
+                    newvolindex?.StartVolume(newvol.RemoteFilename);
+                    List<IRemoteVolume> volumesToDownload = [];
+                    if (report.CompactableVolumes.Any())
                     {
-                        FilelistProcessor.VerifyRemoteList(backend, m_options, db, m_result.BackendWriter, true, transaction);
-                    }
-
-                    BlockVolumeWriter newvol = new BlockVolumeWriter(m_options);
-                    newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, transaction);
-
-                    IndexVolumeWriter newvolindex = null;
-                    if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None)
-                    {
-                        newvolindex = new IndexVolumeWriter(m_options);
-                        newvolindex.VolumeID = db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, transaction);
-                        db.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, transaction);
-                    }
-
-                    long blocksInVolume = 0;
-                    byte[] buffer = new byte[m_options.Blocksize];
-                    var remoteList = db.GetRemoteVolumes().Where(n => n.State == RemoteVolumeState.Uploaded || n.State == RemoteVolumeState.Verified).ToArray();
-
-                    //These are for bookkeeping
-                    var uploadedVolumes = new List<KeyValuePair<string, long>>();
-                    var deletedVolumes = new List<KeyValuePair<string, long>>();
-                    var downloadedVolumes = new List<KeyValuePair<string, long>>();
-
-                    //We start by deleting unused volumes to save space before uploading new stuff
-                    List<IRemoteVolume> fullyDeleteable = [];
-                    if (report.DeleteableVolumes.Any())
-                    {
-                        var deleteableVolumesAsHashSet = new HashSet<string>(report.DeleteableVolumes);
-                        fullyDeleteable =
+                        var compactableVolumesAsHashSet = new HashSet<string>(report.CompactableVolumes);
+                        volumesToDownload =
                             remoteList
-                                .Where(n => deleteableVolumesAsHashSet.Contains(n.Name))
+                                .Where(n => compactableVolumesAsHashSet.Contains(n.Name))
                                 .Cast<IRemoteVolume>()
                                 .ToList();
                     }
-                    deletedVolumes.AddRange(DoDelete(db, backend, fullyDeleteable, ref transaction));
 
-                    // This list is used to pick up unused volumes,
-                    // so they can be deleted once the upload of the
-                    // required fragments is complete
-                    var deleteableVolumes = new List<IRemoteVolume>();
-
-                    if (report.ShouldCompact)
+                    using (var q = db.CreateBlockQueryHelper(transaction))
                     {
-                        newvolindex?.StartVolume(newvol.RemoteFilename);
-                        List<IRemoteVolume> volumesToDownload = [];
-                        if (report.CompactableVolumes.Any())
+                        await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(volumesToDownload, m_result.TaskControl.ProgressToken))
                         {
-                            var compactableVolumesAsHashSet = new HashSet<string>(report.CompactableVolumes);
-                            volumesToDownload =
-                                remoteList
-                                    .Where(n => compactableVolumesAsHashSet.Contains(n.Name))
-                                    .Cast<IRemoteVolume>()
-                                    .ToList();
-                        }
-
-                        using (var q = db.CreateBlockQueryHelper(transaction))
-                        {
-                            foreach (var entry in new AsyncDownloader(volumesToDownload, backend))
+                            using (tmpfile)
                             {
-                                using (var tmpfile = entry.TempFile)
+                                var entry = new RemoteVolume(name, hash, size);
+                                if (!m_result.TaskControl.ProgressRendevouz().Await())
                                 {
-                                    if (!m_result.TaskControl.ProgressRendevouz().Await())
-                                    {
-                                        backend.WaitForComplete(db, transaction);
-                                        return false;
-                                    }
+                                    backendManager.WaitForEmptyAsync(db, transaction, cancellationToken).Await();
+                                    return (false, transaction);
+                                }
 
-                                    downloadedVolumes.Add(new KeyValuePair<string, long>(entry.Name, entry.Size));
-                                    var inst = VolumeBase.ParseFilename(entry.Name);
-                                    using (var f = new BlockVolumeReader(inst.CompressionModule, tmpfile, m_options))
+                                downloadedVolumes.Add(new KeyValuePair<string, long>(entry.Name, entry.Size));
+                                var inst = VolumeBase.ParseFilename(entry.Name);
+                                using (var f = new BlockVolumeReader(inst.CompressionModule, tmpfile, m_options))
+                                {
+                                    foreach (var e in f.Blocks)
                                     {
-                                        foreach (var e in f.Blocks)
+                                        if (q.UseBlock(e.Key, e.Value, transaction))
                                         {
-                                            if (q.UseBlock(e.Key, e.Value, transaction))
+                                            //TODO: How do we get the compression hint? Reverse query for filename in db?
+                                            var s = f.ReadBlock(e.Key, buffer);
+                                            if (s != e.Value)
+                                                throw new Exception(string.Format("Size mismatch problem for block {0}, {1} vs {2}", e.Key, s, e.Value));
+
+                                            newvol.AddBlock(e.Key, buffer, 0, s, Duplicati.Library.Interface.CompressionHint.Compressible);
+                                            if (newvolindex != null)
+                                                newvolindex.AddBlock(e.Key, e.Value);
+
+                                            db.RegisterDuplicatedBlock(e.Key, e.Value, newvol.VolumeID, transaction);
+                                            blocksInVolume++;
+
+                                            if (newvol.Filesize > m_options.VolumeSize)
                                             {
-                                                //TODO: How do we get the compression hint? Reverse query for filename in db?
-                                                var s = f.ReadBlock(e.Key, buffer);
-                                                if (s != e.Value)
-                                                    throw new Exception(string.Format("Size mismatch problem for block {0}, {1} vs {2}", e.Key, s, e.Value));
+                                                FinishVolumeAndUpload(db, backendManager, newvol, newvolindex, uploadedVolumes);
 
-                                                newvol.AddBlock(e.Key, buffer, 0, s, Duplicati.Library.Interface.CompressionHint.Compressible);
-                                                if (newvolindex != null)
-                                                    newvolindex.AddBlock(e.Key, e.Value);
+                                                newvol = new BlockVolumeWriter(m_options);
+                                                newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, transaction);
 
-                                                db.RegisterDuplicatedBlock(e.Key, e.Value, newvol.VolumeID, transaction);
-                                                blocksInVolume++;
-
-                                                if (newvol.Filesize > m_options.VolumeSize)
+                                                if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None)
                                                 {
-                                                    FinishVolumeAndUpload(db, backend, newvol, newvolindex, uploadedVolumes);
+                                                    newvolindex = new IndexVolumeWriter(m_options);
+                                                    newvolindex.VolumeID = db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, transaction);
+                                                    db.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, transaction);
+                                                    newvolindex.StartVolume(newvol.RemoteFilename);
+                                                }
 
-                                                    newvol = new BlockVolumeWriter(m_options);
-                                                    newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, transaction);
+                                                blocksInVolume = 0;
 
-                                                    if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None)
-                                                    {
-                                                        newvolindex = new IndexVolumeWriter(m_options);
-                                                        newvolindex.VolumeID = db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, transaction);
-                                                        db.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, transaction);
-                                                        newvolindex.StartVolume(newvol.RemoteFilename);
-                                                    }
+                                                // Wait for the backend to catch up
+                                                backendManager.WaitForEmptyAsync(db, transaction, cancellationToken).Await();
 
-                                                    blocksInVolume = 0;
-
-                                                    // Wait for the backend to catch up
-                                                    backend.WaitForEmpty(db, transaction);
-
-                                                    // Commit as we have uploaded a volume
-                                                    if (!m_options.Dryrun)
-                                                    {
-                                                        transaction.Commit();
-                                                        transaction = db.BeginTransaction();
-                                                    }
+                                                // Commit as we have uploaded a volume
+                                                if (!m_options.Dryrun)
+                                                {
+                                                    transaction.Commit();
+                                                    transaction = db.BeginTransaction();
                                                 }
                                             }
                                         }
                                     }
-
-                                    deleteableVolumes.Add(entry);
                                 }
-                            }
 
-                            if (blocksInVolume > 0)
-                            {
-                                FinishVolumeAndUpload(db, backend, newvol, newvolindex, uploadedVolumes);
+                                deleteableVolumes.Add(entry);
                             }
-                            else
+                        }
+
+                        if (blocksInVolume > 0)
+                        {
+                            FinishVolumeAndUpload(db, backendManager, newvol, newvolindex, uploadedVolumes);
+                        }
+                        else
+                        {
+                            db.RemoveRemoteVolume(newvol.RemoteFilename, transaction);
+                            if (newvolindex != null)
                             {
-                                db.RemoveRemoteVolume(newvol.RemoteFilename, transaction);
-                                if (newvolindex != null)
-                                {
-                                    db.RemoveRemoteVolume(newvolindex.RemoteFilename, transaction);
-                                    newvolindex.FinishVolume(null, 0);
-                                }
+                                db.RemoveRemoteVolume(newvolindex.RemoteFilename, transaction);
+                                newvolindex.FinishVolume(null, 0);
                             }
                         }
                     }
-                    else
-                    {
-                        newvolindex?.Dispose();
-                        newvol.Dispose();
-                    }
-
-                    deletedVolumes.AddRange(DoDelete(db, backend, deleteableVolumes, ref transaction));
-
-                    var downloadSize = downloadedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a, x) => a + x.Value);
-                    var deletedSize = deletedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a, x) => a + x.Value);
-                    var uploadSize = uploadedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a, x) => a + x.Value);
-
-                    m_result.DeletedFileCount = deletedVolumes.Count;
-                    m_result.DownloadedFileCount = downloadedVolumes.Count;
-                    m_result.UploadedFileCount = uploadedVolumes.Count;
-                    m_result.DeletedFileSize = deletedSize;
-                    m_result.DownloadedFileSize = downloadSize;
-                    m_result.UploadedFileSize = uploadSize;
-                    m_result.Dryrun = m_options.Dryrun;
-
-                    if (m_result.Dryrun)
-                    {
-                        if (downloadedVolumes.Count == 0)
-                            Logging.Log.WriteDryrunMessage(LOGTAG, "CompactResults", "Would delete {0} files, which would reduce storage by {1}", m_result.DeletedFileCount, Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize));
-                        else
-                            Logging.Log.WriteDryrunMessage(LOGTAG, "CompactResults", "Would download {0} file(s) with a total size of {1}, delete {2} file(s) with a total size of {3}, and compact to {4} file(s) with a size of {5}, which would reduce storage by {6} file(s) and {7}",
-                                                                    m_result.DownloadedFileCount,
-                                                                    Library.Utility.Utility.FormatSizeString(m_result.DownloadedFileSize),
-                                                                    m_result.DeletedFileCount,
-                                                                    Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize), m_result.UploadedFileCount,
-                                                                    Library.Utility.Utility.FormatSizeString(m_result.UploadedFileSize),
-                                                                    m_result.DeletedFileCount - m_result.UploadedFileCount,
-                                                                    Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize - m_result.UploadedFileSize));
-                    }
-                    else
-                    {
-                        if (m_result.DownloadedFileCount == 0)
-                            Logging.Log.WriteInformationMessage(LOGTAG, "CompactResults", "Deleted {0} files, which reduced storage by {1}", m_result.DeletedFileCount, Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize));
-                        else
-                            Logging.Log.WriteInformationMessage(LOGTAG, "CompactResults", "Downloaded {0} file(s) with a total size of {1}, deleted {2} file(s) with a total size of {3}, and compacted to {4} file(s) with a size of {5}, which reduced storage by {6} file(s) and {7}",
-                                                              m_result.DownloadedFileCount,
-                                                              Library.Utility.Utility.FormatSizeString(downloadSize),
-                                                              m_result.DeletedFileCount,
-                                                              Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize),
-                                                              m_result.UploadedFileCount,
-                                                              Library.Utility.Utility.FormatSizeString(m_result.UploadedFileSize),
-                                                              m_result.DeletedFileCount - m_result.UploadedFileCount,
-                                                              Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize - m_result.UploadedFileSize));
-                    }
-
-                    backend.WaitForComplete(db, transaction);
+                }
+                else
+                {
+                    newvolindex?.Dispose();
+                    newvol.Dispose();
                 }
 
+                deletedVolumes.AddRange(DoDelete(db, backendManager, deleteableVolumes, ref transaction, cancellationToken));
+
+                var downloadSize = downloadedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a, x) => a + x.Value);
+                var deletedSize = deletedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a, x) => a + x.Value);
+                var uploadSize = uploadedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a, x) => a + x.Value);
+
+                m_result.DeletedFileCount = deletedVolumes.Count;
+                m_result.DownloadedFileCount = downloadedVolumes.Count;
+                m_result.UploadedFileCount = uploadedVolumes.Count;
+                m_result.DeletedFileSize = deletedSize;
+                m_result.DownloadedFileSize = downloadSize;
+                m_result.UploadedFileSize = uploadSize;
+                m_result.Dryrun = m_options.Dryrun;
+
+                if (m_result.Dryrun)
+                {
+                    if (downloadedVolumes.Count == 0)
+                        Logging.Log.WriteDryrunMessage(LOGTAG, "CompactResults", "Would delete {0} files, which would reduce storage by {1}", m_result.DeletedFileCount, Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize));
+                    else
+                        Logging.Log.WriteDryrunMessage(LOGTAG, "CompactResults", "Would download {0} file(s) with a total size of {1}, delete {2} file(s) with a total size of {3}, and compact to {4} file(s) with a size of {5}, which would reduce storage by {6} file(s) and {7}",
+                                                                m_result.DownloadedFileCount,
+                                                                Library.Utility.Utility.FormatSizeString(m_result.DownloadedFileSize),
+                                                                m_result.DeletedFileCount,
+                                                                Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize), m_result.UploadedFileCount,
+                                                                Library.Utility.Utility.FormatSizeString(m_result.UploadedFileSize),
+                                                                m_result.DeletedFileCount - m_result.UploadedFileCount,
+                                                                Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize - m_result.UploadedFileSize));
+                }
+                else
+                {
+                    if (m_result.DownloadedFileCount == 0)
+                        Logging.Log.WriteInformationMessage(LOGTAG, "CompactResults", "Deleted {0} files, which reduced storage by {1}", m_result.DeletedFileCount, Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize));
+                    else
+                        Logging.Log.WriteInformationMessage(LOGTAG, "CompactResults", "Downloaded {0} file(s) with a total size of {1}, deleted {2} file(s) with a total size of {3}, and compacted to {4} file(s) with a size of {5}, which reduced storage by {6} file(s) and {7}",
+                                                          m_result.DownloadedFileCount,
+                                                          Library.Utility.Utility.FormatSizeString(downloadSize),
+                                                          m_result.DeletedFileCount,
+                                                          Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize),
+                                                          m_result.UploadedFileCount,
+                                                          Library.Utility.Utility.FormatSizeString(m_result.UploadedFileSize),
+                                                          m_result.DeletedFileCount - m_result.UploadedFileCount,
+                                                          Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize - m_result.UploadedFileSize));
+                }
+
+                backendManager.WaitForEmptyAsync(db, transaction, cancellationToken).Await();
+
                 m_result.EndTime = DateTime.UtcNow;
-                return (m_result.DeletedFileCount + m_result.UploadedFileCount) > 0;
+                return ((m_result.DeletedFileCount + m_result.UploadedFileCount) > 0, transaction);
             }
             else
             {
                 m_result.EndTime = DateTime.UtcNow;
-                return false;
+                return (false, transaction);
             }
         }
 
-        private IEnumerable<KeyValuePair<string, long>> DoDelete(LocalDeleteDatabase db, BackendManager backend, IEnumerable<IRemoteVolume> deleteableVolumes, ref System.Data.IDbTransaction transaction)
+        private IEnumerable<KeyValuePair<string, long>> DoDelete(LocalDeleteDatabase db, IBackendManager backend, IEnumerable<IRemoteVolume> deleteableVolumes, ref System.Data.IDbTransaction transaction, CancellationToken cancellationToken)
         {
             // Find volumes that can be deleted
             var remoteFilesToRemove = db.ReOrderDeleteableVolumes(deleteableVolumes, transaction).ToList();
@@ -324,7 +322,7 @@ namespace Duplicati.Library.Main.Operation
             }
 
             // Before we commit the current state, make sure the backend has caught up
-            backend.WaitForEmpty(db, transaction);
+            backend.WaitForEmptyAsync(db, transaction, cancellationToken).Await();
 
             if (!m_options.Dryrun)
             {
@@ -332,10 +330,10 @@ namespace Duplicati.Library.Main.Operation
                 transaction = db.BeginTransaction();
             }
 
-            return PerformDelete(backend, remoteFilesToRemove);
+            return PerformDelete(backend, remoteFilesToRemove, cancellationToken);
         }
 
-        private void FinishVolumeAndUpload(LocalDeleteDatabase db, BackendManager backend, BlockVolumeWriter newvol, IndexVolumeWriter newvolindex, List<KeyValuePair<string, long>> uploadedVolumes)
+        private void FinishVolumeAndUpload(LocalDeleteDatabase db, IBackendManager backendManager, BlockVolumeWriter newvol, IndexVolumeWriter newvolindex, List<KeyValuePair<string, long>> uploadedVolumes)
         {
             Action indexVolumeFinished = () =>
             {
@@ -352,17 +350,17 @@ namespace Duplicati.Library.Main.Operation
             if (newvolindex != null)
                 uploadedVolumes.Add(new KeyValuePair<string, long>(newvolindex.RemoteFilename, newvolindex.Filesize));
             if (!m_options.Dryrun)
-                backend.Put(newvol, newvolindex, indexVolumeFinished);
+                backendManager.PutAsync(newvol, newvolindex, indexVolumeFinished, false, CancellationToken.None).Await();
             else
                 Logging.Log.WriteDryrunMessage(LOGTAG, "WouldUploadGeneratedBlockset", "Would upload generated blockset of size {0}", Library.Utility.Utility.FormatSizeString(newvol.Filesize));
         }
 
-        private IEnumerable<KeyValuePair<string, long>> PerformDelete(BackendManager backend, IEnumerable<IRemoteVolume> list)
+        private IEnumerable<KeyValuePair<string, long>> PerformDelete(IBackendManager backendManager, IEnumerable<IRemoteVolume> list, CancellationToken cancelToken)
         {
             foreach (var f in list)
             {
                 if (!m_options.Dryrun)
-                    backend.Delete(f.Name, f.Size);
+                    backendManager.DeleteAsync(f.Name, f.Size, false, cancelToken).Await();
                 else
                     Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteRemoteFile", "Would delete remote file: {0}, size: {1}", f.Name, Library.Utility.Utility.FormatSizeString(f.Size));
 

Duplicati/Library/Main/Operation/DeleteHandler.cs

@@ -1,22 +1,22 @@
-// Copyright (C) 2025, The Duplicati Team
-// https://duplicati.com, hello@duplicati.com
-// 
-// Permission is hereby granted, free of charge, to any person obtaining a 
-// copy of this software and associated documentation files (the "Software"), 
-// to deal in the Software without restriction, including without limitation 
-// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
-// and/or sell copies of the Software, and to permit persons to whom the 
-// Software is furnished to do so, subject to the following conditions:
-// 
-// The above copyright notice and this permission notice shall be included in 
-// all copies or substantial portions of the Software.
-// 
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+// Copyright (C) 2025, The Duplicati Team
+// https://duplicati.com, hello@duplicati.com
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a 
+// copy of this software and associated documentation files (the "Software"), 
+// to deal in the Software without restriction, including without limitation 
+// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+// and/or sell copies of the Software, and to permit persons to whom the 
+// Software is furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in 
+// all copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
 // DEALINGS IN THE SOFTWARE.
 
 using System;
@@ -25,6 +25,7 @@ using System.Collections.Generic;
 using Duplicati.Library.Interface;
 using Duplicati.Library.Main.Database;
 using Duplicati.Library.Utility;
+using System.Threading;
 
 namespace Duplicati.Library.Main.Operation
 {
@@ -36,17 +37,15 @@ namespace Duplicati.Library.Main.Operation
         internal static readonly string LOGTAG = Logging.Log.LogTagFromType<DeleteHandler>();
 
         private readonly DeleteResults m_result;
-        protected readonly string m_backendurl;
         protected readonly Options m_options;
 
-        public DeleteHandler(string backend, Options options, DeleteResults result)
+        public DeleteHandler(Options options, DeleteResults result)
         {
-            m_backendurl = backend;
             m_options = options;
             m_result = result;
         }
 
-        public void Run()
+        public void Run(IBackendManager backendManager)
         {
             if (!System.IO.File.Exists(m_options.Dbpath))
                 throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseFileMissing");
@@ -60,7 +59,7 @@ namespace Duplicati.Library.Main.Operation
                     Utility.UpdateOptionsFromDb(db, m_options);
                     Utility.VerifyOptionsAndUpdateDatabase(db, m_options);
 
-                    DoRun(db, ref tr, false, false, null);
+                    DoRun(db, ref tr, false, false, backendManager);
 
                     if (!m_options.Dryrun)
                     {
@@ -83,93 +82,86 @@ namespace Duplicati.Library.Main.Operation
             }
         }
 
-        public void DoRun(Database.LocalDeleteDatabase db, ref System.Data.IDbTransaction transaction, bool hasVerifiedBackend, bool forceCompact, BackendManager sharedManager)
+        public void DoRun(Database.LocalDeleteDatabase db, ref System.Data.IDbTransaction transaction, bool hasVerifiedBackend, bool forceCompact, IBackendManager backendManager)
         {
-            // Workaround where we allow a running backendmanager to be used
-            using (var bk = sharedManager == null ? new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db) : null)
+            CancellationToken cancellationToken = CancellationToken.None;
+            if (!hasVerifiedBackend)
+                FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, true, transaction).Await();
+
+            IListResultFileset[] filesets = db.FilesetsWithBackupVersion.ToArray();
+            List<IListResultFileset> versionsToDelete =
+            [
+                .. new SpecificVersionsRemover(this.m_options).GetFilesetsToDelete(filesets),
+                .. new KeepTimeRemover(this.m_options).GetFilesetsToDelete(filesets),
+                .. new RetentionPolicyRemover(this.m_options).GetFilesetsToDelete(filesets),
+            ];
+
+            // When determining the number of full versions to keep, we need to ignore the versions already marked for removal.
+            versionsToDelete.AddRange(new KeepVersionsRemover(this.m_options).GetFilesetsToDelete(filesets.Except(versionsToDelete)));
+
+            if (!m_options.AllowFullRemoval && filesets.Length == versionsToDelete.Count)
             {
-                var backend = bk ?? sharedManager;
-
-                if (!hasVerifiedBackend)
-                {
-                    FilelistProcessor.VerifyRemoteList(backend, m_options, db, m_result.BackendWriter, true, transaction);
-                }
-
-                IListResultFileset[] filesets = db.FilesetsWithBackupVersion.ToArray();
-                List<IListResultFileset> versionsToDelete = new List<IListResultFileset>();
-                versionsToDelete.AddRange(new SpecificVersionsRemover(this.m_options).GetFilesetsToDelete(filesets));
-                versionsToDelete.AddRange(new KeepTimeRemover(this.m_options).GetFilesetsToDelete(filesets));
-                versionsToDelete.AddRange(new RetentionPolicyRemover(this.m_options).GetFilesetsToDelete(filesets));
-
-                // When determining the number of full versions to keep, we need to ignore the versions already marked for removal.
-                versionsToDelete.AddRange(new KeepVersionsRemover(this.m_options).GetFilesetsToDelete(filesets.Except(versionsToDelete)));
-
-                if (!m_options.AllowFullRemoval && filesets.Length == versionsToDelete.Count)
-                {
-                    Logging.Log.WriteInformationMessage(LOGTAG, "PreventingLastFilesetRemoval", "Preventing removal of last fileset, use --{0} to allow removal ...", "allow-full-removal");
-                    versionsToDelete = versionsToDelete.OrderBy(x => x.Version).Skip(1).ToList();
-                }
-
-                if (versionsToDelete.Count > 0)
-                    Logging.Log.WriteInformationMessage(LOGTAG, "DeleteRemoteFileset", "Deleting {0} remote fileset(s) ...", versionsToDelete.Count);
-
-                var lst = db.DropFilesetsFromTable(versionsToDelete.Select(x => x.Time).ToArray(), transaction).ToArray();
-                foreach (var f in lst)
-                    db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, transaction);
-
-                if (!m_options.Dryrun)
-                {
-                    transaction.Commit();
-                    transaction = db.BeginTransaction();
-                }
-
-                foreach (var f in lst)
-                {
-                    if (!m_result.TaskControl.ProgressRendevouz().Await())
-                    {
-                        backend.WaitForComplete(db, transaction);
-                        return;
-                    }
-
-                    if (!m_options.Dryrun)
-                        backend.Delete(f.Key, f.Value);
-                    else
-                        Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteRemoteFileset", "Would delete remote fileset: {0}", f.Key);
-                }
-
-                if (sharedManager == null)
-                    backend.WaitForComplete(db, transaction);
-                else
-                    backend.WaitForEmpty(db, transaction);
-
-                var count = lst.Length;
-                if (!m_options.Dryrun)
-                {
-                    if (count == 0)
-                        Logging.Log.WriteInformationMessage(LOGTAG, "DeleteResults", "No remote filesets were deleted");
-                    else
-                        Logging.Log.WriteInformationMessage(LOGTAG, "DeleteResults", "Deleted {0} remote fileset(s)", count);
-                }
-                else
-                {
-
-                    if (count == 0)
-                        Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteResults", "No remote filesets would be deleted");
-                    else
-                        Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteResults", "{0} remote fileset(s) would be deleted", count);
-
-                    if (count > 0 && m_options.Dryrun)
-                        Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteHelp", "Remove --dry-run to actually delete files");
-                }
-
-                if (!m_options.NoAutoCompact && (forceCompact || versionsToDelete.Count > 0))
-                {
-                    m_result.CompactResults = new CompactResults(m_result);
-                    new CompactHandler(m_backendurl, m_options, (CompactResults)m_result.CompactResults).DoCompact(db, true, ref transaction, sharedManager);
-                }
-
-                m_result.SetResults(versionsToDelete.Select(v => new Tuple<long, DateTime>(v.Version, v.Time)), m_options.Dryrun);
+                Logging.Log.WriteInformationMessage(LOGTAG, "PreventingLastFilesetRemoval", "Preventing removal of last fileset, use --{0} to allow removal ...", "allow-full-removal");
+                versionsToDelete = versionsToDelete.OrderBy(x => x.Version).Skip(1).ToList();
             }
+
+            if (versionsToDelete.Count > 0)
+                Logging.Log.WriteInformationMessage(LOGTAG, "DeleteRemoteFileset", "Deleting {0} remote fileset(s) ...", versionsToDelete.Count);
+
+            var lst = db.DropFilesetsFromTable(versionsToDelete.Select(x => x.Time).ToArray(), transaction).ToArray();
+            foreach (var f in lst)
+                db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, transaction);
+
+            if (!m_options.Dryrun)
+            {
+                transaction.Commit();
+                transaction = db.BeginTransaction();
+            }
+
+            foreach (var f in lst)
+            {
+                if (!m_result.TaskControl.ProgressRendevouz().Await())
+                {
+                    backendManager.WaitForEmptyAsync(db, transaction, cancellationToken).Await();
+                    return;
+                }
+
+                if (!m_options.Dryrun)
+                    backendManager.DeleteAsync(f.Key, f.Value, false, cancellationToken).Await();
+                else
+                    Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteRemoteFileset", "Would delete remote fileset: {0}", f.Key);
+            }
+
+            backendManager.WaitForEmptyAsync(db, transaction, cancellationToken).Await();
+
+            var count = lst.Length;
+            if (!m_options.Dryrun)
+            {
+                if (count == 0)
+                    Logging.Log.WriteInformationMessage(LOGTAG, "DeleteResults", "No remote filesets were deleted");
+                else
+                    Logging.Log.WriteInformationMessage(LOGTAG, "DeleteResults", "Deleted {0} remote fileset(s)", count);
+            }
+            else
+            {
+
+                if (count == 0)
+                    Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteResults", "No remote filesets would be deleted");
+                else
+                    Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteResults", "{0} remote fileset(s) would be deleted", count);
+
+                if (count > 0 && m_options.Dryrun)
+                    Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteHelp", "Remove --dry-run to actually delete files");
+            }
+
+            if (!m_options.NoAutoCompact && (forceCompact || versionsToDelete.Count > 0))
+            {
+                m_result.CompactResults = new CompactResults(m_result);
+                var (_, tr) = new CompactHandler(m_options, (CompactResults)m_result.CompactResults).DoCompact(db, true, transaction, backendManager).Await();
+                transaction = tr;
+            }
+
+            m_result.SetResults(versionsToDelete.Select(v => new Tuple<long, DateTime>(v.Version, v.Time)), m_options.Dryrun);
         }
     }
 

Duplicati/Library/Main/Operation/FilelistProcessor.cs

@@ -1,22 +1,22 @@
-// Copyright (C) 2025, The Duplicati Team
-// https://duplicati.com, hello@duplicati.com
-// 
-// Permission is hereby granted, free of charge, to any person obtaining a 
-// copy of this software and associated documentation files (the "Software"), 
-// to deal in the Software without restriction, including without limitation 
-// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
-// and/or sell copies of the Software, and to permit persons to whom the 
-// Software is furnished to do so, subject to the following conditions:
-// 
-// The above copyright notice and this permission notice shall be included in 
-// all copies or substantial portions of the Software.
-// 
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+// Copyright (C) 2025, The Duplicati Team
+// https://duplicati.com, hello@duplicati.com
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a 
+// copy of this software and associated documentation files (the "Software"), 
+// to deal in the Software without restriction, including without limitation 
+// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+// and/or sell copies of the Software, and to permit persons to whom the 
+// Software is furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in 
+// all copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
 // DEALINGS IN THE SOFTWARE.
 
 using System;
@@ -27,6 +27,7 @@ using System.Linq;
 using Duplicati.Library.Interface;
 using System.Threading;
 using Duplicati.Library.Utility;
+using System.Threading.Tasks;
 
 namespace Duplicati.Library.Main.Operation
 {
@@ -42,7 +43,7 @@ namespace Duplicati.Library.Main.Operation
         /// Throws an error if there are issues with the remote storage
         /// </summary>
         /// <param name="database">The database to compare with</param>
-        public static void VerifyLocalList(BackendManager backend, LocalDatabase database)
+        public static async Task VerifyLocalList(IBackendManager backendManager, LocalDatabase database, CancellationToken cancellationToken)
         {
             var locallist = database.GetRemoteVolumes();
             foreach (var i in locallist)
@@ -60,7 +61,7 @@ namespace Duplicati.Library.Main.Operation
                         Logging.Log.WriteInformationMessage(LOGTAG, "RemovingStaleFile", "Removing remote file listed as {0}: {1}", i.State, i.Name);
                         try
                         {
-                            backend.Delete(i.Name, i.Size, true);
+                            await backendManager.DeleteAsync(i.Name, i.Size, true, cancellationToken).ConfigureAwait(false);
                         }
                         catch (Exception ex)
                         {
@@ -74,17 +75,17 @@ namespace Duplicati.Library.Main.Operation
                         break;
                 }
 
-                backend.FlushDbMessages();
+                await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false);
             }
         }
 
-        public static void VerifyRemoteList(BackendManager backend, Options options, LocalDatabase database, IBackendWriter backendWriter, bool latestVolumesOnly, IDbTransaction transaction)
+        public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter backendWriter, bool latestVolumesOnly, IDbTransaction transaction)
         {
             if (!options.NoBackendverification)
             {
                 LocalBackupDatabase backupDatabase = new LocalBackupDatabase(database, options);
                 IEnumerable<string> protectedFiles = backupDatabase.GetTemporaryFilelistVolumeNames(latestVolumesOnly, transaction);
-                FilelistProcessor.VerifyRemoteList(backend, options, database, backendWriter, protectedFiles);
+                await VerifyRemoteList(backend, options, database, backendWriter, protectedFiles).ConfigureAwait(false);
             }
         }
 
@@ -98,9 +99,9 @@ namespace Duplicati.Library.Main.Operation
         /// <param name="log">The log instance to use</param>
         /// <param name="protectedFiles">Filenames that should be exempted from deletion</param>
         /// <param name="logErrors">Disable the logging of errors to prevent spamming the log; exceptions will be thrown regardless</param>
-        public static void VerifyRemoteList(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log, IEnumerable<string> protectedFiles = null, bool logErrors = true)
+        public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter log, IEnumerable<string> protectedFiles = null, bool logErrors = true)
         {
-            var tp = RemoteListAnalysis(backend, options, database, log, protectedFiles);
+            var tp = await RemoteListAnalysis(backend, options, database, log, protectedFiles).ConfigureAwait(false);
             long extraCount = 0;
             long missingCount = 0;
 
@@ -177,17 +178,15 @@ namespace Duplicati.Library.Main.Operation
         /// </summary>
         /// <param name="backendurl">The backend url</param>
         /// <param name="options">The options to use</param>
-        /// <param name="result">The result writer</param>
         /// <param name="db">The attached database</param>
         /// <param name="transaction">An optional transaction object</param>
-        public static void UploadVerificationFile(string backendurl, Options options, IBackendWriter result, LocalDatabase db, System.Data.IDbTransaction transaction)
+        public static async Task UploadVerificationFile(IBackendManager backendManager, Options options, LocalDatabase db, IDbTransaction transaction)
         {
-            using (var backend = new BackendManager(backendurl, options, result, db))
             using (var tempfile = new Library.Utility.TempFile())
             {
                 var remotename = options.Prefix + "-verification.json";
-                using (var stream = new System.IO.StreamWriter(tempfile, false, System.Text.Encoding.UTF8))
-                    FilelistProcessor.CreateVerificationFile(db, stream);
+                await using (var stream = new System.IO.StreamWriter(tempfile, false, System.Text.Encoding.UTF8))
+                    CreateVerificationFile(db, stream);
 
                 if (options.Dryrun)
                 {
@@ -195,8 +194,8 @@ namespace Duplicati.Library.Main.Operation
                 }
                 else
                 {
-                    backend.PutUnencrypted(remotename, tempfile);
-                    backend.WaitForComplete(db, transaction);
+                    await backendManager.PutVerificationFileAsync(remotename, tempfile, CancellationToken.None).ConfigureAwait(false);
+                    await backendManager.WaitForEmptyAsync(db, transaction, CancellationToken.None).ConfigureAwait(false);
                 }
             }
         }
@@ -209,9 +208,9 @@ namespace Duplicati.Library.Main.Operation
         /// <param name="options">The options used</param>
         /// <param name="database">The database to compare with</param>
         /// <param name="protectedFiles">Filenames that should be exempted from deletion</param>
-        public static RemoteAnalysisResult RemoteListAnalysis(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log, IEnumerable<string> protectedFiles)
+        public static async Task<RemoteAnalysisResult> RemoteListAnalysis(IBackendManager backendManager, Options options, LocalDatabase database, IBackendWriter log, IEnumerable<string> protectedFiles)
         {
-            var rawlist = backend.List();
+            var rawlist = await backendManager.ListAsync(CancellationToken.None).ConfigureAwait(false);
             var lookup = new Dictionary<string, Volumes.IParsedVolume>();
             protectedFiles = protectedFiles ?? Enumerable.Empty<string>();
 
@@ -243,7 +242,7 @@ namespace Duplicati.Library.Main.Operation
             log.BackupListCount = database.FilesetTimes.Count();
             log.LastBackupDate = filesets.Count == 0 ? new DateTime(0) : filesets[0].Time.ToLocalTime();
 
-            CheckQuota(backend, options, log, knownFileSize);
+            await CheckQuota(backendManager, options, log, knownFileSize).ConfigureAwait(false);
 
             foreach (var s in remotelist)
                 lookup[s.File.Name] = s;
@@ -282,7 +281,7 @@ namespace Duplicati.Library.Main.Operation
                         if (remoteFound)
                         {
                             Logging.Log.WriteInformationMessage(LOGTAG, "RemoveUnwantedRemoteFile", "removing remote file listed as {0}: {1}", i.State, i.Name);
-                            backend.Delete(i.Name, i.Size, true);
+                            await backendManager.DeleteAsync(i.Name, i.Size, true, CancellationToken.None).ConfigureAwait(false);
                         }
                         else
                         {
@@ -333,7 +332,7 @@ namespace Duplicati.Library.Main.Operation
                             else
                             {
                                 Logging.Log.WriteInformationMessage(LOGTAG, "Remove incomplete file", "removing incomplete remote file listed as {0}: {1}", i.State, i.Name);
-                                backend.Delete(i.Name, i.Size, true);
+                                await backendManager.DeleteAsync(i.Name, i.Size, true, CancellationToken.None).ConfigureAwait(false);
                             }
                         }
                         break;
@@ -361,7 +360,7 @@ namespace Duplicati.Library.Main.Operation
                         break;
                 }
 
-                backend.FlushDbMessages();
+                await backendManager.WaitForEmptyAsync(database, null, CancellationToken.None).ConfigureAwait(false);
             }
 
             // cleanup deleted volumes in DB en block
@@ -380,12 +379,12 @@ namespace Duplicati.Library.Main.Operation
             };
         }
 
-        private static void CheckQuota(BackendManager backend, Options options, IBackendWriter log, long knownFileSize)
+        private static async Task CheckQuota(IBackendManager backendManager, Options options, IBackendWriter log, long knownFileSize)
         {
             if (options.QuotaDisable)
                 return;
 
-            var quota = backend.GetQuotaInfoAsync(CancellationToken.None).Await();
+            var quota = await backendManager.GetQuotaInfoAsync(CancellationToken.None).ConfigureAwait(false);
             if (quota != null)
             {
                 log.TotalQuotaSpace = quota.TotalQuotaSpace;

Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs

@@ -1,22 +1,22 @@
-// Copyright (C) 2025, The Duplicati Team
-// https://duplicati.com, hello@duplicati.com
-// 
-// Permission is hereby granted, free of charge, to any person obtaining a 
-// copy of this software and associated documentation files (the "Software"), 
-// to deal in the Software without restriction, including without limitation 
-// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
-// and/or sell copies of the Software, and to permit persons to whom the 
-// Software is furnished to do so, subject to the following conditions:
-// 
-// The above copyright notice and this permission notice shall be included in 
-// all copies or substantial portions of the Software.
-// 
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+// Copyright (C) 2025, The Duplicati Team
+// https://duplicati.com, hello@duplicati.com
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a 
+// copy of this software and associated documentation files (the "Software"), 
+// to deal in the Software without restriction, including without limitation 
+// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+// and/or sell copies of the Software, and to permit persons to whom the 
+// Software is furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in 
+// all copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
 // DEALINGS IN THE SOFTWARE.
 
 using System;
@@ -24,6 +24,7 @@ using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
 using Duplicati.Library.Interface;
+using Duplicati.Library.Utility;
 
 namespace Duplicati.Library.Main.Operation
 {
@@ -33,28 +34,26 @@ namespace Duplicati.Library.Main.Operation
         /// The tag used for logging
         /// </summary>
         private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(ListBrokenFilesHandler));
-        protected readonly string m_backendurl;
         protected readonly Options m_options;
         protected readonly ListBrokenFilesResults m_result;
 
-        public ListBrokenFilesHandler(string backend, Options options, ListBrokenFilesResults result)
+        public ListBrokenFilesHandler(Options options, ListBrokenFilesResults result)
         {
-            m_backendurl = backend;
             m_options = options;
             m_result = result;
         }
 
-        public void Run(Library.Utility.IFilter filter, Func<long, DateTime, long, string, long, bool> callbackhandler = null)
+        public void Run(IBackendManager backendManager, IFilter filter, Func<long, DateTime, long, string, long, bool> callbackhandler = null)
         {
             if (!System.IO.File.Exists(m_options.Dbpath))
                 throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist");
 
             using (var db = new Database.LocalListBrokenFilesDatabase(m_options.Dbpath))
             using (var tr = db.BeginTransaction())
-                DoRun(db, tr, filter, callbackhandler);
+                DoRun(backendManager, db, tr, filter, callbackhandler);
         }
 
-        public static Tuple<DateTime, long, long>[] GetBrokenFilesetsFromRemote(string backendurl, BasicResults result, Database.LocalListBrokenFilesDatabase db, System.Data.IDbTransaction transaction, Options options, out List<Database.RemoteVolumeEntry> missing)
+        public static Tuple<DateTime, long, long>[] GetBrokenFilesetsFromRemote(IBackendManager backendManager, BasicResults result, Database.LocalListBrokenFilesDatabase db, System.Data.IDbTransaction transaction, Options options, out List<Database.RemoteVolumeEntry> missing)
         {
             missing = null;
             var brokensets = db.GetBrokenFilesets(options.Time, options.Version, transaction).ToArray();
@@ -66,35 +65,32 @@ namespace Duplicati.Library.Main.Operation
 
                 Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesetsInDatabase", "No broken filesets found in database, checking for missing remote files");
 
-                using (var backend = new BackendManager(backendurl, options, result.BackendWriter, db))
+                var remotestate = FilelistProcessor.RemoteListAnalysis(backendManager, options, db, result.BackendWriter, null).Await();
+                if (!remotestate.ParsedVolumes.Any())
+                    throw new UserInformationException("No remote volumes were found, refusing purge", "CannotPurgeWithNoRemoteVolumes");
+
+                missing = remotestate.MissingVolumes.ToList();
+                if (missing.Count == 0)
                 {
-                    var remotestate = FilelistProcessor.RemoteListAnalysis(backend, options, db, result.BackendWriter, null);
-                    if (!remotestate.ParsedVolumes.Any())
-                        throw new UserInformationException("No remote volumes were found, refusing purge", "CannotPurgeWithNoRemoteVolumes");
-
-                    missing = remotestate.MissingVolumes.ToList();
-                    if (missing.Count == 0)
-                    {
-                        Logging.Log.WriteInformationMessage(LOGTAG, "NoMissingFilesFound", "Skipping operation because no files were found to be missing, and no filesets were recorded as broken.");
-                        return null;
-                    }
-
-                    // Mark all volumes as disposable
-                    foreach (var f in missing)
-                        db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, transaction);
-
-                    Logging.Log.WriteInformationMessage(LOGTAG, "MarkedRemoteFilesForDeletion", "Marked {0} remote files for deletion", missing.Count);
-
-                    // Drop all content from tables
-                    db.RemoveMissingBlocks(missing.Select(x => x.Name), transaction);
+                    Logging.Log.WriteInformationMessage(LOGTAG, "NoMissingFilesFound", "Skipping operation because no files were found to be missing, and no filesets were recorded as broken.");
+                    return null;
                 }
+
+                // Mark all volumes as disposable
+                foreach (var f in missing)
+                    db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, transaction);
+
+                Logging.Log.WriteInformationMessage(LOGTAG, "MarkedRemoteFilesForDeletion", "Marked {0} remote files for deletion", missing.Count);
+
+                // Drop all content from tables
+                db.RemoveMissingBlocks(missing.Select(x => x.Name), transaction);
                 brokensets = db.GetBrokenFilesets(options.Time, options.Version, transaction).ToArray();
             }
 
             return brokensets;
         }
 
-        private void DoRun(Database.LocalListBrokenFilesDatabase db, System.Data.IDbTransaction transaction, Library.Utility.IFilter filter, Func<long, DateTime, long, string, long, bool> callbackhandler)
+        private void DoRun(IBackendManager backendManager, Database.LocalListBrokenFilesDatabase db, System.Data.IDbTransaction transaction, Library.Utility.IFilter filter, Func<long, DateTime, long, string, long, bool> callbackhandler)
         {
             if (filter != null && !filter.Empty)
                 throw new UserInformationException("Filters are not supported for this operation", "FiltersAreNotSupportedForListBrokenFiles");
@@ -103,7 +99,7 @@ namespace Duplicati.Library.Main.Operation
                 throw new UserInformationException("The command does not work on partially recreated databases", "ListBrokenFilesDoesNotWorkOnPartialDatabase");
 
             List<Database.RemoteVolumeEntry> missing;
-            var brokensets = GetBrokenFilesetsFromRemote(m_backendurl, m_result, db, transaction, m_options, out missing);
+            var brokensets = GetBrokenFilesetsFromRemote(backendManager, m_result, db, transaction, m_options, out missing);
             if (brokensets == null)
                 return;
 
@@ -123,8 +119,8 @@ namespace Duplicati.Library.Main.Operation
 
             var fstimes = db.FilesetTimes.ToList();
 
-            var brokenfilesets = 
-                brokensets.Select(x => new 
+            var brokenfilesets =
+                brokensets.Select(x => new
                 {
                     Version = fstimes.FindIndex(y => y.Key == x.Item2),
                     Timestamp = x.Item1,
@@ -138,8 +134,8 @@ namespace Duplicati.Library.Main.Operation
             m_result.BrokenFiles =
                 brokenfilesets.Select(
                     x => new Tuple<long, DateTime, IEnumerable<Tuple<string, long>>>(
-                                x.Version, 
-                                x.Timestamp, 
+                                x.Version,
+                                x.Timestamp,
                                 callbackhandler == null && !m_options.ListSetsOnly
                                     ? db.GetBrokenFilenames(x.FilesetID, transaction).ToArray().AsEnumerable()
                                     : new MockList<Tuple<string, long>>((int)x.BrokenCount)
@@ -169,7 +165,7 @@ namespace Duplicati.Library.Main.Operation
 
             public int Count { get; private set; }
 
-            public bool IsReadOnly { get { return true; }}
+            public bool IsReadOnly => true;
 
             public void Add(T item)
             {

Duplicati/Library/Main/Operation/ListChangesHandler.cs

@@ -1,27 +1,28 @@
-// Copyright (C) 2025, The Duplicati Team
-// https://duplicati.com, hello@duplicati.com
-// 
-// Permission is hereby granted, free of charge, to any person obtaining a 
-// copy of this software and associated documentation files (the "Software"), 
-// to deal in the Software without restriction, including without limitation 
-// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
-// and/or sell copies of the Software, and to permit persons to whom the 
-// Software is furnished to do so, subject to the following conditions:
-// 
-// The above copyright notice and this permission notice shall be included in 
-// all copies or substantial portions of the Software.
-// 
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+// Copyright (C) 2025, The Duplicati Team
+// https://duplicati.com, hello@duplicati.com
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a 
+// copy of this software and associated documentation files (the "Software"), 
+// to deal in the Software without restriction, including without limitation 
+// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+// and/or sell copies of the Software, and to permit persons to whom the 
+// Software is furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in 
+// all copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
 // DEALINGS IN THE SOFTWARE.
 
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading;
 using Duplicati.Library.Interface;
 using Duplicati.Library.Utility;
 
@@ -34,13 +35,11 @@ namespace Duplicati.Library.Main.Operation
         /// </summary>
         private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(ListChangesHandler));
 
-        private readonly string m_backendurl;
         private readonly Options m_options;
         private readonly ListChangesResults m_result;
 
-        public ListChangesHandler(string backend, Options options, ListChangesResults result)
+        public ListChangesHandler(Options options, ListChangesResults result)
         {
-            m_backendurl = backend;
             m_options = options;
             m_result = result;
         }
@@ -65,8 +64,9 @@ namespace Duplicati.Library.Main.Operation
             return res;
         }
 
-        public void Run(string baseVersion, string compareVersion, IEnumerable<string> filterstrings = null, Library.Utility.IFilter compositefilter = null, Action<IListChangesResults, IEnumerable<Tuple<Library.Interface.ListChangesChangeType, Library.Interface.ListChangesElementType, string>>> callback = null)
+        public void Run(string baseVersion, string compareVersion, IBackendManager backendManager, IEnumerable<string> filterstrings, IFilter compositefilter, Action<IListChangesResults, IEnumerable<Tuple<ListChangesChangeType, ListChangesElementType, string>>> callback)
         {
+            var cancellationToken = CancellationToken.None;
             var filter = Library.Utility.JoinedFilterExpression.Join(new Library.Utility.FilterExpression(filterstrings), compositefilter);
 
             var useLocalDb = !m_options.NoLocalDb && System.IO.File.Exists(m_options.Dbpath);
@@ -81,7 +81,6 @@ namespace Duplicati.Library.Main.Operation
 
             using (var tmpdb = useLocalDb ? null : new Library.Utility.TempFile())
             using (var db = new Database.LocalListChangesDatabase(useLocalDb ? m_options.Dbpath : (string)tmpdb))
-            using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db))
             using (var storageKeeper = db.CreateStorageHelper())
             {
                 m_result.SetDatabase(db);
@@ -107,7 +106,7 @@ namespace Duplicati.Library.Main.Operation
                 {
                     Logging.Log.WriteInformationMessage(LOGTAG, "NoLocalDatabase", "No local database, accessing remote store");
 
-                    var parsedlist = (from n in backend.List()
+                    var parsedlist = (from n in backendManager.ListAsync(cancellationToken).Await()
                                       let p = Volumes.VolumeBase.ParseFilename(n)
                                       where p != null && p.FileType == RemoteVolumeType.Files
                                       orderby p.Time descending
@@ -124,37 +123,37 @@ namespace Duplicati.Library.Main.Operation
                     numberedList.Remove(bt);
                     SelectTime(compareVersion, numberedList, out compareVersionIndex, out compareVersionTime, out compareFile);
 
-                    Func<FilelistEntryType, Library.Interface.ListChangesElementType> conv = (x) =>
+                    Func<FilelistEntryType, ListChangesElementType> conv = (x) =>
                     {
                         switch (x)
                         {
                             case FilelistEntryType.File:
-                                return Library.Interface.ListChangesElementType.File;
+                                return ListChangesElementType.File;
                             case FilelistEntryType.Folder:
-                                return Library.Interface.ListChangesElementType.Folder;
+                                return ListChangesElementType.Folder;
                             case FilelistEntryType.Symlink:
-                                return Library.Interface.ListChangesElementType.Symlink;
+                                return ListChangesElementType.Symlink;
                             default:
-                                return (Library.Interface.ListChangesElementType)(-1);
+                                return (ListChangesElementType)(-1);
                         }
                     };
 
                     if (!m_result.TaskControl.ProgressRendevouz().Await())
                         return;
 
-                    using (var tmpfile = backend.Get(baseFile.File.Name, baseFile.File.Size, null))
+                    using (var tmpfile = backendManager.GetAsync(baseFile.File.Name, null, baseFile.File.Size, cancellationToken).Await())
                     using (var rd = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(baseFile.File.Name), tmpfile, m_options))
                         foreach (var f in rd.Files)
-                            if (Library.Utility.FilterExpression.Matches(filter, f.Path))
+                            if (FilterExpression.Matches(filter, f.Path))
                                 storageKeeper.AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), false);
 
                     if (!m_result.TaskControl.ProgressRendevouz().Await())
                         return;
 
-                    using (var tmpfile = backend.Get(compareFile.File.Name, compareFile.File.Size, null))
+                    using (var tmpfile = backendManager.GetAsync(compareFile.File.Name, null, compareFile.File.Size, cancellationToken).Await())
                     using (var rd = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(compareFile.File.Name), tmpfile, m_options))
                         foreach (var f in rd.Files)
-                            if (Library.Utility.FilterExpression.Matches(filter, f.Path))
+                            if (FilterExpression.Matches(filter, f.Path))
                                 storageKeeper.AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), true);
                 }
 

Duplicati/Library/Main/Operation/ListControlFilesHandler.cs

@@ -1,58 +1,56 @@
-// Copyright (C) 2025, The Duplicati Team
-// https://duplicati.com, hello@duplicati.com
-// 
-// Permission is hereby granted, free of charge, to any person obtaining a 
-// copy of this software and associated documentation files (the "Software"), 
-// to deal in the Software without restriction, including without limitation 
-// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
-// and/or sell copies of the Software, and to permit persons to whom the 
-// Software is furnished to do so, subject to the following conditions:
-// 
-// The above copyright notice and this permission notice shall be included in 
-// all copies or substantial portions of the Software.
-// 
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+// Copyright (C) 2025, The Duplicati Team
+// https://duplicati.com, hello@duplicati.com
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a 
+// copy of this software and associated documentation files (the "Software"), 
+// to deal in the Software without restriction, including without limitation 
+// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+// and/or sell copies of the Software, and to permit persons to whom the 
+// Software is furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in 
+// all copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
 // DEALINGS IN THE SOFTWARE.
 
 using System;
-using System.Linq;
 using System.Collections.Generic;
 using Duplicati.Library.Main.Database;
 using Duplicati.Library.Utility;
+using System.Threading;
 
 namespace Duplicati.Library.Main.Operation
 {
     internal class ListControlFilesHandler
     {
         private readonly Options m_options;
-        private readonly string m_backendurl;
         private readonly ListResults m_result;
 
-        public ListControlFilesHandler(string backendurl, Options options, ListResults result)
+        public ListControlFilesHandler(Options options, ListResults result)
         {
             m_options = options;
-            m_backendurl = backendurl;
             m_result = result;
         }
 
-        public void Run(IEnumerable<string> filterstrings = null, Library.Utility.IFilter compositefilter = null)
+        public void Run(IBackendManager backendManager, IEnumerable<string> filterstrings, Library.Utility.IFilter compositefilter)
         {
-            using (var tmpdb = new Library.Utility.TempFile())
-            using (var db = new Database.LocalDatabase(System.IO.File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "ListControlFiles", true))
-            using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db))
+            var cancellationToken = CancellationToken.None;
+            using (var tmpdb = new TempFile())
+            using (var db = new LocalDatabase(System.IO.File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "ListControlFiles", true))
             {
                 m_result.SetDatabase(db);
 
-                var filter = Library.Utility.JoinedFilterExpression.Join(new Library.Utility.FilterExpression(filterstrings), compositefilter);
+                var filter = JoinedFilterExpression.Join(new Library.Utility.FilterExpression(filterstrings), compositefilter);
 
                 try
                 {
-                    var filteredList = ListFilesHandler.ParseAndFilterFilesets(backend.List(), m_options);
+                    var filteredList = ListFilesHandler.ParseAndFilterFilesets(backendManager.ListAsync(cancellationToken).Await(), m_options);
                     if (filteredList.Count == 0)
                         throw new Exception("No filesets found on remote target");
 
@@ -68,7 +66,7 @@ namespace Duplicati.Library.Main.Operation
                             var entry = db.GetRemoteVolume(file.Name);
 
                             var files = new List<Library.Interface.IListResultFile>();
-                            using (var tmpfile = backend.Get(file.Name, entry.Size < 0 ? file.Size : entry.Size, entry.Hash))
+                            using (var tmpfile = backendManager.GetAsync(file.Name, entry.Hash, entry.Size < 0 ? file.Size : entry.Size, cancellationToken).Await())
                             using (var tmp = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(file.Name), tmpfile, m_options))
                                 foreach (var cf in tmp.ControlFiles)
                                     if (Library.Utility.FilterExpression.Matches(filter, cf.Key))
@@ -90,7 +88,7 @@ namespace Duplicati.Library.Main.Operation
                 }
                 finally
                 {
-                    backend.WaitForComplete(db, null);
+                    backendManager.WaitForEmptyAsync(db, null, cancellationToken).Await();
                 }
             }
         }

Duplicati/Library/Main/Operation/ListFilesHandler.cs

@@ -1,27 +1,27 @@
-// Copyright (C) 2025, The Duplicati Team
-// https://duplicati.com, hello@duplicati.com
-// 
-// Permission is hereby granted, free of charge, to any person obtaining a 
-// copy of this software and associated documentation files (the "Software"), 
-// to deal in the Software without restriction, including without limitation 
-// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
-// and/or sell copies of the Software, and to permit persons to whom the 
-// Software is furnished to do so, subject to the following conditions:
-// 
-// The above copyright notice and this permission notice shall be included in 
-// all copies or substantial portions of the Software.
-// 
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+// Copyright (C) 2025, The Duplicati Team
+// https://duplicati.com, hello@duplicati.com
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a 
+// copy of this software and associated documentation files (the "Software"), 
+// to deal in the Software without restriction, including without limitation 
+// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+// and/or sell copies of the Software, and to permit persons to whom the 
+// Software is furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in 
+// all copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
 // DEALINGS IN THE SOFTWARE.
-using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
 using Duplicati.Library.Interface;
 using Duplicati.Library.Main.Database;
 using Duplicati.Library.Main.Volumes;
@@ -36,22 +36,21 @@ namespace Duplicati.Library.Main.Operation
         /// </summary>
         private static readonly string LOGTAG = Logging.Log.LogTagFromType<ListFilesHandler>();
 
-        private readonly string m_backendurl;
         private readonly Options m_options;
         private readonly ListResults m_result;
 
-        public ListFilesHandler(string backend, Options options, ListResults result)
+        public ListFilesHandler(Options options, ListResults result)
         {
-            m_backendurl = backend;
             m_options = options;
             m_result = result;
         }
 
-        public void Run(IEnumerable<string> filterstrings = null, Library.Utility.IFilter compositefilter = null)
+        public async Task Run(IBackendManager backendManager, IEnumerable<string> filterstrings, IFilter compositefilter)
         {
-            var parsedfilter = new Library.Utility.FilterExpression(filterstrings);
-            var filter = Library.Utility.JoinedFilterExpression.Join(parsedfilter, compositefilter);
-            var simpleList = !((filter is FilterExpression expression && expression.Type == Library.Utility.FilterType.Simple) || m_options.AllVersions);
+            var cancellationToken = m_result.TaskControl.ProgressToken;
+            var parsedfilter = new FilterExpression(filterstrings);
+            var filter = JoinedFilterExpression.Join(parsedfilter, compositefilter);
+            var simpleList = !((filter is FilterExpression expression && expression.Type == FilterType.Simple) || m_options.AllVersions);
 
             //Use a speedy local query
             if (!m_options.NoLocalDb && System.IO.File.Exists(m_options.Dbpath))
@@ -120,17 +119,16 @@ namespace Duplicati.Library.Main.Operation
                 throw new UserInformationException("Listing prefixes is not supported without a local database, consider using the \"repair\" option to rebuild the database.", "PrefixListingRequiresLocalDatabase");
 
             // Otherwise, grab info from remote location
-            using (var tmpdb = new Library.Utility.TempFile())
-            using (var db = new Database.LocalDatabase(tmpdb, "List", true))
-            using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db))
+            using (var tmpdb = new TempFile())
+            using (var db = new LocalDatabase(tmpdb, "List", true))
             {
                 m_result.SetDatabase(db);
 
-                var filteredList = ParseAndFilterFilesets(backend.List(), m_options);
+                var filteredList = ParseAndFilterFilesets(await backendManager.ListAsync(cancellationToken).ConfigureAwait(false), m_options);
                 if (filteredList.Count == 0)
                     throw new UserInformationException("No filesets found on remote target", "EmptyRemoteFolder");
 
-                var numberSeq = CreateResultSequence(filteredList, backend, m_options);
+                var numberSeq = await CreateResultSequence(filteredList, backendManager, m_options, cancellationToken);
                 if (filter.Empty)
                 {
                     m_result.SetResult(numberSeq, null);
@@ -142,11 +140,11 @@ namespace Duplicati.Library.Main.Operation
                 filteredList.RemoveAt(0);
                 Dictionary<string, List<long>> res;
 
-                if (!m_result.TaskControl.ProgressRendevouz().Await())
+                if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
                     return;
 
-                using (var tmpfile = backend.Get(firstEntry.File.Name, firstEntry.File.Size, null))
-                using (var rd = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(firstEntry.File.Name), tmpfile, m_options))
+                using (var tmpfile = await backendManager.GetAsync(firstEntry.File.Name, null, firstEntry.File.Size, cancellationToken).ConfigureAwait(false))
+                using (var rd = new FilesetVolumeReader(RestoreHandler.GetCompressionModule(firstEntry.File.Name), tmpfile, m_options))
                     if (simpleList)
                     {
                         m_result.SetResult(
@@ -177,11 +175,14 @@ namespace Duplicati.Library.Main.Operation
                     }
 
                 long flindex = 1;
-                foreach (var flentry in filteredList)
-                    using (var tmpfile = backend.Get(flentry.Value.File.Name, flentry.Value.File == null ? -1 : flentry.Value.File.Size, null))
-                    using (var rd = new Volumes.FilesetVolumeReader(flentry.Value.CompressionModule, tmpfile, m_options))
+                var filteredListMap = filteredList.ToDictionary(x => x.Value.File.Name, x => x.Value);
+                await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(filteredList.Select(x => new RemoteVolumeMapper(x.Value)), cancellationToken))
+                {
+                    var flentry = filteredListMap[name];
+                    using (tmpfile)
+                    using (var rd = new FilesetVolumeReader(flentry.CompressionModule, tmpfile, m_options))
                     {
-                        if (!m_result.TaskControl.ProgressRendevouz().Await())
+                        if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
                             return;
 
                         foreach (var p in from n in rd.Files where Library.Utility.FilterExpression.Matches(filter, n.Path) select n)
@@ -203,6 +204,7 @@ namespace Duplicati.Library.Main.Operation
 
                         flindex++;
                     }
+                }
 
                 m_result.SetResult(
                     numberSeq,
@@ -224,22 +226,30 @@ namespace Duplicati.Library.Main.Operation
             return filelistFilter(parsedlist).ToList();
         }
 
-        private static IEnumerable<IListResultFileset> CreateResultSequence(IEnumerable<KeyValuePair<long, IParsedVolume>> filteredList, BackendManager backendManager, Options options)
+        private class RemoteVolumeMapper(IParsedVolume Volume) : IRemoteVolume
         {
-            List<IListResultFileset> list = new List<IListResultFileset>();
-            foreach (KeyValuePair<long, IParsedVolume> entry in filteredList)
+            public string Name => Volume.File.Name;
+            public string Hash => null;
+            public long Size => Volume.File.Size;
+        }
+
+        private static async Task<IEnumerable<IListResultFileset>> CreateResultSequence(IEnumerable<KeyValuePair<long, IParsedVolume>> filteredList, IBackendManager backendManager, Options options, CancellationToken cancelToken)
+        {
+            var list = new List<IListResultFileset>();
+            var map = filteredList.ToDictionary(x => x.Value.File.Name, x => x);
+            await foreach (var (file, hash, size, name) in backendManager.GetFilesOverlappedAsync(filteredList.Select(x => new RemoteVolumeMapper(x.Value)), cancelToken))
             {
-                AsyncDownloader downloader = new AsyncDownloader(new IRemoteVolume[] { new RemoteVolume(entry.Value.File) }, backendManager);
-                foreach (IAsyncDownloadedFile file in downloader)
+                // We must obtain the partial/full status from the fileset file in the dlist files.
+                // Without this, the restore dialog will show all versions as full, or all versions
+                // as partial. While the dlist files are already downloaded elsewhere, doing so again
+                // here is the most direct way to obtain the partial/full status without a major
+                // refactoring. Since restoring directly from the backend files should be a relatively
+                // rare event, we can work on improving the performance later.
+                using (file)
                 {
-                    // We must obtain the partial/full status from the fileset file in the dlist files.
-                    // Without this, the restore dialog will show all versions as full, or all versions
-                    // as partial. While the dlist files are already downloaded elsewhere, doing so again
-                    // here is the most direct way to obtain the partial/full status without a major
-                    // refactoring. Since restoring directly from the backend files should be a relatively
-                    // rare event, we can work on improving the performance later.
-                    VolumeBase.FilesetData filesetData = VolumeReaderBase.GetFilesetData(entry.Value.CompressionModule, file.TempFile, options);
-                    list.Add(new ListResultFileset(entry.Key, filesetData.IsFullBackup ? BackupType.FULL_BACKUP : BackupType.PARTIAL_BACKUP, entry.Value.Time.ToLocalTime(), -1, -1));
+                    var ent = map[name];
+                    var filesetData = VolumeReaderBase.GetFilesetData(ent.Value.CompressionModule, file, options);
+                    list.Add(new ListResultFileset(ent.Key, filesetData.IsFullBackup ? BackupType.FULL_BACKUP : BackupType.PARTIAL_BACKUP, ent.Value.Time.ToLocalTime(), -1, -1));
                 }
             }
 

Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs

@@ -32,18 +32,16 @@ namespace Duplicati.Library.Main.Operation
         /// The tag used for logging
         /// </summary>
         private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(PurgeBrokenFilesHandler));
-        protected readonly string m_backendurl;
         protected readonly Options m_options;
         protected readonly PurgeBrokenFilesResults m_result;
 
-        public PurgeBrokenFilesHandler(string backend, Options options, PurgeBrokenFilesResults result)
+        public PurgeBrokenFilesHandler(Options options, PurgeBrokenFilesResults result)
         {
-            m_backendurl = backend;
             m_options = options;
             m_result = result;
         }
 
-        public void Run(Library.Utility.IFilter filter)
+        public void Run(IBackendManager backendManager, Library.Utility.IFilter filter)
         {
             if (!System.IO.File.Exists(m_options.Dbpath))
                 throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist");
@@ -52,17 +50,17 @@ namespace Duplicati.Library.Main.Operation
                 throw new UserInformationException("Filters are not supported for this operation", "FiltersNotAllowedOnPurgeBrokenFiles");
 
             List<Database.RemoteVolumeEntry> missing = null;
-            
+
             using (var db = new Database.LocalListBrokenFilesDatabase(m_options.Dbpath))
             using (var tr = db.BeginTransaction())
             {
                 if (db.PartiallyRecreated)
                     throw new UserInformationException("The command does not work on partially recreated databases", "CannotPurgeOnPartialDatabase");
 
-                var sets = ListBrokenFilesHandler.GetBrokenFilesetsFromRemote(m_backendurl, m_result, db, tr, m_options, out missing);
+                var sets = ListBrokenFilesHandler.GetBrokenFilesetsFromRemote(backendManager, m_result, db, tr, m_options, out missing);
                 if (sets == null)
                     return;
-                
+
                 if (sets.Length == 0)
                 {
                     if (missing == null)
@@ -73,7 +71,7 @@ namespace Duplicati.Library.Main.Operation
                         Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenSetsButMissingRemoteFiles", string.Format("Found no broken filesets, but {0} missing remote files. Purging from database.", missing.Count));
                 }
                 else
-                { 
+                {
                     Logging.Log.WriteInformationMessage(LOGTAG, "FoundBrokenFilesets", "Found {0} broken filesets with {1} affected files, purging files", sets.Length, sets.Sum(x => x.Item3));
 
                     var pgoffset = 0.0f;
@@ -116,8 +114,8 @@ namespace Duplicati.Library.Main.Operation
                                 opts.RawOptions.Remove("time");
                                 opts.RawOptions["no-auto-compact"] = "true";
 
-                                new DeleteHandler(m_backendurl, opts, (DeleteResults)m_result.DeleteResults)
-                                    .DoRun(rmdb, ref deltr, true, false, null);
+                                new DeleteHandler(opts, (DeleteResults)m_result.DeleteResults)
+                                    .DoRun(rmdb, ref deltr, true, false, backendManager);
 
                                 if (!m_options.Dryrun)
                                 {
@@ -163,7 +161,7 @@ namespace Duplicati.Library.Main.Operation
                                 opts.RawOptions.Remove("time");
                                 opts.RawOptions["no-auto-compact"] = "true";
 
-                                new PurgeFilesHandler(m_backendurl, opts, (PurgeFilesResults)m_result.PurgeResults).Run(pgdb, pgoffset, pgspan, (cmd, filesetid, tablename) =>
+                                new PurgeFilesHandler(opts, (PurgeFilesResults)m_result.PurgeResults).Run(backendManager, pgdb, pgoffset, pgspan, (cmd, filesetid, tablename) =>
                                 {
                                     if (filesetid != bs.FilesetID)
                                         throw new Exception(string.Format("Unexpected filesetid: {0}, expected {1}", filesetid, bs.FilesetID));
@@ -180,7 +178,7 @@ namespace Duplicati.Library.Main.Operation
                 m_result.OperationProgressUpdater.UpdateProgress(0.95f);
 
                 if (!m_options.Dryrun && db.RepairInProgress)
-                {                    
+                {
                     Logging.Log.WriteInformationMessage(LOGTAG, "ValidatingDatabase", "Database was previously marked as in-progress, checking if it is valid after purging files");
                     db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, null);
                     Logging.Log.WriteInformationMessage(LOGTAG, "UpdatingDatabase", "Purge completed, and consistency checks completed, marking database as complete");

Duplicati/Library/Main/Operation/PurgeFilesHandler.cs

@@ -1,29 +1,29 @@
-// Copyright (C) 2025, The Duplicati Team
-// https://duplicati.com, hello@duplicati.com
-// 
-// Permission is hereby granted, free of charge, to any person obtaining a 
-// copy of this software and associated documentation files (the "Software"), 
-// to deal in the Software without restriction, including without limitation 
-// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
-// and/or sell copies of the Software, and to permit persons to whom the 
-// Software is furnished to do so, subject to the following conditions:
-// 
-// The above copyright notice and this permission notice shall be included in 
-// all copies or substantial portions of the Software.
-// 
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+// Copyright (C) 2025, The Duplicati Team
+// https://duplicati.com, hello@duplicati.com
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a 
+// copy of this software and associated documentation files (the "Software"), 
+// to deal in the Software without restriction, including without limitation 
+// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+// and/or sell copies of the Software, and to permit persons to whom the 
+// Software is furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in 
+// all copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
 // DEALINGS IN THE SOFTWARE.
 
 using System;
-using System.ComponentModel;
 using System.Linq;
+using System.Threading;
 using Duplicati.Library.Interface;
-using Duplicati.Library.Main.Database;
+using Duplicati.Library.Utility;
 
 namespace Duplicati.Library.Main.Operation
 {
@@ -33,18 +33,16 @@ namespace Duplicati.Library.Main.Operation
         /// The tag used for logging
         /// </summary>
         private static readonly string LOGTAG = Logging.Log.LogTagFromType<PurgeFilesHandler>();
-        protected readonly string m_backendurl;
         protected readonly Options m_options;
         protected readonly PurgeFilesResults m_result;
 
-        public PurgeFilesHandler(string backend, Options options, PurgeFilesResults result)
+        public PurgeFilesHandler(Options options, PurgeFilesResults result)
         {
-            m_backendurl = backend;
             m_options = options;
             m_result = result;
         }
 
-        public void Run(Library.Utility.IFilter filter)
+        public void Run(IBackendManager backendManager, Library.Utility.IFilter filter)
         {
             if (filter == null || filter.Empty)
                 throw new UserInformationException("Cannot purge with an empty filter, as that would cause all files to be removed.\nTo remove an entire backup set, use the \"delete\" command.", "EmptyFilterPurgeNotAllowed");
@@ -53,214 +51,213 @@ namespace Duplicati.Library.Main.Operation
                 throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist");
 
             using (var db = new Database.LocalPurgeDatabase(m_options.Dbpath))
-                DoRun(db, filter, null, 0, 1);
+                DoRun(backendManager, db, filter, null, 0, 1);
         }
 
-        public void Run(Database.LocalPurgeDatabase db, float pgoffset, float pgspan, Action<System.Data.IDbCommand, long, string> filtercommand)
+        public void Run(IBackendManager backendManager, Database.LocalPurgeDatabase db, float pgoffset, float pgspan, Action<System.Data.IDbCommand, long, string> filtercommand)
         {
-            DoRun(db, null, filtercommand, pgoffset, pgspan);
+            DoRun(backendManager, db, null, filtercommand, pgoffset, pgspan);
         }
 
-        private void DoRun(Database.LocalPurgeDatabase db, Library.Utility.IFilter filter, Action<System.Data.IDbCommand, long, string> filtercommand, float pgoffset, float pgspan)
+        private void DoRun(IBackendManager backendManager, Database.LocalPurgeDatabase db, IFilter filter, Action<System.Data.IDbCommand, long, string> filtercommand, float pgoffset, float pgspan)
         {
+            var cancellationToken = CancellationToken.None;
             m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Begin);
             Logging.Log.WriteInformationMessage(LOGTAG, "StartingPurge", "Starting purge operation");
 
             var doCompactStep = !m_options.NoAutoCompact && filtercommand == null;
 
-            using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db))
+            if (db.PartiallyRecreated)
+                throw new UserInformationException("The purge command does not work on partially recreated databases", "PurgeNotAllowedOnPartialDatabase");
+
+            if (db.RepairInProgress && filtercommand == null)
+                throw new UserInformationException(string.Format("The purge command does not work on an incomplete database, try the {0} operation.", "purge-broken-files"), "PurgeNotAllowedOnIncompleteDatabase");
+
+            var versions = db.GetFilesetIDs(m_options.Time, m_options.Version).OrderByDescending(x => x).ToArray();
+            if (versions.Length <= 0)
+                throw new UserInformationException("No filesets matched the supplied time or versions", "NoFilesetFoundForTimeOrVersion");
+
+            var orphans = db.CountOrphanFiles(null);
+            if (orphans != 0)
+                throw new UserInformationException(string.Format("Unable to start the purge process as there are {0} orphan file(s)", orphans), "CannotPurgeWithOrphans");
+
+            Utility.UpdateOptionsFromDb(db, m_options);
+            Utility.VerifyOptionsAndUpdateDatabase(db, m_options);
+
+            if (filtercommand == null)
             {
-                if (db.PartiallyRecreated)
-                    throw new UserInformationException("The purge command does not work on partially recreated databases", "PurgeNotAllowedOnPartialDatabase");
+                db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, false, null);
 
-                if (db.RepairInProgress && filtercommand == null)
-                    throw new UserInformationException(string.Format("The purge command does not work on an incomplete database, try the {0} operation.", "purge-broken-files"), "PurgeNotAllowedOnIncompleteDatabase");
+                if (m_options.NoBackendverification)
+                    FilelistProcessor.VerifyLocalList(backendManager, db, cancellationToken).Await();
+                else
+                    FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, null).Await();
+            }
 
-                var versions = db.GetFilesetIDs(m_options.Time, m_options.Version).OrderByDescending(x => x).ToArray();
-                if (versions.Length <= 0)
-                    throw new UserInformationException("No filesets matched the supplied time or versions", "NoFilesetFoundForTimeOrVersion");
+            var filesets = db.FilesetTimes.OrderByDescending(x => x.Value).ToArray();
 
-                var orphans = db.CountOrphanFiles(null);
-                if (orphans != 0)
-                    throw new UserInformationException(string.Format("Unable to start the purge process as there are {0} orphan file(s)", orphans), "CannotPurgeWithOrphans");
+            var versionprogress = ((doCompactStep ? 0.75f : 1.0f) / versions.Length) * pgspan;
+            var currentprogress = pgoffset;
+            var progress = 0;
 
-                Utility.UpdateOptionsFromDb(db, m_options);
-                Utility.VerifyOptionsAndUpdateDatabase(db, m_options);
+            m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Process);
+            m_result.OperationProgressUpdater.UpdateProgress(currentprogress);
 
-                if (filtercommand == null)
+            // Reverse makes sure we re-write the old versions first
+            foreach (var versionid in versions.Reverse())
+            {
+                progress++;
+                Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingFilelistVolumes", "Processing filelist volume {0} of {1}", progress, versions.Length);
+
+                using (var tr = db.BeginTransaction())
                 {
-                    db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, false, null);
-
-                    if (m_options.NoBackendverification)
-                        FilelistProcessor.VerifyLocalList(backend, db);
-                    else
-                        FilelistProcessor.VerifyRemoteList(backend, m_options, db, m_result.BackendWriter, null);
-                }
-
-                var filesets = db.FilesetTimes.OrderByDescending(x => x.Value).ToArray();
-
-                var versionprogress = ((doCompactStep ? 0.75f : 1.0f) / versions.Length) * pgspan;
-                var currentprogress = pgoffset;
-                var progress = 0;
-
-                m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Process);
-                m_result.OperationProgressUpdater.UpdateProgress(currentprogress);
-
-                // Reverse makes sure we re-write the old versions first
-                foreach (var versionid in versions.Reverse())
-                {
-                    progress++;
-                    Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingFilelistVolumes", "Processing filelist volume {0} of {1}", progress, versions.Length);
-
-                    using (var tr = db.BeginTransaction())
-                    {
-                        var ix = -1;
-                        for (var i = 0; i < filesets.Length; i++)
-                            if (filesets[i].Key == versionid)
-                            {
-                                ix = i;
-                                break;
-                            }
-
-                        if (ix < 0)
-                            throw new InvalidProgramException(string.Format("Fileset was reported with id {0}, but could not be found?", versionid));
-
-                        var secs = 0;
-                        while (secs < 60)
+                    var ix = -1;
+                    for (var i = 0; i < filesets.Length; i++)
+                        if (filesets[i].Key == versionid)
                         {
-                            secs++;
-                            var tfn = Volumes.VolumeBase.GenerateFilename(RemoteVolumeType.Files, m_options, null, filesets[ix].Value.AddSeconds(secs));
-                            if (db.GetRemoteVolumeID(tfn, tr) < 0)
-                                break;
+                            ix = i;
+                            break;
                         }
 
-                        var tsOriginal = filesets[ix].Value;
-                        var ts = tsOriginal.AddSeconds(secs);
+                    if (ix < 0)
+                        throw new InvalidProgramException(string.Format("Fileset was reported with id {0}, but could not be found?", versionid));
 
-                        var prevfilename = db.GetRemoteVolumeNameForFileset(filesets[ix].Key, tr);
+                    var secs = 0;
+                    while (secs < 60)
+                    {
+                        secs++;
+                        var tfn = Volumes.VolumeBase.GenerateFilename(RemoteVolumeType.Files, m_options, null, filesets[ix].Value.AddSeconds(secs));
+                        if (db.GetRemoteVolumeID(tfn, tr) < 0)
+                            break;
+                    }
 
-                        if (secs >= 60)
-                            throw new Exception(string.Format("Unable to create a new fileset for {0} because the resulting timestamp {1} is more than 60 seconds away", prevfilename, ts));
+                    var tsOriginal = filesets[ix].Value;
+                    var ts = tsOriginal.AddSeconds(secs);
 
-                        if (ix != 0 && filesets[ix - 1].Value <= ts)
-                            throw new Exception(string.Format("Unable to create a new fileset for {0} because the resulting timestamp {1} is larger than the next timestamp {2}", prevfilename, ts, filesets[ix - 1].Value));
+                    var prevfilename = db.GetRemoteVolumeNameForFileset(filesets[ix].Key, tr);
 
-                        using (var tempset = db.CreateTemporaryFileset(versionid, tr))
+                    if (secs >= 60)
+                        throw new Exception(string.Format("Unable to create a new fileset for {0} because the resulting timestamp {1} is more than 60 seconds away", prevfilename, ts));
+
+                    if (ix != 0 && filesets[ix - 1].Value <= ts)
+                        throw new Exception(string.Format("Unable to create a new fileset for {0} because the resulting timestamp {1} is larger than the next timestamp {2}", prevfilename, ts, filesets[ix - 1].Value));
+
+                    using (var tempset = db.CreateTemporaryFileset(versionid, tr))
+                    {
+                        if (filtercommand == null)
+                            tempset.ApplyFilter(filter);
+                        else
+                            tempset.ApplyFilter(filtercommand);
+
+                        if (tempset.RemovedFileCount == 0)
                         {
-                            if (filtercommand == null)
-                                tempset.ApplyFilter(filter);
-                            else
-                                tempset.ApplyFilter(filtercommand);
+                            Logging.Log.WriteInformationMessage(LOGTAG, "NotWritingNewFileset", "Not writing a new fileset for {0} as it was not changed", prevfilename);
+                            currentprogress += versionprogress;
+                            tr.Rollback();
+                            continue;
+                        }
+                        else
+                        {
+                            using (var tf = new Library.Utility.TempFile())
+                            using (var vol = new Volumes.FilesetVolumeWriter(m_options, ts))
+                            {
+                                var isOriginalFilesetFullBackup = db.IsFilesetFullBackup(tsOriginal);
+                                var newids = tempset.ConvertToPermanentFileset(vol.RemoteFilename, ts, isOriginalFilesetFullBackup);
+                                vol.VolumeID = newids.Item1;
+                                vol.CreateFilesetFile(isOriginalFilesetFullBackup);
 
-                            if (tempset.RemovedFileCount == 0)
-                            {
-                                Logging.Log.WriteInformationMessage(LOGTAG, "NotWritingNewFileset", "Not writing a new fileset for {0} as it was not changed", prevfilename);
-                                currentprogress += versionprogress;
-                                tr.Rollback();
-                                continue;
-                            }
-                            else
-                            {
-                                using (var tf = new Library.Utility.TempFile())
-                                using (var vol = new Volumes.FilesetVolumeWriter(m_options, ts))
+                                Logging.Log.WriteInformationMessage(LOGTAG, "ReplacingFileset", "Replacing fileset {0} with {1} which has with {2} fewer file(s) ({3} reduction)", prevfilename, vol.RemoteFilename, tempset.RemovedFileCount, Library.Utility.Utility.FormatSizeString(tempset.RemovedFileSize));
+
+                                db.WriteFileset(vol, newids.Item2, tr);
+
+                                m_result.RemovedFileSize += tempset.RemovedFileSize;
+                                m_result.RemovedFileCount += tempset.RemovedFileCount;
+                                m_result.RewrittenFileLists++;
+
+                                currentprogress += (versionprogress / 2);
+                                m_result.OperationProgressUpdater.UpdateProgress(currentprogress);
+
+                                if (m_options.Dryrun || m_options.FullResult)
                                 {
-                                    var isOriginalFilesetFullBackup = db.IsFilesetFullBackup(tsOriginal);
-                                    var newids = tempset.ConvertToPermanentFileset(vol.RemoteFilename, ts, isOriginalFilesetFullBackup);
-                                    vol.VolumeID = newids.Item1;
-                                    vol.CreateFilesetFile(isOriginalFilesetFullBackup);
-
-                                    Logging.Log.WriteInformationMessage(LOGTAG, "ReplacingFileset", "Replacing fileset {0} with {1} which has with {2} fewer file(s) ({3} reduction)", prevfilename, vol.RemoteFilename, tempset.RemovedFileCount, Library.Utility.Utility.FormatSizeString(tempset.RemovedFileSize));
-
-                                    db.WriteFileset(vol, newids.Item2, tr);
-
-                                    m_result.RemovedFileSize += tempset.RemovedFileSize;
-                                    m_result.RemovedFileCount += tempset.RemovedFileCount;
-                                    m_result.RewrittenFileLists++;
-
-                                    currentprogress += (versionprogress / 2);
-                                    m_result.OperationProgressUpdater.UpdateProgress(currentprogress);
-
-                                    if (m_options.Dryrun || m_options.FullResult)
+                                    foreach (var fe in tempset.ListAllDeletedFiles())
                                     {
-                                        foreach (var fe in tempset.ListAllDeletedFiles())
-                                        {
-                                            var msg = string.Format("  Purging file {0} ({1})", fe.Key, Library.Utility.Utility.FormatSizeString(fe.Value));
+                                        var msg = string.Format("  Purging file {0} ({1})", fe.Key, Library.Utility.Utility.FormatSizeString(fe.Value));
 
-                                            Logging.Log.WriteProfilingMessage(LOGTAG, "PurgeFile", msg);
-                                            Logging.Log.WriteVerboseMessage(LOGTAG, "PurgeFile", msg);
-
-                                            if (m_options.Dryrun)
-                                                Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPurgeFile", msg);
-                                        }
+                                        Logging.Log.WriteProfilingMessage(LOGTAG, "PurgeFile", msg);
+                                        Logging.Log.WriteVerboseMessage(LOGTAG, "PurgeFile", msg);
 
                                         if (m_options.Dryrun)
-                                            Logging.Log.WriteDryrunMessage(LOGTAG, "WouldWriteRemoteFiles", "Would write files to remote storage");
-
-                                        Logging.Log.WriteVerboseMessage(LOGTAG, "WritingRemoteFiles", "Writing files to remote storage");
+                                            Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPurgeFile", msg);
                                     }
 
                                     if (m_options.Dryrun)
-                                    {
-                                        Logging.Log.WriteDryrunMessage(LOGTAG, "WouldUploadAndDelete", "Would upload file {0} ({1}) and delete file {2}, removing {3} files", vol.RemoteFilename, Library.Utility.Utility.FormatSizeString(vol.Filesize), prevfilename, tempset.RemovedFileCount);
-                                        tr.Rollback();
-                                    }
-                                    else
-                                    {
-                                        var lst = db.DropFilesetsFromTable(new[] { filesets[ix].Value }, tr).ToArray();
-                                        foreach (var f in lst)
-                                            db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, tr);
+                                        Logging.Log.WriteDryrunMessage(LOGTAG, "WouldWriteRemoteFiles", "Would write files to remote storage");
 
-                                        tr.Commit();
-                                        backend.Put(vol, synchronous: true);
-                                        backend.Delete(prevfilename, -1, true);
-                                        backend.FlushDbMessages();
-                                    }
+                                    Logging.Log.WriteVerboseMessage(LOGTAG, "WritingRemoteFiles", "Writing files to remote storage");
+                                }
+
+                                if (m_options.Dryrun)
+                                {
+                                    Logging.Log.WriteDryrunMessage(LOGTAG, "WouldUploadAndDelete", "Would upload file {0} ({1}) and delete file {2}, removing {3} files", vol.RemoteFilename, Library.Utility.Utility.FormatSizeString(vol.Filesize), prevfilename, tempset.RemovedFileCount);
+                                    tr.Rollback();
+                                }
+                                else
+                                {
+                                    var lst = db.DropFilesetsFromTable(new[] { filesets[ix].Value }, tr).ToArray();
+                                    foreach (var f in lst)
+                                        db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, tr);
+
+                                    tr.Commit();
+                                    backendManager.PutAsync(vol, null, null, true, cancellationToken).Await();
+                                    backendManager.DeleteAsync(prevfilename, -1, true, cancellationToken).Await();
+                                    backendManager.WaitForEmptyAsync(db, null, cancellationToken).Await();
                                 }
                             }
                         }
                     }
-
-                    currentprogress += (versionprogress / 2);
-                    m_result.OperationProgressUpdater.UpdateProgress(currentprogress);
                 }
 
+                currentprogress += (versionprogress / 2);
+                m_result.OperationProgressUpdater.UpdateProgress(currentprogress);
+            }
 
-                if (doCompactStep)
+
+            if (doCompactStep)
+            {
+                if (m_result.RewrittenFileLists == 0)
                 {
-                    if (m_result.RewrittenFileLists == 0)
+                    Logging.Log.WriteInformationMessage(LOGTAG, "SkippingCompacting", "Skipping compacting as no new volumes were written");
+                }
+                else
+                {
+                    m_result.OperationProgressUpdater.UpdateProgress(pgoffset + (0.75f * pgspan));
+                    m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Compact);
+                    m_result.CompactResults = new CompactResults(m_result);
+                    using (var cdb = new Database.LocalDeleteDatabase(db))
                     {
-                        Logging.Log.WriteInformationMessage(LOGTAG, "SkippingCompacting", "Skipping compacting as no new volumes were written");
-                    }
-                    else
-                    {
-                        m_result.OperationProgressUpdater.UpdateProgress(pgoffset + (0.75f * pgspan));
-                        m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Compact);
-                        m_result.CompactResults = new CompactResults(m_result);
-                        using (var cdb = new Database.LocalDeleteDatabase(db))
+                        var tr = cdb.BeginTransaction();
+                        try
                         {
-                            var tr = cdb.BeginTransaction();
-                            try
-                            {
-                                new CompactHandler(backend.BackendUrl, m_options, (CompactResults)m_result.CompactResults).DoCompact(cdb, true, ref tr, backend);
-                            }
-                            catch
-                            {
-                                try { tr.Rollback(); }
-                                catch { }
-                            }
-                            finally
-                            {
-                                try { tr.Commit(); }
-                                catch { }
-                            }
+                            var (_, tr2) = new CompactHandler(m_options, (CompactResults)m_result.CompactResults).DoCompact(cdb, true, tr, backendManager).Await();
+                            tr2 = tr;
+                        }
+                        catch
+                        {
+                            try { tr.Rollback(); }
+                            catch { }
+                        }
+                        finally
+                        {
+                            try { tr.Commit(); }
+                            catch { }
                         }
                     }
-
-                    m_result.OperationProgressUpdater.UpdateProgress(pgoffset + pgspan);
-                    m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Complete);
                 }
 
-                backend.WaitForComplete(db, null);
+                m_result.OperationProgressUpdater.UpdateProgress(pgoffset + pgspan);
+                m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Complete);
+
+                backendManager.WaitForEmptyAsync(db, null, cancellationToken).Await();
             }
         }
     }

Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs

@@ -21,6 +21,8 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
 using Duplicati.Library.Interface;
 using Duplicati.Library.Main.Database;
 using Duplicati.Library.Main.Volumes;
@@ -35,17 +37,15 @@ namespace Duplicati.Library.Main.Operation
         /// </summary>
         private static readonly string LOGTAG = Logging.Log.LogTagFromType<RecreateDatabaseHandler>();
 
-        private readonly string m_backendurl;
         private readonly Options m_options;
         private readonly RecreateDatabaseResults m_result;
 
         public delegate IEnumerable<KeyValuePair<long, IParsedVolume>> NumberedFilterFilelistDelegate(IEnumerable<IParsedVolume> filelist);
         public delegate void BlockVolumePostProcessor(string volumename, BlockVolumeReader reader);
 
-        public RecreateDatabaseHandler(string backendurl, Options options, RecreateDatabaseResults result)
+        public RecreateDatabaseHandler(Options options, RecreateDatabaseResults result)
         {
             m_options = options;
-            m_backendurl = backendurl;
             m_result = result;
         }
 
@@ -53,10 +53,11 @@ namespace Duplicati.Library.Main.Operation
         /// Run the recreate procedure
         /// </summary>
         /// <param name="path">Path to the database that will be created</param>
+        /// <param name="backendManager">The backend manager to use for downloading files</param>
         /// <param name="filelistfilter">A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist</param>
         /// <param name="filter">Filters the files in a filelist to prevent downloading unwanted data</param>
         /// <param name="blockprocessor">A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists</param>
-        public void Run(string path, Library.Utility.IFilter filter = null, NumberedFilterFilelistDelegate filelistfilter = null, BlockVolumePostProcessor blockprocessor = null)
+        public void Run(string path, IBackendManager backendManager, IFilter filter, NumberedFilterFilelistDelegate filelistfilter, BlockVolumePostProcessor blockprocessor)
         {
             if (System.IO.File.Exists(path))
                 throw new UserInformationException(string.Format("Cannot recreate database because file already exists: {0}", path), "RecreateTargetDatabaseExists");
@@ -64,7 +65,7 @@ namespace Duplicati.Library.Main.Operation
             using (var db = new LocalDatabase(path, "Recreate", true))
             {
                 m_result.SetDatabase(db);
-                DoRun(db, false, filter, filelistfilter, blockprocessor);
+                DoRun(backendManager, db, false, filter, filelistfilter, blockprocessor).Await();
                 db.WriteResults();
             }
         }
@@ -75,7 +76,7 @@ namespace Duplicati.Library.Main.Operation
         /// <param name="filelistfilter">A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist</param>
         /// <param name="filter">Filters the files in a filelist to prevent downloading unwanted data</param>
         /// <param name="blockprocessor">A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists</param>
-        public void RunUpdate(Library.Utility.IFilter filter = null, NumberedFilterFilelistDelegate filelistfilter = null, BlockVolumePostProcessor blockprocessor = null)
+        public void RunUpdate(IBackendManager backendManager, Library.Utility.IFilter filter, NumberedFilterFilelistDelegate filelistfilter, BlockVolumePostProcessor blockprocessor)
         {
             if (!m_options.RepairOnlyPaths)
                 throw new UserInformationException(string.Format("Can only update with paths, try setting --{0}", "repair-only-paths"), "RepairUpdateRequiresPathsOnly");
@@ -97,7 +98,7 @@ namespace Duplicati.Library.Main.Operation
                 if (preexistingOptionsInDatabase)
                     Utility.VerifyOptionsAndUpdateDatabase(db, m_options, null);
 
-                DoRun(db, true, filter, filelistfilter, blockprocessor);
+                DoRun(backendManager, db, true, filter, filelistfilter, blockprocessor).Await();
                 db.WriteResults();
             }
         }
@@ -110,13 +111,13 @@ namespace Duplicati.Library.Main.Operation
         /// <param name="filter">A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist</param>
         /// <param name="filelistfilter">Filters the files in a filelist to prevent downloading unwanted data</param>
         /// <param name="blockprocessor">A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists</param>
-        internal void DoRun(LocalDatabase dbparent, bool updating, Library.Utility.IFilter filter = null, NumberedFilterFilelistDelegate filelistfilter = null, BlockVolumePostProcessor blockprocessor = null)
+        internal async Task DoRun(IBackendManager backendManager, LocalDatabase dbparent, bool updating, IFilter filter = null, NumberedFilterFilelistDelegate filelistfilter = null, BlockVolumePostProcessor blockprocessor = null)
         {
+            var cancellationToken = CancellationToken.None;
             m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Recreate_Running);
 
             //We build a local database in steps.
             using (var restoredb = new LocalRecreateDatabase(dbparent, m_options))
-            using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, restoredb))
             {
                 restoredb.RepairInProgress = true;
                 var expRecreateDb = false; // experimental recreate db code flag
@@ -127,7 +128,7 @@ namespace Duplicati.Library.Main.Operation
                     expRecreateDb = true;
                 }
 
-                var rawlist = backend.List();
+                var rawlist = await backendManager.ListAsync(cancellationToken);
 
                 //First step is to examine the remote storage to see what
                 // kind of data we can find
@@ -142,7 +143,7 @@ namespace Duplicati.Library.Main.Operation
 
                 if (remotefiles.Length == 0)
                 {
-                    if (rawlist.Count == 0)
+                    if (rawlist.Count() == 0)
                         throw new UserInformationException("No files were found at the remote location, perhaps the target url is incorrect?", "EmptyRemoteLocation");
                     else
                     {
@@ -155,11 +156,11 @@ namespace Duplicati.Library.Main.Operation
 
                         var types = tmp.Distinct().ToArray();
                         if (tmp.Length == 0)
-                            throw new UserInformationException(string.Format("Found {0} files at the remote storage, but none that could be parsed", rawlist.Count), "EmptyRemoteLocation");
+                            throw new UserInformationException(string.Format("Found {0} files at the remote storage, but none that could be parsed", rawlist.Count()), "EmptyRemoteLocation");
                         else if (types.Length == 1)
                             throw new UserInformationException(string.Format("Found {0} parse-able files with the prefix {1}, did you forget to set the backup prefix?", tmp.Length, types[0]), "EmptyRemoteLocationWithPrefix");
                         else
-                            throw new UserInformationException(string.Format("Found {0} parse-able files (of {1} files) with different prefixes: {2}, did you forget to set the backup prefix?", tmp.Length, rawlist.Count, string.Join(", ", types)), "EmptyRemoteLocationWithPrefix");
+                            throw new UserInformationException(string.Format("Found {0} parse-able files (of {1} files) with different prefixes: {2}, did you forget to set the backup prefix?", tmp.Length, rawlist.Count(), string.Join(", ", types)), "EmptyRemoteLocationWithPrefix");
                     }
                 }
 
@@ -206,12 +207,14 @@ namespace Duplicati.Library.Main.Operation
                     var blocksize = m_options.Blocksize;
                     var hashes_pr_block = blocksize / m_options.BlockhashSize;
 
-                    foreach (var entry in new AsyncDownloader(filelistWork, backend))
+                    await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(filelistWork, m_result.TaskControl.ProgressToken))
+                    {
+                        var entry = new RemoteVolume(name, hash, size);
                         try
                         {
-                            if (!m_result.TaskControl.ProgressRendevouz().Await())
+                            if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
                             {
-                                backend.WaitForComplete(restoredb, null);
+                                await backendManager.WaitForEmptyAsync(restoredb, tr, cancellationToken).ConfigureAwait(false);
                                 m_result.EndTime = DateTime.UtcNow;
                                 return;
                             }
@@ -227,12 +230,12 @@ namespace Duplicati.Library.Main.Operation
                                 Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingFilelistVolumes", "Processing filelist volume {0} of {1}", progress, filelistWork.Count);
                             }
 
-                            using (var tmpfile = entry.TempFile)
+                            using (tmpfile)
                             {
                                 isFirstFilelist = false;
 
-                                if (entry.Hash != null && entry.Size > 0)
-                                    restoredb.UpdateRemoteVolume(entry.Name, RemoteVolumeState.Verified, entry.Size, entry.Hash, tr);
+                                if (!string.IsNullOrWhiteSpace(hash) && size > 0)
+                                    restoredb.UpdateRemoteVolume(entry.Name, RemoteVolumeState.Verified, size, hash, tr);
 
                                 var parsed = VolumeBase.ParseFilename(entry.Name);
 
@@ -288,7 +291,7 @@ namespace Duplicati.Library.Main.Operation
                                                             restoredb.AddSmallBlocksetLink(fe.Hash, fe.Blockhash, fe.Blocksize, tr);
                                                         else if (m_options.BlockHashAlgorithm == m_options.FileHashAlgorithm)
                                                             restoredb.AddSmallBlocksetLink(fe.Hash, fe.Hash, fe.Size, tr);
-                                                        else
+                                                        else if (fe.Size > 0)
                                                             Logging.Log.WriteWarningMessage(LOGTAG, "MissingBlockHash", null, "No block hash found for file: {0}", fe.Path);
                                                     }
 
@@ -338,6 +341,7 @@ namespace Duplicati.Library.Main.Operation
                                 throw;
 
                         }
+                    }
 
                     //Make sure we write the config if it has been read from a manifest
                     if (hasUpdatedOptions)
@@ -368,12 +372,13 @@ namespace Duplicati.Library.Main.Operation
 
                         var progress = 0;
 
-                        foreach (var sf in new AsyncDownloader(indexfiles, backend))
+                        await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(indexfiles, m_result.TaskControl.ProgressToken))
+                        {
                             try
                             {
-                                if (!m_result.TaskControl.ProgressRendevouz().Await())
+                                if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
                                 {
-                                    backend.WaitForComplete(restoredb, null);
+                                    await backendManager.WaitForEmptyAsync(restoredb, tr, cancellationToken).ConfigureAwait(false);
                                     m_result.EndTime = DateTime.UtcNow;
                                     return;
                                 }
@@ -382,12 +387,12 @@ namespace Duplicati.Library.Main.Operation
                                 m_result.OperationProgressUpdater.UpdateProgress((((float)progress / indexfiles.Count) * 0.5f) + 0.2f);
                                 Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingIndexlistVolumes", "Processing indexlist volume {0} of {1}", progress, indexfiles.Count);
 
-                                using (var tmpfile = sf.TempFile)
+                                using (tmpfile)
                                 {
-                                    if (sf.Hash != null && sf.Size > 0)
-                                        restoredb.UpdateRemoteVolume(sf.Name, RemoteVolumeState.Verified, sf.Size, sf.Hash, tr);
+                                    if (!string.IsNullOrWhiteSpace(hash) && size > 0)
+                                        restoredb.UpdateRemoteVolume(name, RemoteVolumeState.Verified, size, hash, tr);
 
-                                    using (var svr = new IndexVolumeReader(RestoreHandler.GetCompressionModule(sf.Name), tmpfile, m_options, hashsize))
+                                    using (var svr = new IndexVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options, hashsize))
                                     {
                                         foreach (var a in svr.Volumes)
                                         {
@@ -405,7 +410,7 @@ namespace Duplicati.Library.Main.Operation
                                                 var p = VolumeBase.ParseFilename(filename);
                                                 if (p == null)
                                                     throw new Exception(string.Format("Unable to parse filename: {0}", filename));
-                                                Logging.Log.WriteWarningMessage(LOGTAG, "MissingFileDetected", null, "Remote file referenced as {0} by {1}, but not found in list, registering a missing remote file", filename, sf.Name);
+                                                Logging.Log.WriteWarningMessage(LOGTAG, "MissingFileDetected", null, "Remote file referenced as {0} by {1}, but not found in list, registering a missing remote file", filename, name);
                                                 missing = true;
                                                 volumeID = restoredb.RegisterRemoteVolume(filename, p.FileType, RemoteVolumeState.Temporary, tr);
                                             }
@@ -416,7 +421,7 @@ namespace Duplicati.Library.Main.Operation
                                                 restoredb.UpdateBlock(b.Key, b.Value, volumeID, tr, ref anyChange);
 
                                             restoredb.UpdateRemoteVolume(filename, missing ? RemoteVolumeState.Temporary : RemoteVolumeState.Verified, a.Length, a.Hash, tr);
-                                            restoredb.AddIndexBlockLink(restoredb.GetRemoteVolumeID(sf.Name), volumeID, tr);
+                                            restoredb.AddIndexBlockLink(restoredb.GetRemoteVolumeID(name), volumeID, tr);
                                         }
 
                                         //If there are blocklists in the index file, add them to the temp blocklist hashes table
@@ -434,13 +439,13 @@ namespace Duplicati.Library.Main.Operation
                                             }
                                             catch (System.IO.InvalidDataException e)
                                             {
-                                                Logging.Log.WriteVerboseMessage(LOGTAG, "InvalidDataBlocklist", e, "Exception while processing blocklists in {0}", sf.Name);
+                                                Logging.Log.WriteVerboseMessage(LOGTAG, "InvalidDataBlocklist", e, "Exception while processing blocklists in {0}", name);
                                                 ++wrongHashes;
                                             }
                                         }
                                         if (wrongHashes != 0)
                                         {
-                                            Logging.Log.WriteWarningMessage(LOGTAG, "WrongBlocklistHashes", null, "{0} had invalid blocklists which could not be used. Consider deleting this index file and run repair to recreate it.", sf.Name);
+                                            Logging.Log.WriteWarningMessage(LOGTAG, "WrongBlocklistHashes", null, "{0} had invalid blocklists which could not be used. Consider deleting this index file and run repair to recreate it.", name);
                                         }
                                     }
                                 }
@@ -448,7 +453,7 @@ namespace Duplicati.Library.Main.Operation
                             catch (Exception ex)
                             {
                                 //Not fatal
-                                Logging.Log.WriteErrorMessage(LOGTAG, "IndexFileProcessingFailed", ex, "Failed to process index file: {0}", sf.Name);
+                                Logging.Log.WriteErrorMessage(LOGTAG, "IndexFileProcessingFailed", ex, "Failed to process index file: {0}", name);
                                 if (ex is System.Threading.ThreadAbortException)
                                 {
                                     m_result.EndTime = DateTime.UtcNow;
@@ -458,6 +463,7 @@ namespace Duplicati.Library.Main.Operation
                                 if (m_options.UnittestMode)
                                     throw;
                             }
+                        }
 
                         using (new Logging.Timer(LOGTAG, "CommitRecreateDb", "CommitRecreatedDb"))
                             tr.Commit();
@@ -504,17 +510,17 @@ namespace Duplicati.Library.Main.Operation
                         }
 
                         var progress = 0;
-                        foreach (var sf in new AsyncDownloader(lst, backend))
+                        await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(lst, m_result.TaskControl.ProgressToken))
                         {
                             try
                             {
-                                using (var tmpfile = sf.TempFile)
-                                using (var rd = new BlockVolumeReader(RestoreHandler.GetCompressionModule(sf.Name), tmpfile, m_options))
+                                using (tmpfile)
+                                using (var rd = new BlockVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options))
                                 using (var tr = restoredb.BeginTransaction())
                                 {
                                     if (!m_result.TaskControl.ProgressRendevouz().Await())
                                     {
-                                        backend.WaitForComplete(restoredb, null);
+                                        backendManager.WaitForEmptyAsync(restoredb, tr, cancellationToken).Await();
                                         m_result.EndTime = DateTime.UtcNow;
                                         return;
                                     }
@@ -523,9 +529,9 @@ namespace Duplicati.Library.Main.Operation
                                     m_result.OperationProgressUpdater.UpdateProgress((((float)progress / lst.Count) * 0.1f) + 0.7f + (i * 0.1f));
                                     Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingBlocklistVolumes", "Pass {0} of 3, processing blocklist volume {1} of {2}", (i + 1), progress, lst.Count);
 
-                                    var volumeid = restoredb.GetRemoteVolumeID(sf.Name);
+                                    var volumeid = restoredb.GetRemoteVolumeID(name);
 
-                                    restoredb.UpdateRemoteVolume(sf.Name, RemoteVolumeState.Uploaded, sf.Size, sf.Hash, tr);
+                                    restoredb.UpdateRemoteVolume(name, RemoteVolumeState.Uploaded, size, hash, tr);
 
                                     bool anyChange = false;
                                     // Update the block table so we know about the block/volume map
@@ -552,7 +558,7 @@ namespace Duplicati.Library.Main.Operation
                                     {
                                         if (i == 2)
                                         {
-                                            Logging.Log.WriteWarningMessage(LOGTAG, "UpdatingTables", null, "Unexpected changes caused by block {0}", sf.Name);
+                                            Logging.Log.WriteWarningMessage(LOGTAG, "UpdatingTables", null, "Unexpected changes caused by block {0}", name);
                                         }
                                         if (expRecreateDb)
                                             restoredb.AddBlockAndBlockSetEntryFromTemp(hashsize, m_options.Blocksize, tr, false);
@@ -565,12 +571,12 @@ namespace Duplicati.Library.Main.Operation
 
                                     //At this point we can patch files with data from the block volume
                                     if (blockprocessor != null)
-                                        blockprocessor(sf.Name, rd);
+                                        blockprocessor(name, rd);
                                 }
                             }
                             catch (Exception ex)
                             {
-                                Logging.Log.WriteWarningMessage(LOGTAG, "FailedRebuildingWithFile", ex, "Failed to use information from {0} to rebuild database: {1}", sf.Name, ex.Message);
+                                Logging.Log.WriteWarningMessage(LOGTAG, "FailedRebuildingWithFile", ex, "Failed to use information from {0} to rebuild database: {1}", name, ex.Message);
                                 if (m_options.UnittestMode)
                                     throw;
                             }
@@ -578,7 +584,7 @@ namespace Duplicati.Library.Main.Operation
                     }
                 }
 
-                backend.WaitForComplete(restoredb, null);
+                backendManager.WaitForEmptyAsync(restoredb, null, cancellationToken).Await();
 
                 if (!m_options.RepairOnlyPaths)
                 {

Duplicati/Library/Main/Operation/RepairHandler.cs

@@ -1,28 +1,28 @@
-// Copyright (C) 2025, The Duplicati Team
-// https://duplicati.com, hello@duplicati.com
-// 
-// Permission is hereby granted, free of charge, to any person obtaining a 
-// copy of this software and associated documentation files (the "Software"), 
-// to deal in the Software without restriction, including without limitation 
-// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
-// and/or sell copies of the Software, and to permit persons to whom the 
-// Software is furnished to do so, subject to the following conditions:
-// 
-// The above copyright notice and this permission notice shall be included in 
-// all copies or substantial portions of the Software.
-// 
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+// Copyright (C) 2025, The Duplicati Team
+// https://duplicati.com, hello@duplicati.com
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a 
+// copy of this software and associated documentation files (the "Software"), 
+// to deal in the Software without restriction, including without limitation 
+// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+// and/or sell copies of the Software, and to permit persons to whom the 
+// Software is furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in 
+// all copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
 // DEALINGS IN THE SOFTWARE.
 using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Security.Cryptography;
-using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
 using Duplicati.Library.Interface;
 using Duplicati.Library.Main.Database;
 using Duplicati.Library.Main.Volumes;
@@ -36,13 +36,11 @@ namespace Duplicati.Library.Main.Operation
         /// The tag used for logging
         /// </summary>
         private static readonly string LOGTAG = Logging.Log.LogTagFromType<RepairHandler>();
-        private readonly string m_backendurl;
         private readonly Options m_options;
         private readonly RepairResults m_result;
 
-        public RepairHandler(string backend, Options options, RepairResults result)
+        public RepairHandler(Options options, RepairResults result)
         {
-            m_backendurl = backend;
             m_options = options;
             m_result = result;
 
@@ -50,11 +48,11 @@ namespace Duplicati.Library.Main.Operation
                 throw new UserInformationException(Strings.Common.PassphraseChangeUnsupported, "PassphraseChangeUnsupported");
         }
 
-        public void Run(Library.Utility.IFilter filter = null)
+        public void Run(IBackendManager backendManager, IFilter filter)
         {
             if (!System.IO.File.Exists(m_options.Dbpath))
             {
-                RunRepairLocal(filter);
+                RunRepairLocal(backendManager, filter);
                 RunRepairCommon();
                 m_result.EndTime = DateTime.UtcNow;
                 return;
@@ -88,20 +86,20 @@ namespace Duplicati.Library.Main.Operation
                     System.IO.File.Move(m_options.Dbpath, baseName);
                 }
 
-                RunRepairLocal(filter);
+                RunRepairLocal(backendManager, filter);
                 RunRepairCommon();
             }
             else
             {
                 RunRepairCommon();
-                RunRepairRemote();
+                RunRepairRemote(backendManager, m_result.TaskControl.ProgressToken).Await();
             }
 
             m_result.EndTime = DateTime.UtcNow;
 
         }
 
-        public void RunRepairLocal(Library.Utility.IFilter filter = null)
+        public void RunRepairLocal(IBackendManager backendManager, Library.Utility.IFilter filter)
         {
             m_result.RecreateDatabaseResults = new RecreateDatabaseResults(m_result);
             using (new Logging.Timer(LOGTAG, "RecreateDbForRepair", "Recreate database for repair"))
@@ -112,12 +110,12 @@ namespace Duplicati.Library.Main.Operation
 
                 var filelistfilter = RestoreHandler.FilterNumberedFilelist(m_options.Time, m_options.Version);
 
-                new RecreateDatabaseHandler(m_backendurl, m_options, (RecreateDatabaseResults)m_result.RecreateDatabaseResults)
-                    .Run(m_options.Dryrun ? (string)f : m_options.Dbpath, filter, filelistfilter);
+                new RecreateDatabaseHandler(m_options, (RecreateDatabaseResults)m_result.RecreateDatabaseResults)
+                    .Run(m_options.Dryrun ? (string)f : m_options.Dbpath, backendManager, filter, filelistfilter, null);
             }
         }
 
-        public void RunRepairRemote()
+        public async Task RunRepairRemote(IBackendManager backendManager, CancellationToken cancellationToken)
         {
             if (!System.IO.File.Exists(m_options.Dbpath))
                 throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "RepairDatabaseFileDoesNotExist");
@@ -125,7 +123,6 @@ namespace Duplicati.Library.Main.Operation
             m_result.OperationProgressUpdater.UpdateProgress(0);
 
             using (var db = new LocalRepairDatabase(m_options.Dbpath))
-            using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db))
             {
                 m_result.SetDatabase(db);
                 Utility.UpdateOptionsFromDb(db, m_options);
@@ -137,7 +134,7 @@ namespace Duplicati.Library.Main.Operation
                 if (db.RepairInProgress)
                     throw new UserInformationException("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsInRepairState");
 
-                var tp = FilelistProcessor.RemoteListAnalysis(backend, m_options, db, m_result.BackendWriter, null);
+                var tp = await FilelistProcessor.RemoteListAnalysis(backendManager, m_options, db, m_result.BackendWriter, null).ConfigureAwait(false);
                 var buffer = new byte[m_options.Blocksize];
                 var hashsize = HashFactory.HashSizeBytes(m_options.BlockHashAlgorithm);
 
@@ -168,20 +165,18 @@ namespace Duplicati.Library.Main.Operation
                             foreach (var n in tp.VerificationRequiredVolumes)
                                 try
                                 {
-                                    if (!m_result.TaskControl.ProgressRendevouz().Await())
+                                    if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
                                     {
-                                        backend.WaitForComplete(db, null);
+                                        await backendManager.WaitForEmptyAsync(testdb, null, cancellationToken).ConfigureAwait(false);
                                         return;
                                     }
 
                                     progress++;
                                     m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess);
 
-                                    long size;
-                                    string hash;
                                     KeyValuePair<string, IEnumerable<KeyValuePair<Duplicati.Library.Interface.TestEntryStatus, string>>> res;
-
-                                    using (var tf = backend.GetWithInfo(n.Name, out size, out hash))
+                                    (var tf, var hash, var size) = await backendManager.GetWithInfoAsync(n.Name, n.Hash, n.Size, cancellationToken).ConfigureAwait(false);
+                                    using (tf)
                                         res = TestHandler.TestVolumeInternals(testdb, n, tf, m_options, 1);
 
                                     if (res.Value.Any())
@@ -207,9 +202,9 @@ namespace Duplicati.Library.Main.Operation
                     foreach (var n in tp.ExtraVolumes)
                         try
                         {
-                            if (!m_result.TaskControl.ProgressRendevouz().Await())
+                            if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
                             {
-                                backend.WaitForComplete(db, null);
+                                await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false);
                                 return;
                             }
 
@@ -222,9 +217,8 @@ namespace Duplicati.Library.Main.Operation
                             {
                                 try
                                 {
-                                    string hash;
-                                    long size;
-                                    using (var tf = backend.GetWithInfo(n.File.Name, out size, out hash))
+                                    (var tf, var hash, var size) = await backendManager.GetWithInfoAsync(n.File.Name, null, n.File.Size, cancellationToken).ConfigureAwait(false);
+                                    using (tf)
                                     using (var ifr = new IndexVolumeReader(n.CompressionModule, tf, m_options, m_options.BlockhashSize))
                                     {
                                         foreach (var rv in ifr.Volumes)
@@ -277,7 +271,7 @@ namespace Duplicati.Library.Main.Operation
                             if (!m_options.Dryrun)
                             {
                                 db.RegisterRemoteVolume(n.File.Name, n.FileType, n.File.Size, RemoteVolumeState.Deleting);
-                                backend.Delete(n.File.Name, n.File.Size);
+                                await backendManager.DeleteAsync(n.File.Name, n.File.Size, false, cancellationToken).ConfigureAwait(false);
                             }
                             else
                                 Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteFile", "would delete file {0}", n.File.Name);
@@ -302,9 +296,9 @@ namespace Duplicati.Library.Main.Operation
 
                         try
                         {
-                            if (!m_result.TaskControl.ProgressRendevouz().Await())
+                            if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
                             {
-                                backend.WaitForComplete(db, null);
+                                await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false);
                                 return;
                             }
 
@@ -331,7 +325,7 @@ namespace Duplicati.Library.Main.Operation
                                 else
                                 {
                                     db.UpdateRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null);
-                                    backend.Put(volumeWriter);
+                                    await backendManager.PutAsync(volumeWriter, null, null, false, cancellationToken).ConfigureAwait(false);
                                 }
                             }
                             else if (n.Type == RemoteVolumeType.Index)
@@ -372,7 +366,7 @@ namespace Duplicati.Library.Main.Operation
                                 else
                                 {
                                     db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null);
-                                    backend.Put(w);
+                                    await backendManager.PutAsync(w, null, null, false, cancellationToken).ConfigureAwait(false);
                                 }
                             }
                             else if (n.Type == RemoteVolumeType.Blocks)
@@ -423,20 +417,20 @@ namespace Duplicati.Library.Main.Operation
                                     }
 
                                     //Then we grab all remote volumes that have the missing blocks
-                                    foreach (var vol in new AsyncDownloader(mbl.GetMissingBlockSources().ToList(), backend))
+                                    await foreach (var (tmpfile, _, _, name) in backendManager.GetFilesOverlappedAsync(mbl.GetMissingBlockSources().ToList(), cancellationToken))
                                     {
                                         try
                                         {
-                                            using (var tmpfile = vol.TempFile)
-                                            using (var f = new BlockVolumeReader(RestoreHandler.GetCompressionModule(vol.Name), tmpfile, m_options))
+                                            using (tmpfile)
+                                            using (var f = new BlockVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options))
                                                 foreach (var b in f.Blocks)
                                                     if (mbl.SetBlockRestored(b.Key, b.Value))
                                                         if (f.ReadBlock(b.Key, buffer) == b.Value)
-                                                            w.AddBlock(b.Key, buffer, 0, (int)b.Value, Duplicati.Library.Interface.CompressionHint.Default);
+                                                            w.AddBlock(b.Key, buffer, 0, (int)b.Value, CompressionHint.Default);
                                         }
                                         catch (Exception ex)
                                         {
-                                            Logging.Log.WriteErrorMessage(LOGTAG, "RemoteFileAccessError", ex, "Failed to access remote file: {0}", vol.Name);
+                                            Logging.Log.WriteErrorMessage(LOGTAG, "RemoteFileAccessError", ex, "Failed to access remote file: {0}", name);
                                         }
                                     }
 
@@ -468,7 +462,7 @@ namespace Duplicati.Library.Main.Operation
                                         else
                                         {
                                             db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null);
-                                            backend.Put(w);
+                                            await backendManager.PutAsync(w, null, null, false, cancellationToken).ConfigureAwait(false);
                                         }
                                     }
                                 }
@@ -494,7 +488,7 @@ namespace Duplicati.Library.Main.Operation
                 }
 
                 m_result.OperationProgressUpdater.UpdateProgress(1);
-                backend.WaitForComplete(db, null);
+                await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false);
                 db.WriteResults();
             }
         }

Duplicati/Library/Main/Operation/Restore/BlockManager.cs

@@ -92,11 +92,15 @@ namespace Duplicati.Library.Main.Operation.Restore
             /// <summary>
             /// Dictionary for keeping track of how many times each block is requested. Used to determine when a block is no longer needed.
             /// </summary>
-            private readonly Dictionary<long, long> m_blockcount = new ();
+            private readonly Dictionary<long, long> m_blockcount = new();
+            /// <summary>
+            /// Lock for the block count dictionary.
+            /// </summary>
+            private readonly object m_blockcount_lock = new();
             /// <summary>
             /// Dictionary for keeping track of how many times each volume is requested. Used to determine when a volume is no longer needed.
             /// </summary>
-            private readonly Dictionary<long, long> m_volumecount = new ();
+            private readonly Dictionary<long, long> m_volumecount = new();
             /// <summary>
             /// The options for the restore.
             /// </summary>
@@ -118,9 +122,9 @@ namespace Duplicati.Library.Main.Operation.Restore
                     m_block_cache = new MemoryCache(cache_options);
                 }
                 this.readers = readers;
-                sw_cacheevict = options.InternalProfiling ? new () : null;
-                sw_checkcounts = options.InternalProfiling ? new () : null;
-                sw_get_wait = options.InternalProfiling ? new () : null;
+                sw_cacheevict = options.InternalProfiling ? new() : null;
+                sw_checkcounts = options.InternalProfiling ? new() : null;
+                sw_get_wait = options.InternalProfiling ? new() : null;
 
                 foreach (var (block_id, volume_id) in db.GetBlocksAndVolumeIDs(options.SkipMetadata))
                 {
@@ -143,12 +147,13 @@ namespace Duplicati.Library.Main.Operation.Restore
             {
                 long error_block_id = -1;
                 long error_volume_id = -1;
+                var emit_evict = false;
 
-                lock (m_blockcount)
+                lock (m_blockcount_lock)
                 {
                     sw_checkcounts?.Start();
 
-                    var block_count = m_blockcount.TryGetValue(blockRequest.BlockID, out var c) ? c-1 : 0;
+                    var block_count = m_blockcount.TryGetValue(blockRequest.BlockID, out var c) ? c - 1 : 0;
 
                     if (block_count > 0)
                     {
@@ -165,7 +170,7 @@ namespace Duplicati.Library.Main.Operation.Restore
                         error_block_id = blockRequest.BlockID;
                     }
 
-                    var vol_count = m_volumecount.TryGetValue(blockRequest.VolumeID, out var vc) ? vc-1 : 0;
+                    var vol_count = m_volumecount.TryGetValue(blockRequest.VolumeID, out var vc) ? vc - 1 : 0;
                     if (vol_count > 0)
                     {
                         m_volumecount[blockRequest.VolumeID] = vol_count;
@@ -175,7 +180,7 @@ namespace Duplicati.Library.Main.Operation.Restore
                         // Notify the `VolumeManager` that it should evict the volume.
                         m_volumecount.Remove(blockRequest.VolumeID);
                         blockRequest.CacheDecrEvict = true;
-                        m_volume_request.Write(blockRequest);
+                        emit_evict = true;
                     }
                     else // vol_count < 0
                     {
@@ -184,6 +189,9 @@ namespace Duplicati.Library.Main.Operation.Restore
                     sw_checkcounts?.Stop();
                 }
 
+                if (emit_evict)
+                    m_volume_request.Write(blockRequest);
+
                 if (error_block_id != -1)
                 {
                     Logging.Log.WriteWarningMessage(LOGTAG, "BlockCountError", null, $"Block {blockRequest.BlockID} has a count below 0");
@@ -222,8 +230,10 @@ namespace Duplicati.Library.Main.Operation.Restore
                 }
                 sw_get_wait?.Stop();
 
-                return new_tcs.Task.ContinueWith(t => {
-                    CheckCounts(block_request);
+                return new_tcs.Task.ContinueWith(t =>
+                {
+                    if (t.IsCompletedSuccessfully)
+                        CheckCounts(block_request);
                     return t.Result;
                 });
             }
@@ -264,7 +274,8 @@ namespace Duplicati.Library.Main.Operation.Restore
             /// </summary>
             public void Retire()
             {
-                if (Interlocked.Decrement(ref readers) <= 0) {
+                if (Interlocked.Decrement(ref readers) <= 0)
+                {
                     m_volume_request.Retire();
                 }
             }
@@ -338,15 +349,16 @@ namespace Duplicati.Library.Main.Operation.Restore
                 using SleepableDictionary cache = new(db, self.Output, options, fp_requests.Length);
 
                 // The volume consumer will read blocks from the input channel (data blocks from the volumes) and store them in the cache.
-                var volume_consumer = Task.Run(async () => {
-                    Stopwatch sw_read = options.InternalProfiling ? new () : null;
-                    Stopwatch sw_set  = options.InternalProfiling ? new () : null;
+                var volume_consumer = Task.Run(async () =>
+                {
+                    Stopwatch sw_read = options.InternalProfiling ? new() : null;
+                    Stopwatch sw_set = options.InternalProfiling ? new() : null;
                     try
                     {
                         while (true)
                         {
                             sw_read?.Start();
-                            var (block_request, data) = await self.Input.ReadAsync();
+                            var (block_request, data) = await self.Input.ReadAsync().ConfigureAwait(false);
                             sw_read?.Stop();
 
                             sw_set?.Start();
@@ -376,17 +388,18 @@ namespace Duplicati.Library.Main.Operation.Restore
                 });
 
                 // The block handlers will read block requests from the `FileProcessor`, access the cache for the blocks, and write the resulting blocks to the `FileProcessor`.
-                var block_handlers = fp_requests.Zip(fp_responses, (req, res) => Task.Run(async () => {
-                    Stopwatch sw_req   = options.InternalProfiling ? new () : null;
-                    Stopwatch sw_resp  = options.InternalProfiling ? new () : null;
-                    Stopwatch sw_cache = options.InternalProfiling ? new () : null;
-                    Stopwatch sw_get   = options.InternalProfiling ? new () : null;
+                var block_handlers = fp_requests.Zip(fp_responses, (req, res) => Task.Run(async () =>
+                {
+                    Stopwatch sw_req = options.InternalProfiling ? new() : null;
+                    Stopwatch sw_resp = options.InternalProfiling ? new() : null;
+                    Stopwatch sw_cache = options.InternalProfiling ? new() : null;
+                    Stopwatch sw_get = options.InternalProfiling ? new() : null;
                     try
                     {
                         while (true)
                         {
                             sw_req?.Start();
-                            var block_request = await req.ReadAsync();
+                            var block_request = await req.ReadAsync().ConfigureAwait(false);
                             sw_req?.Stop();
                             if (block_request.CacheDecrEvict)
                             {
@@ -398,11 +411,11 @@ namespace Duplicati.Library.Main.Operation.Restore
                             else
                             {
                                 sw_get?.Start();
-                                var data = await cache.Get(block_request);
+                                var data = await cache.Get(block_request).ConfigureAwait(false);
                                 sw_get?.Stop();
 
                                 sw_resp?.Start();
-                                await res.WriteAsync(data);
+                                await res.WriteAsync(data).ConfigureAwait(false);
                                 sw_resp?.Stop();
                             }
                         }
@@ -420,7 +433,7 @@ namespace Duplicati.Library.Main.Operation.Restore
                     }
                 })).ToArray();
 
-                await Task.WhenAll([volume_consumer, ..block_handlers]);
+                await Task.WhenAll([volume_consumer, .. block_handlers]).ConfigureAwait(false);
             });
         }
     }

Duplicati/Library/Main/Operation/Restore/Channels.cs

@@ -19,10 +19,10 @@
 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
 // DEALINGS IN THE SOFTWARE.
 
+using System.Threading.Tasks;
 using CoCoL;
 using Duplicati.Library.Main.Volumes;
 using Duplicati.Library.Utility;
-using static Duplicati.Library.Main.BackendManager;
 
 namespace Duplicati.Library.Main.Operation.Restore
 {
@@ -46,7 +46,7 @@ namespace Duplicati.Library.Main.Operation.Restore
         /// <summary>
         /// Channel between <see cref="VolumeManager"/> and <see cref="VolumeDownloader"/>.
         /// </summary>
-        public readonly IChannel<(long, IDownloadWaitHandle)> DownloadRequest = ChannelManager.CreateChannel<(long, IDownloadWaitHandle)>(buffersize: BufferSize);
+        public readonly IChannel<(long, Task<TempFile>)> DownloadRequest = ChannelManager.CreateChannel<(long, Task<TempFile>)>(buffersize: BufferSize);
 
         /// <summary>
         /// Channel between <see cref="VolumeDownloader"/> and <see cref="VolumeDecryptor"/>

Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs

@@ -55,9 +55,9 @@ namespace Duplicati.Library.Main.Operation.Restore
             },
             async self =>
             {
-                Stopwatch sw_read  = options.InternalProfiling ? new () : null;
-                Stopwatch sw_write = options.InternalProfiling ? new () : null;
-                Stopwatch sw_wait  = options.InternalProfiling ? new () : null;
+                Stopwatch sw_read = options.InternalProfiling ? new() : null;
+                Stopwatch sw_write = options.InternalProfiling ? new() : null;
+                Stopwatch sw_wait = options.InternalProfiling ? new() : null;
 
                 try
                 {
@@ -65,23 +65,22 @@ namespace Duplicati.Library.Main.Operation.Restore
                     {
                         // Get the block request from the `BlockManager` process.
                         sw_read?.Start();
-                        var (volume_id, handle) = await self.Input.ReadAsync();
+                        var (volume_id, waittask) = await self.Input.ReadAsync();
                         sw_read?.Stop();
 
-                         // Trigger the download.
+                        // Trigger the download.
                         sw_wait?.Start();
                         TempFile f = null;
                         try
                         {
-                            f = handle.Wait();
+                            f = await waittask.ConfigureAwait(false);
                         }
                         catch (Exception)
                         {
                             var (volume_name, _, _) = db.GetVolumeInfo(volume_id).First();
                             lock (results)
-                            {
                                 results.BrokenRemoteFiles.Add(volume_name);
-                            }
+
                             throw;
                         }
                         sw_wait?.Stop();

Duplicati/Library/Main/Operation/Restore/VolumeManager.cs

@@ -50,7 +50,7 @@ namespace Duplicati.Library.Main.Operation.Restore
         /// <param name="backend">The backend manager, used to fetch the volumes from the backend.</param>
         /// <param name="options">The restore options.</param>
         /// <param name="results">The restore results.</param>
-        public static Task Run(Channels channels, LocalRestoreDatabase db, BackendManager backend, Options options, RestoreResults results)
+        public static Task Run(Channels channels, LocalRestoreDatabase db, IBackendManager backend, Options options, RestoreResults results)
         {
             return AutomationExtensions.RunTask(
                 new
@@ -65,12 +65,12 @@ namespace Duplicati.Library.Main.Operation.Restore
                     Dictionary<long, TempFile> files = [];
                     Dictionary<long, List<BlockRequest>> in_flight = [];
 
-                    Stopwatch sw_cache_set = options.InternalProfiling ? new () : null;
-                    Stopwatch sw_cache_evict = options.InternalProfiling ? new () : null;
-                    Stopwatch sw_query = options.InternalProfiling ? new () : null;
-                    Stopwatch sw_backend = options.InternalProfiling ? new () : null;
-                    Stopwatch sw_request = options.InternalProfiling ? new () : null;
-                    Stopwatch sw_wakeup = options.InternalProfiling ? new () : null;
+                    Stopwatch sw_cache_set = options.InternalProfiling ? new() : null;
+                    Stopwatch sw_cache_evict = options.InternalProfiling ? new() : null;
+                    Stopwatch sw_query = options.InternalProfiling ? new() : null;
+                    Stopwatch sw_backend = options.InternalProfiling ? new() : null;
+                    Stopwatch sw_request = options.InternalProfiling ? new() : null;
+                    Stopwatch sw_wakeup = options.InternalProfiling ? new() : null;
 
                     try
                     {
@@ -110,12 +110,12 @@ namespace Duplicati.Library.Main.Operation.Restore
                                                     var (volume_name, volume_size, volume_hash) = db.GetVolumeInfo(request.VolumeID).First();
                                                     sw_query?.Stop();
                                                     sw_backend?.Start();
-                                                    var handle = backend.GetAsync(volume_name, volume_size, volume_hash);
+                                                    var handle = backend.GetAsync(volume_name, volume_hash, volume_size, results.TaskControl.TransferToken);
                                                     sw_backend?.Stop();
                                                     await self.DownloadRequest.WriteAsync((request.VolumeID, handle));
                                                     in_flight[request.VolumeID] = [request];
                                                 }
-                                            };
+                                            }
                                             sw_request?.Stop();
                                         }
                                         break;

Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs

@@ -1,27 +1,26 @@
-// Copyright (C) 2025, The Duplicati Team
-// https://duplicati.com, hello@duplicati.com
-// 
-// Permission is hereby granted, free of charge, to any person obtaining a 
-// copy of this software and associated documentation files (the "Software"), 
-// to deal in the Software without restriction, including without limitation 
-// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
-// and/or sell copies of the Software, and to permit persons to whom the 
-// Software is furnished to do so, subject to the following conditions:
-// 
-// The above copyright notice and this permission notice shall be included in 
-// all copies or substantial portions of the Software.
-// 
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+// Copyright (C) 2025, The Duplicati Team
+// https://duplicati.com, hello@duplicati.com
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a 
+// copy of this software and associated documentation files (the "Software"), 
+// to deal in the Software without restriction, including without limitation 
+// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+// and/or sell copies of the Software, and to permit persons to whom the 
+// Software is furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in 
+// all copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
 // DEALINGS IN THE SOFTWARE.
 using System;
 using System.Collections.Generic;
-using System.Linq;
-using System.Text;
+using System.Threading;
 using Duplicati.Library.Utility;
 
 namespace Duplicati.Library.Main.Operation
@@ -29,18 +28,17 @@ namespace Duplicati.Library.Main.Operation
     internal class RestoreControlFilesHandler
     {
         private readonly Options m_options;
-        private readonly string m_backendurl;
         private readonly RestoreControlFilesResults m_result;
 
-        public RestoreControlFilesHandler(string backendurl, Options options, RestoreControlFilesResults result)
+        public RestoreControlFilesHandler(Options options, RestoreControlFilesResults result)
         {
             m_options = options;
-            m_backendurl = backendurl;
             m_result = result;
         }
 
-        public void Run(IEnumerable<string> filterstrings = null, Library.Utility.IFilter compositefilter = null)
+        public void Run(IEnumerable<string> filterstrings, IBackendManager backendManager, Library.Utility.IFilter compositefilter)
         {
+            var cancellationToken = CancellationToken.None;
             if (string.IsNullOrEmpty(m_options.Restorepath))
                 throw new Exception("Cannot restore control files without --restore-path");
             if (!System.IO.Directory.Exists(m_options.Restorepath))
@@ -48,7 +46,6 @@ namespace Duplicati.Library.Main.Operation
 
             using (var tmpdb = new Library.Utility.TempFile())
             using (var db = new Database.LocalDatabase(System.IO.File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "RestoreControlFiles", true))
-            using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db))
             {
                 m_result.SetDatabase(db);
 
@@ -56,7 +53,7 @@ namespace Duplicati.Library.Main.Operation
 
                 try
                 {
-                    var filteredList = ListFilesHandler.ParseAndFilterFilesets(backend.List(), m_options);
+                    var filteredList = ListFilesHandler.ParseAndFilterFilesets(backendManager.ListAsync(cancellationToken).Await(), m_options);
                     if (filteredList.Count == 0)
                         throw new Exception("No filesets found on remote target");
 
@@ -67,7 +64,7 @@ namespace Duplicati.Library.Main.Operation
                         {
                             if (!m_result.TaskControl.ProgressRendevouz().Await())
                             {
-                                backend.WaitForComplete(db, null);
+                                backendManager.WaitForEmptyAsync(db, null, cancellationToken).Await();
                                 return;
                             }
 
@@ -75,7 +72,7 @@ namespace Duplicati.Library.Main.Operation
                             var entry = db.GetRemoteVolume(file.Name);
 
                             var res = new List<string>();
-                            using (var tmpfile = backend.Get(file.Name, entry.Size < 0 ? file.Size : entry.Size, entry.Hash))
+                            using (var tmpfile = backendManager.GetAsync(file.Name, entry.Hash, entry.Size < 0 ? file.Size : entry.Size, cancellationToken).Await())
                             using (var tmp = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(file.Name), tmpfile, m_options))
                                 foreach (var cf in tmp.ControlFiles)
                                     if (Library.Utility.FilterExpression.Matches(filter, cf.Key))
@@ -103,7 +100,7 @@ namespace Duplicati.Library.Main.Operation
                 }
                 finally
                 {
-                    backend.WaitForComplete(db, null);
+                    backendManager.WaitForEmptyAsync(db, null, cancellationToken).Await();
                 }
 
                 db.WriteResults();

Duplicati/Library/Main/Operation/RestoreHandler.cs

@@ -1,22 +1,22 @@
-// Copyright (C) 2025, The Duplicati Team
-// https://duplicati.com, hello@duplicati.com
-// 
-// Permission is hereby granted, free of charge, to any person obtaining a 
-// copy of this software and associated documentation files (the "Software"), 
-// to deal in the Software without restriction, including without limitation 
-// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
-// and/or sell copies of the Software, and to permit persons to whom the 
-// Software is furnished to do so, subject to the following conditions:
-// 
-// The above copyright notice and this permission notice shall be included in 
-// all copies or substantial portions of the Software.
-// 
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+// Copyright (C) 2025, The Duplicati Team
+// https://duplicati.com, hello@duplicati.com
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a 
+// copy of this software and associated documentation files (the "Software"), 
+// to deal in the Software without restriction, including without limitation 
+// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+// and/or sell copies of the Software, and to permit persons to whom the 
+// Software is furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in 
+// all copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
 // DEALINGS IN THE SOFTWARE.
 
 using System;
@@ -40,16 +40,14 @@ namespace Duplicati.Library.Main.Operation
         /// </summary>
         private static readonly string LOGTAG = Logging.Log.LogTagFromType<RestoreHandler>();
 
-        private readonly string m_backendurl;
         private readonly Options m_options;
         private byte[] m_blockbuffer;
         private readonly RestoreResults m_result;
         private static readonly string DIRSEP = Util.DirectorySeparatorString;
 
-        public RestoreHandler(string backendurl, Options options, RestoreResults result)
+        public RestoreHandler(Options options, RestoreResults result)
         {
             m_options = options;
-            m_backendurl = backendurl;
             m_result = result;
         }
 
@@ -104,7 +102,7 @@ namespace Duplicati.Library.Main.Operation
                 };
         }
 
-        public void Run(string[] paths, Library.Utility.IFilter filter = null)
+        public void Run(string[] paths, IBackendManager backendManager, Library.Utility.IFilter filter)
         {
             m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_Begin);
 
@@ -128,8 +126,8 @@ namespace Duplicati.Library.Main.Operation
                     db = new LocalRestoreDatabase(tmpdb);
                     m_result.RecreateDatabaseResults = new RecreateDatabaseResults(m_result);
                     using (new Logging.Timer(LOGTAG, "RecreateTempDbForRestore", "Recreate temporary database for restore"))
-                        new RecreateDatabaseHandler(m_backendurl, m_options, (RecreateDatabaseResults)m_result.RecreateDatabaseResults)
-                            .DoRun(db, false, filter, filelistfilter, null);
+                        new RecreateDatabaseHandler(m_options, (RecreateDatabaseResults)m_result.RecreateDatabaseResults)
+                            .DoRun(backendManager, db, false, filter, filelistfilter, null).Await();
 
                     if (!m_options.SkipMetadata)
                         ApplyStoredMetadata(m_options, new RestoreHandlerMetadataStorage());
@@ -141,9 +139,9 @@ namespace Duplicati.Library.Main.Operation
                 }
 
                 if (m_options.RestoreLegacy)
-                    DoRun(db, filter);
+                    DoRun(backendManager, db, filter, m_result.TaskControl.ProgressToken).Await();
                 else
-                    DoRunNew(db, filter);
+                    DoRunNew(backendManager, db, filter, m_result.TaskControl.ProgressToken).Await();
 
                 db.WriteResults();
             }
@@ -309,27 +307,24 @@ namespace Duplicati.Library.Main.Operation
         /// </summary>
         /// <param name="database">The database containing information about the restore.</param>
         /// <param name="filter">The filter of which files to restore.</param>
-        private void DoRunNew(LocalRestoreDatabase database, Library.Utility.IFilter filter)
+        private async Task DoRunNew(IBackendManager backendManager, LocalRestoreDatabase database, Library.Utility.IFilter filter, CancellationToken cancellationToken)
         {
             // Perform initial setup
             Utility.UpdateOptionsFromDb(database, m_options);
             Utility.VerifyOptionsAndUpdateDatabase(database, m_options);
 
-            // Open the backend and metadata storage
-            using var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, database);
-
             // Verify the backend if necessary
             if (!m_options.NoBackendverification)
             {
                 m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PreRestoreVerify);
-                FilelistProcessor.VerifyRemoteList(backend, m_options, database, m_result.BackendWriter, false, null);
+                await FilelistProcessor.VerifyRemoteList(backendManager, m_options, database, m_result.BackendWriter, false, null).ConfigureAwait(false);
             }
 
             // Prepare the block and file list and create the directory structure
             m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateFileList);
             PrepareBlockAndFileList(database, m_options, filter, m_result);
             m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateTargetFolders);
-            CreateDirectoryStructure(database, m_options, m_result);
+            await CreateDirectoryStructure(database, m_options, m_result).ConfigureAwait(false);
             database.SetResult(m_result);
 
             // Create the channels between BlockManager and FileProcessor
@@ -341,7 +336,7 @@ namespace Duplicati.Library.Main.Operation
             var filelister = Restore.FileLister.Run(channels, database, m_options, m_result);
             var fileprocessors = Enumerable.Range(0, m_options.RestoreFileProcessors).Select(i => Restore.FileProcessor.Run(channels, database, fileprocessor_requests[i], fileprocessor_responses[i], m_options, m_result)).ToArray();
             var blockmanager = Restore.BlockManager.Run(channels, database, m_options, fileprocessor_requests, fileprocessor_responses);
-            var volumecache = Restore.VolumeManager.Run(channels, database, backend, m_options, m_result);
+            var volumecache = Restore.VolumeManager.Run(channels, database, backendManager, m_options, m_result);
             var volumedownloaders = Enumerable.Range(0, m_options.RestoreVolumeDownloaders).Select(i => Restore.VolumeDownloader.Run(channels, database, m_options, m_result)).ToArray();
             var volumedecryptors = Enumerable.Range(0, m_options.RestoreVolumeDecryptors).Select(i => Restore.VolumeDecryptor.Run(channels, m_options)).ToArray();
             var volumedecompressors = Enumerable.Range(0, m_options.RestoreVolumeDecompressors).Select(i => Restore.VolumeDecompressor.Run(channels, m_options)).ToArray();
@@ -359,20 +354,22 @@ namespace Duplicati.Library.Main.Operation
                 ];
 
             // Start the progress updater
-            var kill_updater = new CancellationTokenSource();
-            var updater = Task.Run(async () =>
+            using (var kill_updater = new CancellationTokenSource())
             {
-                while (true)
+                var updater = Task.Run(async () =>
                 {
-                    m_result.OperationProgressUpdater.UpdatefilesProcessed(m_result.RestoredFiles, m_result.SizeOfRestoredFiles);
-                    await Task.Delay(1000);
-                }
-            }, kill_updater.Token);
+                    while (!kill_updater.Token.IsCancellationRequested)
+                    {
+                        m_result.OperationProgressUpdater.UpdatefilesProcessed(m_result.RestoredFiles, m_result.SizeOfRestoredFiles);
+                        await Task.Delay(1000, kill_updater.Token).ConfigureAwait(false);
+                    }
+                }, kill_updater.Token);
 
-            Task.WhenAll(all).Wait();
-            kill_updater.Cancel();
+                await Task.WhenAll(all).ConfigureAwait(false);
+                kill_updater.Cancel();
+            }
 
-            if (!m_result.TaskControl.ProgressRendevouz().Await())
+            if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
                 return;
 
             m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PostRestoreVerify);
@@ -399,7 +396,7 @@ namespace Duplicati.Library.Main.Operation
 
             // Drop the temp tables
             database.DropRestoreTable();
-            backend.WaitForComplete(database, null);
+            await backendManager.WaitForEmptyAsync(database, null, CancellationToken.None).ConfigureAwait(false);
 
             // Report that the restore is complete
             m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_Complete);
@@ -412,12 +409,11 @@ namespace Duplicati.Library.Main.Operation
         /// </summary>
         /// <param name="database">The database containing information about the restore.</param>
         /// <param name="filter">The filter of which files to restore.</param>
-        private void DoRun(LocalRestoreDatabase database, Library.Utility.IFilter filter)
+        private async Task DoRun(IBackendManager backendManager, LocalRestoreDatabase database, Library.Utility.IFilter filter, CancellationToken cancellationToken)
         {
             //In this case, we check that the remote storage fits with the database.
             //We can then query the database and find the blocks that we need to do the restore
             //using (var database = new LocalRestoreDatabase(dbparent))
-            using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, database))
             using (var metadatastorage = new RestoreHandlerMetadataStorage())
             {
                 database.SetResult(m_result);
@@ -428,7 +424,7 @@ namespace Duplicati.Library.Main.Operation
                 if (!m_options.NoBackendverification)
                 {
                     m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PreRestoreVerify);
-                    FilelistProcessor.VerifyRemoteList(backend, m_options, database, m_result.BackendWriter, false, null);
+                    await FilelistProcessor.VerifyRemoteList(backendManager, m_options, database, m_result.BackendWriter, false, null).ConfigureAwait(false);
                 }
 
                 //Figure out what files are to be patched, and what blocks are needed
@@ -439,14 +435,14 @@ namespace Duplicati.Library.Main.Operation
                 //Make the entire output setup
                 m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateTargetFolders);
                 using (new Logging.Timer(LOGTAG, "CreateDirectory", "CreateDirectory"))
-                    CreateDirectoryStructure(database, m_options, m_result);
+                    await CreateDirectoryStructure(database, m_options, m_result);
 
                 //If we are patching an existing target folder, do not touch stuff that is already updated
                 m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_ScanForExistingFiles);
                 using (var blockhasher = HashFactory.CreateHasher(m_options.BlockHashAlgorithm))
                 using (var filehasher = HashFactory.CreateHasher(m_options.FileHashAlgorithm))
                 using (new Logging.Timer(LOGTAG, "ScanForExistingTargetBlocks", "ScanForExistingTargetBlocks"))
-                    ScanForExistingTargetBlocks(database, m_blockbuffer, blockhasher, filehasher, m_options, m_result);
+                    await ScanForExistingTargetBlocks(database, m_blockbuffer, blockhasher, filehasher, m_options, m_result).ConfigureAwait(false);
 
                 //Look for existing blocks in the original source files only
                 if (m_options.UseLocalBlocks && !string.IsNullOrEmpty(m_options.Restorepath))
@@ -455,13 +451,13 @@ namespace Duplicati.Library.Main.Operation
                     using (new Logging.Timer(LOGTAG, "ScanForExistingSourceBlocksFast", "ScanForExistingSourceBlocksFast"))
                     {
                         m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_ScanForLocalBlocks);
-                        ScanForExistingSourceBlocksFast(database, m_options, m_blockbuffer, blockhasher, m_result);
+                        await ScanForExistingSourceBlocksFast(database, m_options, m_blockbuffer, blockhasher, m_result).ConfigureAwait(false);
                     }
                 }
 
-                if (!m_result.TaskControl.ProgressRendevouz().Await())
+                if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
                 {
-                    backend.WaitForComplete(database, null);
+                    await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false);
                     return;
                 }
 
@@ -471,12 +467,12 @@ namespace Duplicati.Library.Main.Operation
                     m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PatchWithLocalBlocks);
                     using (var blockhasher = HashFactory.CreateHasher(m_options.BlockHashAlgorithm))
                     using (new Logging.Timer(LOGTAG, "PatchWithLocalBlocks", "PatchWithLocalBlocks"))
-                        ScanForExistingSourceBlocks(database, m_options, m_blockbuffer, blockhasher, m_result, metadatastorage);
+                        await ScanForExistingSourceBlocks(database, m_options, m_blockbuffer, blockhasher, m_result, metadatastorage).ConfigureAwait(false);
                 }
 
-                if (!m_result.TaskControl.ProgressRendevouz().Await())
+                if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
                 {
-                    backend.WaitForComplete(database, null);
+                    await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false);
                     return;
                 }
 
@@ -493,26 +489,28 @@ namespace Duplicati.Library.Main.Operation
 
                 var brokenFiles = new List<string>();
 
-                foreach (var blockvolume in new AsyncDownloader(volumes, backend))
+                await foreach (var (tmpfile, _, _, name) in backendManager.GetFilesOverlappedAsync(volumes, cancellationToken))
+                {
                     try
                     {
-                        if (!m_result.TaskControl.ProgressRendevouz().Await())
+                        if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
                         {
-                            backend.WaitForComplete(database, null);
+                            await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false);
                             return;
                         }
 
-                        using (var tmpfile = blockvolume.TempFile)
-                        using (var blocks = new BlockVolumeReader(GetCompressionModule(blockvolume.Name), tmpfile, m_options))
+                        using (tmpfile)
+                        using (var blocks = new BlockVolumeReader(GetCompressionModule(name), tmpfile, m_options))
                             PatchWithBlocklist(database, blocks, m_options, m_result, m_blockbuffer, metadatastorage);
                     }
                     catch (Exception ex)
                     {
-                        brokenFiles.Add(blockvolume.Name);
-                        Logging.Log.WriteErrorMessage(LOGTAG, "PatchingFailed", ex, "Failed to patch with remote file: \"{0}\", message: {1}", blockvolume.Name, ex.Message);
+                        brokenFiles.Add(name);
+                        Logging.Log.WriteErrorMessage(LOGTAG, "PatchingFailed", ex, "Failed to patch with remote file: \"{0}\", message: {1}", name, ex.Message);
                         if (ex is System.Threading.ThreadAbortException)
                             throw;
                     }
+                }
 
                 var fileErrors = 0L;
 
@@ -545,7 +543,7 @@ namespace Duplicati.Library.Main.Operation
                 if (!m_options.SkipMetadata)
                     ApplyStoredMetadata(m_options, metadatastorage);
 
-                if (!m_result.TaskControl.ProgressRendevouz().Await())
+                if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
                     return;
 
                 m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PostRestoreVerify);
@@ -560,9 +558,9 @@ namespace Duplicati.Library.Main.Operation
                         {
                             try
                             {
-                                if (!m_result.TaskControl.ProgressRendevouz().Await())
+                                if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
                                 {
-                                    backend.WaitForComplete(database, null);
+                                    await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false);
                                     return;
                                 }
 
@@ -600,7 +598,7 @@ namespace Duplicati.Library.Main.Operation
 
                 // Drop the temp tables
                 database.DropRestoreTable();
-                backend.WaitForComplete(database, null);
+                await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false);
             }
 
             m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_Complete);
@@ -676,7 +674,7 @@ namespace Duplicati.Library.Main.Operation
             }
         }
 
-        private static void ScanForExistingSourceBlocksFast(LocalRestoreDatabase database, Options options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, RestoreResults result)
+        private static async Task ScanForExistingSourceBlocksFast(LocalRestoreDatabase database, Options options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, RestoreResults result)
         {
             // Fill BLOCKS with data from known local source files
             using (var blockmarker = database.CreateBlockMarker())
@@ -708,7 +706,7 @@ namespace Duplicati.Library.Main.Operation
                                     {
                                         foreach (var block in entry.Blocks)
                                         {
-                                            if (!result.TaskControl.ProgressRendevouz().Await())
+                                            if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
                                                 return;
 
                                             //TODO: Handle metadata
@@ -748,7 +746,7 @@ namespace Duplicati.Library.Main.Operation
                             if ((++updateCount) % 20 == 0)
                             {
                                 blockmarker.UpdateProcessed(result.OperationProgressUpdater);
-                                if (!result.TaskControl.ProgressRendevouz().Await())
+                                if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
                                     return;
                             }
 
@@ -781,7 +779,7 @@ namespace Duplicati.Library.Main.Operation
             }
         }
 
-        private static void ScanForExistingSourceBlocks(LocalRestoreDatabase database, Options options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, RestoreResults result, RestoreHandlerMetadataStorage metadatastorage)
+        private static async Task ScanForExistingSourceBlocks(LocalRestoreDatabase database, Options options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, RestoreResults result, RestoreHandlerMetadataStorage metadatastorage)
         {
             // Fill BLOCKS with data from known local source files
             using (var blockmarker = database.CreateBlockMarker())
@@ -794,7 +792,7 @@ namespace Duplicati.Library.Main.Operation
                     var patched = false;
                     try
                     {
-                        if (!result.TaskControl.ProgressRendevouz().Await())
+                        if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
                             return;
 
                         var folderpath = SystemIO.IO_OS.PathGetDirectoryName(targetpath);
@@ -811,7 +809,7 @@ namespace Duplicati.Library.Main.Operation
                                 {
                                     try
                                     {
-                                        if (!result.TaskControl.ProgressRendevouz().Await())
+                                        if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
                                             return;
 
                                         if (SystemIO.IO_OS.FileExists(source.Path))
@@ -923,7 +921,7 @@ namespace Duplicati.Library.Main.Operation
 
         }
 
-        private static void CreateDirectoryStructure(LocalRestoreDatabase database, Options options, RestoreResults result)
+        private static async Task CreateDirectoryStructure(LocalRestoreDatabase database, Options options, RestoreResults result)
         {
             // This part is not protected by try/catch as we need the target folder to exist
             if (!string.IsNullOrEmpty(options.Restorepath))
@@ -941,7 +939,7 @@ namespace Duplicati.Library.Main.Operation
             {
                 try
                 {
-                    if (!result.TaskControl.ProgressRendevouz().Await())
+                    if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
                         return;
 
                     if (!SystemIO.IO_OS.DirectoryExists(folder))
@@ -965,7 +963,7 @@ namespace Duplicati.Library.Main.Operation
             }
         }
 
-        private static void ScanForExistingTargetBlocks(LocalRestoreDatabase database, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm blockhasher, System.Security.Cryptography.HashAlgorithm filehasher, Options options, RestoreResults result)
+        private static async Task ScanForExistingTargetBlocks(LocalRestoreDatabase database, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm blockhasher, System.Security.Cryptography.HashAlgorithm filehasher, Options options, RestoreResults result)
         {
             // Scan existing files for existing BLOCKS
             using (var blockmarker = database.CreateBlockMarker())
@@ -982,7 +980,7 @@ namespace Duplicati.Library.Main.Operation
                     {
                         try
                         {
-                            if (!result.TaskControl.ProgressRendevouz().Await())
+                            if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
                                 return;
 
                             var currentfilelength = SystemIO.IO_OS.FileLength(targetpath);
@@ -1045,7 +1043,7 @@ namespace Duplicati.Library.Main.Operation
 
                                             var key = Convert.ToBase64String(blockhasher.ComputeHash(blockbuffer, 0, size));
 
-                                            if (calcFileHashTask != null) calcFileHashTask.Await(); // wait because blockbuffer will be overwritten.
+                                            if (calcFileHashTask != null) await calcFileHashTask.ConfigureAwait(false); // wait because blockbuffer will be overwritten.
 
                                             if (key == targetblock.Hash)
                                             {
@@ -1099,7 +1097,7 @@ namespace Duplicati.Library.Main.Operation
                             if ((++updateCount) % 20 == 0)
                             {
                                 blockmarker.UpdateProcessed(result.OperationProgressUpdater);
-                                if (!result.TaskControl.ProgressRendevouz().Await())
+                                if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
                                     return;
                             }
                         }

Duplicati/Library/Main/Operation/TestHandler.cs

@@ -1,22 +1,22 @@
-// Copyright (C) 2025, The Duplicati Team
-// https://duplicati.com, hello@duplicati.com
-// 
-// Permission is hereby granted, free of charge, to any person obtaining a 
-// copy of this software and associated documentation files (the "Software"), 
-// to deal in the Software without restriction, including without limitation 
-// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
-// and/or sell copies of the Software, and to permit persons to whom the 
-// Software is furnished to do so, subject to the following conditions:
-// 
-// The above copyright notice and this permission notice shall be included in 
-// all copies or substantial portions of the Software.
-// 
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+// Copyright (C) 2025, The Duplicati Team
+// https://duplicati.com, hello@duplicati.com
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a 
+// copy of this software and associated documentation files (the "Software"), 
+// to deal in the Software without restriction, including without limitation 
+// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+// and/or sell copies of the Software, and to permit persons to whom the 
+// Software is furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in 
+// all copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
 // DEALINGS IN THE SOFTWARE.
 
 using System;
@@ -24,8 +24,9 @@ using Duplicati.Library.Main.Database;
 using System.Linq;
 using System.Collections.Generic;
 using Duplicati.Library.Interface;
-using System.Security.Cryptography;
 using Duplicati.Library.Utility;
+using System.Threading;
+using System.Threading.Tasks;
 
 namespace Duplicati.Library.Main.Operation
 {
@@ -37,36 +38,33 @@ namespace Duplicati.Library.Main.Operation
         private static readonly string LOGTAG = Logging.Log.LogTagFromType<TestHandler>();
 
         private readonly Options m_options;
-        private readonly string m_backendurl;
         private readonly TestResults m_results;
 
-        public TestHandler(string backendurl, Options options, TestResults results)
+        public TestHandler(Options options, TestResults results)
         {
             m_options = options;
-            m_backendurl = backendurl;
             m_results = results;
         }
 
-        public void Run(long samples)
+        public void Run(long samples, IBackendManager backendManager)
         {
             if (!System.IO.File.Exists(m_options.Dbpath))
                 throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist");
 
             using (var db = new LocalTestDatabase(m_options.Dbpath))
-            using (var backend = new BackendManager(m_backendurl, m_options, m_results.BackendWriter, db))
             {
                 db.SetResult(m_results);
                 Utility.UpdateOptionsFromDb(db, m_options);
                 Utility.VerifyOptionsAndUpdateDatabase(db, m_options);
                 db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, !m_options.DisableFilelistConsistencyChecks, null);
-                FilelistProcessor.VerifyRemoteList(backend, m_options, db, m_results.BackendWriter, true, null);
+                FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_results.BackendWriter, true, null).Await();
 
-                DoRun(samples, db, backend);
+                DoRun(samples, db, backendManager).Await();
                 db.WriteResults();
             }
         }
 
-        public void DoRun(long samples, LocalTestDatabase db, BackendManager backend)
+        public async Task DoRun(long samples, LocalTestDatabase db, IBackendManager backend)
         {
             var files = db.SelectTestTargets(samples, m_options).ToList();
 
@@ -76,13 +74,14 @@ namespace Duplicati.Library.Main.Operation
 
             if (m_options.FullRemoteVerification != Options.RemoteTestStrategy.False)
             {
-                foreach (var vol in new AsyncDownloader(files, backend))
+                await foreach (var (tf, hash, size, name) in backend.GetFilesOverlappedAsync(files, CancellationToken.None))
                 {
+                    var vol = new RemoteVolume(name, hash, size);
                     try
                     {
                         if (!m_results.TaskControl.ProgressRendevouz().Await())
                         {
-                            backend.WaitForComplete(db, null);
+                            backend.WaitForEmptyAsync(db, null, CancellationToken.None).Await();
                             m_results.EndTime = DateTime.UtcNow;
                             return;
                         }
@@ -91,7 +90,7 @@ namespace Duplicati.Library.Main.Operation
                         m_results.OperationProgressUpdater.UpdateProgress((float)progress / files.Count);
 
                         KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>> res;
-                        using (var tf = vol.TempFile)
+                        using (tf)
                             res = TestVolumeInternals(db, vol, tf, m_options, m_options.FullBlockVerification ? 1.0 : 0.2);
                         m_results.AddResult(res.Key, res.Value);
 
@@ -152,10 +151,10 @@ namespace Duplicati.Library.Main.Operation
                         {
                             Logging.Log.WriteInformationMessage(LOGTAG, "MissingRemoteHash", "No hash or size recorded for {0}, performing full verification", f.Name);
                             KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>> res;
-                            string hash;
-                            long size;
 
-                            using (var tf = backend.GetWithInfo(f.Name, out size, out hash))
+                            (var tf, var hash, var size) = backend.GetWithInfoAsync(f.Name, f.Hash, f.Size, CancellationToken.None).Await();
+
+                            using (tf)
                                 res = TestVolumeInternals(db, f, tf, m_options, 1);
                             m_results.AddResult(res.Key, res.Value);
 
@@ -176,16 +175,19 @@ namespace Duplicati.Library.Main.Operation
                             }
                         }
                         else
-                            backend.GetForTesting(f.Name, f.Size, f.Hash);
+                        {
+                            using (var tf = backend.GetAsync(f.Name, f.Hash, f.Size, CancellationToken.None).Await())
+                            { }
+                        }
 
                         db.UpdateVerificationCount(f.Name);
-                        m_results.AddResult(f.Name, new KeyValuePair<Duplicati.Library.Interface.TestEntryStatus, string>[0]);
+                        m_results.AddResult(f.Name, []);
                     }
                     catch (Exception ex)
                     {
-                        m_results.AddResult(f.Name, new KeyValuePair<Duplicati.Library.Interface.TestEntryStatus, string>[] { new KeyValuePair<Duplicati.Library.Interface.TestEntryStatus, string>(Duplicati.Library.Interface.TestEntryStatus.Error, ex.Message) });
+                        m_results.AddResult(f.Name, [new KeyValuePair<TestEntryStatus, string>(TestEntryStatus.Error, ex.Message)]);
                         Logging.Log.WriteErrorMessage(LOGTAG, "FailedToProcessFile", ex, "Failed to process file {0}", f.Name);
-                        if (ex is System.Threading.ThreadAbortException)
+                        if (ex is ThreadAbortException || ex is TaskCanceledException)
                         {
                             m_results.EndTime = DateTime.UtcNow;
                             throw;

Duplicati/Library/Main/Options.cs

@@ -1,22 +1,22 @@
-// Copyright (C) 2025, The Duplicati Team
-// https://duplicati.com, hello@duplicati.com
-// 
-// Permission is hereby granted, free of charge, to any person obtaining a 
-// copy of this software and associated documentation files (the "Software"), 
-// to deal in the Software without restriction, including without limitation 
-// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
-// and/or sell copies of the Software, and to permit persons to whom the 
-// Software is furnished to do so, subject to the following conditions:
-// 
-// The above copyright notice and this permission notice shall be included in 
-// all copies or substantial portions of the Software.
-// 
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+// Copyright (C) 2025, The Duplicati Team
+// https://duplicati.com, hello@duplicati.com
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a 
+// copy of this software and associated documentation files (the "Software"), 
+// to deal in the Software without restriction, including without limitation 
+// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+// and/or sell copies of the Software, and to permit persons to whom the 
+// Software is furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in 
+// all copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
 // DEALINGS IN THE SOFTWARE.
 
 using System;
@@ -382,12 +382,13 @@ namespace Duplicati.Library.Main
                     new CommandLineArgument("symlink-policy", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.SymlinkpolicyShort, Strings.Options.SymlinkpolicyLong("store", "ignore", "follow"), Enum.GetName(typeof(SymlinkStrategy), SymlinkStrategy.Store), null, Enum.GetNames(typeof(SymlinkStrategy))),
                     new CommandLineArgument("hardlink-policy", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.HardlinkpolicyShort, Strings.Options.HardlinkpolicyLong("first", "all", "none"), Enum.GetName(typeof(HardlinkStrategy), HardlinkStrategy.All), null, Enum.GetNames(typeof(HardlinkStrategy))),
                     new CommandLineArgument("exclude-files-attributes", CommandLineArgument.ArgumentType.String, Strings.Options.ExcludefilesattributesShort, Strings.Options.ExcludefilesattributesLong(Enum.GetNames(typeof(System.IO.FileAttributes)))),
-                    new CommandLineArgument("backup-name", CommandLineArgument.ArgumentType.String, Strings.Options.BackupnameShort, Strings.Options.BackupnameLong, DefaultBackupName),
                     new CommandLineArgument("compression-extension-file", CommandLineArgument.ArgumentType.Path, Strings.Options.CompressionextensionfileShort, Strings.Options.CompressionextensionfileLong(DEFAULT_COMPRESSED_EXTENSION_FILE), DEFAULT_COMPRESSED_EXTENSION_FILE),
 
-                    new CommandLineArgument("backup-id", CommandLineArgument.ArgumentType.String, Strings.Options.BackupidShort, Strings.Options.BackupidLong, ""),
                     new CommandLineArgument("machine-id", CommandLineArgument.ArgumentType.String, Strings.Options.MachineidShort, Strings.Options.MachineidLong, Library.AutoUpdater.UpdaterManager.InstallID),
                     new CommandLineArgument("machine-name", CommandLineArgument.ArgumentType.String, Strings.Options.MachinenameShort, Strings.Options.MachinenameLong, Library.AutoUpdater.UpdaterManager.MachineName),
+                    new CommandLineArgument("backup-id", CommandLineArgument.ArgumentType.String, Strings.Options.BackupidShort, Strings.Options.BackupidLong, ""),
+                    new CommandLineArgument("backup-name", CommandLineArgument.ArgumentType.String, Strings.Options.BackupnameShort, Strings.Options.BackupnameLong, DefaultBackupName),
+                    new CommandLineArgument("next-scheduled-run", CommandLineArgument.ArgumentType.String, Strings.Options.NextscheduledrunShort, Strings.Options.NextscheduledrunLong),
 
                     new CommandLineArgument("verbose", CommandLineArgument.ArgumentType.Boolean, Strings.Options.VerboseShort, Strings.Options.VerboseLong, "false", null, null, Strings.Options.VerboseDeprecated),
                     new CommandLineArgument("full-result", CommandLineArgument.ArgumentType.Boolean, Strings.Options.FullresultShort, Strings.Options.FullresultLong, "false"),

Duplicati/Library/Main/ResultClasses.cs

@@ -68,6 +68,11 @@ namespace Duplicati.Library.Main
         ITaskControl TaskControl { get; }
     }
 
+    internal interface IBackendWriterProvider
+    {
+        IBackendWriter BackendWriter { get; }
+    }
+
     internal class BackendWriter : BasicResults, IBackendWriter, IBackendStatstics, IParsedBackendStatistics
     {
         public BackendWriter(BasicResults p) : base(p) { }
@@ -145,7 +150,7 @@ namespace Duplicati.Library.Main
     }
 
 
-    internal abstract class BasicResults : IBasicResults, ISetCommonOptions, Logging.ILogDestination, ITaskControlProvider
+    internal abstract class BasicResults : IBasicResults, ISetCommonOptions, Logging.ILogDestination, ITaskControlProvider, IBackendWriterProvider
     {
         /// <summary>
         /// The tag used for logging

Duplicati/Library/Main/Strings.cs

@@ -1,22 +1,22 @@
-// Copyright (C) 2025, The Duplicati Team
-// https://duplicati.com, hello@duplicati.com
-// 
-// Permission is hereby granted, free of charge, to any person obtaining a 
-// copy of this software and associated documentation files (the "Software"), 
-// to deal in the Software without restriction, including without limitation 
-// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
-// and/or sell copies of the Software, and to permit persons to whom the 
-// Software is furnished to do so, subject to the following conditions:
-// 
-// The above copyright notice and this permission notice shall be included in 
-// all copies or substantial portions of the Software.
-// 
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+// Copyright (C) 2025, The Duplicati Team
+// https://duplicati.com, hello@duplicati.com
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a 
+// copy of this software and associated documentation files (the "Software"), 
+// to deal in the Software without restriction, including without limitation 
+// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+// and/or sell copies of the Software, and to permit persons to whom the 
+// Software is furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in 
+// all copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
 // DEALINGS IN THE SOFTWARE.
 using System;
 using Duplicati.Library.Localization.Short;
@@ -171,6 +171,8 @@ namespace Duplicati.Library.Main.Strings
         public static string MachineidShort { get { return LC.L(@"Machine ID"); } }
         public static string MachinenameLong { get { return LC.L(@"The name of the machine running the backup. This can be used to identify the machine when sending mail or running scripts."); } }
         public static string MachinenameShort { get { return LC.L(@"Machine name"); } }
+        public static string NextscheduledrunShort { get { return LC.L(@"The time of the next scheduled run"); } }
+        public static string NextscheduledrunLong { get { return LC.L(@"This property is a reporting option and does not affect the actual scheduled time. Use this option to inform a reporting destination about the next expected time the backup will run."); } }
         public static string CompressionextensionfileLong(string path) { return LC.L(@"Use this option to point to a text file where each line contains a file extension that indicates a non-compressible file. Files that have an extension found in the file will not be compressed, but simply stored in the archive. The file format ignores any lines that do not start with a period, and considers a space to indicate the end of the extension. A default file is supplied, that also serves as an example. The default file is placed in {0}.", path); }
         public static string CompressionextensionfileShort { get { return LC.L(@"Manage non-compressible file extensions"); } }
         public static string BlocksizeLong { get { return LC.L(@"The block size determines how files are fragmented. Choosing a large value will cause a larger overhead on file changes, choosing a small value will cause a large overhead on storage of file lists. Note that the value cannot be changed after remote files are created."); } }
@@ -268,7 +270,7 @@ namespace Duplicati.Library.Main.Strings
         public static string ConcurrencyblockhashersShort { get { return LC.L(@"Specify the number of concurrent hashing processes"); } }
         public static string ConcurrencycompressorsLong { get { return LC.L(@"Use this option to set the number of processes that perform compression of output data."); } }
         public static string ConcurrencycompressorsShort { get { return LC.L(@"Specify the number of concurrent compression processes"); } }
-        public static string ConcurrencyfileprocessorsShort{ get { return LC.L(@"[EXPERIMENTAL]Specify the number of concurrent files to open"); } }
+        public static string ConcurrencyfileprocessorsShort { get { return LC.L(@"[EXPERIMENTAL]Specify the number of concurrent files to open"); } }
         public static string ConcurrencyfileprocessorsLong { get { return LC.L(@"Use this option to set the number of concurrent files to open. This could accelerate big backups involving lot of files, such as an initial backup"); } }
         public static string DisablesyntehticfilelistLong { get { return LC.L(@"If Duplicati detects that the previous backup did not complete, it will generate a filelist that is a merge of the last completed backup and the contents that were uploaded in the incomplete backup session."); } }
         public static string DisablesyntheticfilelistShort { get { return LC.L(@"Disable synthetic filelist"); } }
@@ -355,4 +357,9 @@ namespace Duplicati.Library.Main.Strings
         public static string SnapshotFailedError(string message) { return LC.L(@"Failed to create a snapshot: {0}", message); }
     }
 
+    internal static class BackendMananger
+    {
+        public static string EncryptionModuleNotFound(string name) { return LC.L(@"The encryption module {0} was not found", name); }
+    }
+
 }

Duplicati/Library/Main/Volumes/IParsedVolume.cs

@@ -18,21 +18,18 @@
 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
 // DEALINGS IN THE SOFTWARE.
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-
-namespace Duplicati.Library.Main.Volumes
-{
-    public interface IParsedVolume
-    {
-        RemoteVolumeType FileType { get; }
-        string Prefix { get; }
-        string Guid { get; }
-        DateTime Time { get; }
-        string CompressionModule { get; }
-        string EncryptionModule { get; }
-        Library.Interface.IFileEntry File { get; }
-    }
-}
+using System;
+
+namespace Duplicati.Library.Main.Volumes
+{
+    public interface IParsedVolume
+    {
+        RemoteVolumeType FileType { get; }
+        string Prefix { get; }
+        string Guid { get; }
+        DateTime Time { get; }
+        string CompressionModule { get; }
+        string EncryptionModule { get; }
+        Interface.IFileEntry File { get; }
+    }
+}

Duplicati/Library/Main/Volumes/VolumeWriterBase.cs

@@ -89,15 +89,6 @@ namespace Duplicati.Library.Main.Volumes
                 sr.Write(ManifestData.GetManifestInstance(m_blocksize, m_blockhash, m_filehash));
         }
 
-        internal BackendHandler.FileEntryItem CreateFileEntryForUpload(Options options)
-        {
-            var fileEntry = new BackendHandler.FileEntryItem(BackendActionType.Put, this.RemoteFilename);
-            fileEntry.SetLocalfilename(this.LocalFilename);
-            fileEntry.Encrypt(options);
-            fileEntry.UpdateHashAndSize(options);
-            return fileEntry;
-        }
-
         public virtual void Dispose()
         {
             if (m_compression != null)

Duplicati/Library/Modules/Builtin/ReportHelper.cs

@@ -315,6 +315,108 @@ namespace Duplicati.Library.Modules.Builtin
         protected virtual string ReplaceTemplate(string input, object result, Exception exception, bool subjectline, ResultExportFormat format)
             => ReplaceTemplate(input, result, exception, subjectline, ResultFormatSerializerProvider.GetSerializer(format));
 
+        /// <summary>
+        /// The operation name template key
+        /// </summary>
+        private const string OPERATIONNAME = "OperationName";
+        /// <summary>
+        /// The remote url template key
+        /// </summary>
+        private const string REMOTEURL = "RemoteUrl";
+        /// <summary>
+        /// The local path template key
+        /// </summary>
+        private const string LOCALPATH = "LocalPath";
+        /// <summary>
+        /// The parsed result template key
+        /// </summary>
+        private const string PARSEDRESULT = "ParsedResult";
+        /// <summary>
+        /// The machine id template key
+        /// </summary>
+        private const string MACHINE_ID = "machine-id";
+        /// <summary>
+        /// The backup id template key
+        /// </summary>
+        private const string BACKUP_ID = "backup-id";
+        /// <summary>
+        /// The backup name template key
+        /// </summary>
+        private const string BACKUP_NAME = "backup-name";
+        /// <summary>
+        /// The machine name template key
+        /// </summary>
+        private const string MACHINE_NAME = "machine-name";
+        /// <summary>
+        /// The operating system template key
+        /// </summary>
+        private const string OPERATING_SYSTEM = "operating-system";
+        /// <summary>
+        /// The installation type template key
+        /// </summary>
+        private const string INSTALLATION_TYPE = "installation-type";
+        /// <summary>
+        /// The destination type template key
+        /// </summary>
+        private const string DESTINATION_TYPE = "destination-type";
+        /// <summary>
+        /// The next scheduled run template key
+        /// </summary>
+        private const string NEXT_SCHEDULED_RUN = "next-scheduled-run";
+
+        /// <summary>
+        /// The list of regular template keys
+        /// </summary>
+        private static readonly IReadOnlySet<string> OPERATION_TEMPLATE_KEYS = new HashSet<string>([
+            OPERATIONNAME, REMOTEURL, LOCALPATH, PARSEDRESULT
+        ], StringComparer.OrdinalIgnoreCase);
+
+        /// <summary>
+        /// The list of extra template keys
+        /// </summary>
+        private static readonly IReadOnlySet<string> EXTRA_TEMPLATE_KEYS = new HashSet<string>([
+            MACHINE_ID, BACKUP_ID, BACKUP_NAME, MACHINE_NAME,
+            OPERATING_SYSTEM, INSTALLATION_TYPE, DESTINATION_TYPE, NEXT_SCHEDULED_RUN
+        ], StringComparer.OrdinalIgnoreCase);
+
+        /// <summary>
+        /// Gets the default value for a template key
+        /// </summary>
+        /// <param name="name">The name of the template key</param>
+        /// <returns>The default value</returns>
+        private string GetDefaultValue(string name)
+        {
+            switch (name)
+            {
+                case OPERATIONNAME:
+                    return m_operationname;
+                case REMOTEURL:
+                    return m_remoteurl;
+                case LOCALPATH:
+                    return m_localpath == null ? "" : string.Join(System.IO.Path.PathSeparator.ToString(), m_localpath);
+                case PARSEDRESULT:
+                    return m_parsedresultlevel;
+                case MACHINE_ID:
+                    return AutoUpdater.UpdaterManager.MachineID;
+                case BACKUP_ID:
+                    return Utility.Utility.ByteArrayAsHexString(Utility.Utility.RepeatedHashWithSalt(m_remoteurl, SALT));
+                case BACKUP_NAME:
+                    return System.IO.Path.GetFileNameWithoutExtension(Utility.Utility.getEntryAssembly().Location);
+                case MACHINE_NAME:
+                    return AutoUpdater.UpdaterManager.MachineName;
+                case OPERATING_SYSTEM:
+                    return OperatingSystem.IsWindows() ? "Windows" : OperatingSystem.IsLinux() ? "Linux" : OperatingSystem.IsMacOS() ? "MacOS" : "Unknown";
+                case INSTALLATION_TYPE:
+                    return AutoUpdater.UpdaterManager.PackageTypeId;
+                case DESTINATION_TYPE:
+                    // Only return the url scheme, as the rest could contain sensitive information
+                    var ix = m_remoteurl?.IndexOf("://", StringComparison.OrdinalIgnoreCase) ?? -1;
+                    return ix >= 0 && ix < 15 ? m_remoteurl.Substring(0, ix) : "file";
+                default:
+                    return null;
+            }
+        }
+
         /// <summary>
         /// Helper method to perform template expansion
         /// </summary>
@@ -331,55 +433,29 @@ namespace Duplicati.Library.Modules.Builtin
             {
                 var extra = new Dictionary<string, string>();
 
-                if (input.IndexOf("%OPERATIONNAME%", StringComparison.OrdinalIgnoreCase) >= 0)
-                    extra["OperationName"] = m_operationname;
-                if (input.IndexOf("%REMOTEURL%", StringComparison.OrdinalIgnoreCase) >= 0)
-                    extra["RemoteUrl"] = m_remoteurl;
-                if (input.IndexOf("%LOCALPATH%", StringComparison.OrdinalIgnoreCase) >= 0 && m_localpath != null)
-                    extra["LocalPath"] = string.Join(System.IO.Path.PathSeparator.ToString(), m_localpath);
-                if (input.IndexOf("%PARSEDRESULT%", StringComparison.OrdinalIgnoreCase) >= 0)
-                    extra["ParsedResult"] = m_parsedresultlevel;
-
-                // If the options contains the key, it is captured by the loop over m_options
-                // so we only patch it in case it is missing
-
-                if (input.IndexOf("%machine-id%", StringComparison.OrdinalIgnoreCase) >= 0)
-                {
-                    if (!m_options.ContainsKey("machine-id"))
-                        extra["machine-id"] = Library.AutoUpdater.UpdaterManager.MachineID;
-                }
-
-                if (input.IndexOf("%backup-id%", StringComparison.OrdinalIgnoreCase) >= 0)
-                {
-                    if (!m_options.ContainsKey("backup-id"))
-                        extra["backup-id"] = Library.Utility.Utility.ByteArrayAsHexString(Library.Utility.Utility.RepeatedHashWithSalt(m_remoteurl, SALT));
-                }
-
-                if (input.IndexOf("%backup-name%", StringComparison.OrdinalIgnoreCase) >= 0)
-                {
-                    if (!m_options.ContainsKey("backup-name"))
-                        extra["backup-name"] = System.IO.Path.GetFileNameWithoutExtension(Duplicati.Library.Utility.Utility.getEntryAssembly().Location);
-                }
-
-                if (input.IndexOf("%machine-name%", StringComparison.OrdinalIgnoreCase) >= 0)
-                {
-                    if (!m_options.ContainsKey("machine-name"))
-                        extra["machine-name"] = Library.AutoUpdater.UpdaterManager.MachineName;
-                }
+                // Add the default values, if found in the template
+                foreach (var key in OPERATION_TEMPLATE_KEYS)
+                    if (input.IndexOf($"%{key}%", StringComparison.OrdinalIgnoreCase) >= 0)
+                        extra[key] = GetDefaultValue(key);
 
+                // Add any options that are whitelisted or used in the template
                 foreach (KeyValuePair<string, string> kv in m_options)
-                    if (input.IndexOf($"%{kv.Key}%", StringComparison.OrdinalIgnoreCase) >= 0)
+                    if (EXTRA_TEMPLATE_KEYS.Contains(kv.Key) || input.IndexOf($"%{kv.Key}%", StringComparison.OrdinalIgnoreCase) >= 0)
                         extra[kv.Key] = kv.Value;
 
+                // Add any missing default values
+                foreach (var key in EXTRA_TEMPLATE_KEYS)
+                    if (!extra.ContainsKey(key))
+                        extra[key] = GetDefaultValue(key);
+
                 return resultFormatSerializer.Serialize(result, exception, LogLines, extra);
             }
             else
             {
 
-                input = Regex.Replace(input, "\\%OPERATIONNAME\\%", m_operationname ?? "", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
-                input = Regex.Replace(input, "\\%REMOTEURL\\%", m_remoteurl ?? "", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
-                input = Regex.Replace(input, "\\%LOCALPATH\\%", m_localpath == null ? "" : string.Join(System.IO.Path.PathSeparator.ToString(), m_localpath), RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
-                input = Regex.Replace(input, "\\%PARSEDRESULT\\%", m_parsedresultlevel ?? "", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
+                foreach (var key in OPERATION_TEMPLATE_KEYS)
+                    input = Regex.Replace(input, $"%{key}%", GetDefaultValue(key) ?? "", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
+
                 if (subjectline)
                 {
                     input = Regex.Replace(input, "\\%RESULT\\%", m_parsedresultlevel ?? "", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
@@ -393,18 +469,11 @@ namespace Duplicati.Library.Modules.Builtin
                 foreach (KeyValuePair<string, string> kv in m_options)
                     input = Regex.Replace(input, "\\%" + kv.Key + "\\%", kv.Value ?? "", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
 
-                if (!m_options.ContainsKey("backup-name"))
-                    input = Regex.Replace(input, "\\%backup-name\\%", System.IO.Path.GetFileNameWithoutExtension(Duplicati.Library.Utility.Utility.getEntryAssembly().Location) ?? "", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
-
-                if (!m_options.ContainsKey("machine-name"))
-                    input = Regex.Replace(input, "\\%machine-name\\%", Library.AutoUpdater.UpdaterManager.MachineName, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
-
-                if (!m_options.ContainsKey("backup-id"))
-                    input = Regex.Replace(input, "\\%backup-id\\%", Library.Utility.Utility.ByteArrayAsHexString(Library.Utility.Utility.RepeatedHashWithSalt(m_remoteurl, SALT)), RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
-
-                if (!m_options.ContainsKey("machine-id"))
-                    input = Regex.Replace(input, "\\%machine-id\\%", Library.AutoUpdater.UpdaterManager.MachineID, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
+                foreach (var key in EXTRA_TEMPLATE_KEYS)
+                    if (!m_options.ContainsKey(key))
+                        input = Regex.Replace(input, $"%{key}%", GetDefaultValue(key) ?? "", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
 
+                // Remove any remaining template keys
                 input = Regex.Replace(input, "\\%[^\\%]+\\%", "");
                 return input;
             }

Duplicati/Library/RestAPI/Scheduler.cs

@@ -357,8 +357,23 @@ namespace Duplicati.Server
                                         }
                                         else
                                         {
+                                            Dictionary<string, string> taskOptions = null;
+                                            try
+                                            {
+                                                var nextRun = GetNextValidTime(start,
+                                                    new DateTime(
+                                                        Math.Max(DateTime.UtcNow.AddSeconds(1).Ticks, start.AddSeconds(1).Ticks),
+                                                        DateTimeKind.Utc), sc.Repeat, sc.AllowedDays, timeZoneInfo);
+
+                                                taskOptions = new Dictionary<string, string>()
+                                                { { "next-scheduled-run", Utility.SerializeDateTime(nextRun.ToUniversalTime()) } };
+                                            }
+                                            catch
+                                            {
+                                            }
+
                                             jobsToRun.Add(Server.Runner.CreateTask(
-                                                Duplicati.Server.Serialization.DuplicatiOperation.Backup, entry));
+                                                Serialization.DuplicatiOperation.Backup, entry, taskOptions));
                                         }
                                     }
                                 }

Duplicati/Library/Utility/MachineNameReader.cs

@@ -0,0 +1,95 @@
+using System;
+using System.Management;
+using System.Runtime.Versioning;
+
+#nullable enable
+
+namespace Duplicati.Library.Utility;
+
+/// <summary>
+/// Class for reading the machine name
+/// </summary>
+public static class MachineNameReader
+{
+    /// <summary>
+    /// Makes a best effort to get the machine name
+    /// </summary>
+    /// <returns>The machine name</returns>
+    public static string GetMachineName()
+    {
+        string? machineName = null;
+        if (OperatingSystem.IsWindows())
+            machineName = GetMachineNameWindows();
+        else if (OperatingSystem.IsMacOS())
+            machineName = GetMachineNameMacOS();
+        else if (OperatingSystem.IsLinux())
+            machineName = GetMachineNameLinux();
+
+        return string.IsNullOrWhiteSpace(machineName)
+            ? Environment.MachineName
+            : machineName;
+    }
+    /// <summary>
+    /// Executes a command and reads the output
+    /// </summary>
+    /// <param name="command">The command to execute</param>
+    /// <param name="arguments">The arguments to pass to the command</param>
+    /// <returns>The output of the command</returns>
+    private static string ExecuteAndReadOutput(string command, string arguments)
+    {
+        try
+        {
+            var process = new System.Diagnostics.Process
+            {
+                StartInfo = new System.Diagnostics.ProcessStartInfo
+                {
+                    FileName = command,
+                    Arguments = arguments,
+                    RedirectStandardOutput = true,
+                    UseShellExecute = false,
+                    CreateNoWindow = true
+                }
+            };
+            process.Start();
+            process.WaitForExit(TimeSpan.FromSeconds(1));
+            if (!process.HasExited)
+            {
+                process.Kill();
+                return string.Empty;
+            }
+
+            if (process.ExitCode != 0)
+                return string.Empty;
+
+            return process.StandardOutput.ReadToEnd().Trim();
+        }
+        catch
+        {
+            return string.Empty;
+        }
+    }
+
+    /// <summary>
+    /// Gets the machine name if running MacOS
+    /// </summary>
+    /// <returns>The machine name</returns>
+    [SupportedOSPlatform("macos")]
+    private static string? GetMachineNameMacOS()
+        => ExecuteAndReadOutput("scutil", "--get ComputerName");
+
+    /// <summary>
+    /// Gets the machine name if running Windows
+    /// </summary>
+    /// <returns>The machine name</returns>
+    [SupportedOSPlatform("windows")]
+    private static string? GetMachineNameWindows()
+        => null; // No special handling for Windows, always uses NetBIOS name
+
+    /// <summary>
+    /// Gets the machine name if running Linux
+    /// </summary>
+    /// <returns>The machine name</returns>
+    [SupportedOSPlatform("linux")]
+    private static string? GetMachineNameLinux()
+        => null; // No special handling for Linux, always uses hostname
+}

Duplicati/Library/Utility/Utility.cs

@@ -1534,5 +1534,26 @@ namespace Duplicati.Library.Utility
             collection.Import(pfxPath, password, X509KeyStorageFlags.Exportable | X509KeyStorageFlags.PersistKeySet);
             return collection;
         }
+
+        /// <summary>
+        /// Flattens an exception and its inner exceptions
+        /// </summary>
+        /// <param name="ex">The exception to flatten</param>
+        /// <returns>An enumerable of exceptions</returns>
+        public static IEnumerable<Exception> FlattenException(Exception? ex)
+        {
+            if (ex == null)
+                yield break;
+
+            yield return ex;
+
+            if (ex is AggregateException aex)
+                foreach (var iex in aex.Flatten().InnerExceptions)
+                    foreach (var iex2 in FlattenException(iex))
+                        yield return iex2;
+
+            foreach (var iex in FlattenException(ex.InnerException))
+                yield return iex;
+        }
     }
 }

Duplicati/UnitTest/BorderTests.cs

@@ -1,22 +1,22 @@
-// Copyright (C) 2025, The Duplicati Team
-// https://duplicati.com, hello@duplicati.com
-// 
-// Permission is hereby granted, free of charge, to any person obtaining a 
-// copy of this software and associated documentation files (the "Software"), 
-// to deal in the Software without restriction, including without limitation 
-// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
-// and/or sell copies of the Software, and to permit persons to whom the 
-// Software is furnished to do so, subject to the following conditions:
-// 
-// The above copyright notice and this permission notice shall be included in 
-// all copies or substantial portions of the Software.
-// 
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+// Copyright (C) 2025, The Duplicati Team
+// https://duplicati.com, hello@duplicati.com
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a 
+// copy of this software and associated documentation files (the "Software"), 
+// to deal in the Software without restriction, including without limitation 
+// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+// and/or sell copies of the Software, and to permit persons to whom the 
+// Software is furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in 
+// all copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
 // DEALINGS IN THE SOFTWARE.
 
 using System;
@@ -30,17 +30,6 @@ namespace Duplicati.UnitTest
 {
     public class BorderTests : BasicSetupHelper
     {
-        private readonly string recreatedDatabaseFile = Path.Combine(BASEFOLDER, "recreated-database.sqlite");
-
-        [TearDown]
-        public void TearDown()
-        {
-            if (File.Exists(this.recreatedDatabaseFile))
-            {
-                File.Delete(this.recreatedDatabaseFile);
-            }
-        }
-
         [Test]
         [Category("Border")]
         public void Run10kNoProgress()
@@ -317,15 +306,17 @@ namespace Duplicati.UnitTest
                 Assert.IsFalse(r.Verifications.Any(p => p.Value.Any()));
             }
 
-            testopts["dbpath"] = this.recreatedDatabaseFile;
+            var recreatedDatabaseFile = Path.Combine(BASEFOLDER, "recreated-database.sqlite");
+            if (File.Exists(recreatedDatabaseFile))
+                File.Delete(recreatedDatabaseFile);
+
+            testopts["dbpath"] = recreatedDatabaseFile;
 
             using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
             {
                 IRepairResults repairResults = c.Repair();
                 Assert.AreEqual(0, repairResults.Errors.Count());
-
-                // TODO: This sometimes results in a "No block hash found for file: C:\projects\duplicati\testdata\backup-data\a-0" warning.
-                // Because of this, we don't check for warnings here.
+                Assert.AreEqual(0, repairResults.Warnings.Count());
             }
 
             using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))

Duplicati/UnitTest/CommandLineOperationsTests.cs

@@ -1,22 +1,22 @@
-// Copyright (C) 2025, The Duplicati Team
-// https://duplicati.com, hello@duplicati.com
-// 
-// Permission is hereby granted, free of charge, to any person obtaining a 
-// copy of this software and associated documentation files (the "Software"), 
-// to deal in the Software without restriction, including without limitation 
-// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
-// and/or sell copies of the Software, and to permit persons to whom the 
-// Software is furnished to do so, subject to the following conditions:
-// 
-// The above copyright notice and this permission notice shall be included in 
-// all copies or substantial portions of the Software.
-// 
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+// Copyright (C) 2025, The Duplicati Team
+// https://duplicati.com, hello@duplicati.com
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a 
+// copy of this software and associated documentation files (the "Software"), 
+// to deal in the Software without restriction, including without limitation 
+// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+// and/or sell copies of the Software, and to permit persons to whom the 
+// Software is furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in 
+// all copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
 // DEALINGS IN THE SOFTWARE.
 
 using System;
@@ -32,7 +32,7 @@ namespace Duplicati.UnitTest
 {
     public class CommandLineOperationsTests : BasicSetupHelper
     {
-        private static readonly string S3_URL = $"https://testfiles.duplicati.com/";
+        public const string S3_URL = $"https://testfiles.duplicati.com/";
 
         /// <summary>
         /// The log tag
@@ -79,7 +79,7 @@ namespace Duplicati.UnitTest
         private void DownloadS3FileIfNewer(string destinationFilePath, string url, int retries = 5)
             => DownloadS3FileIfNewerAsync(destinationFilePath, url, retries).Await();
 
-        private async Task DownloadS3FileIfNewerAsync(string destinationFilePath, string url, int retries = 5)
+        public static async Task DownloadS3FileIfNewerAsync(string destinationFilePath, string url, int retries = 5)
         {
             do
             {
@@ -92,7 +92,6 @@ namespace Duplicati.UnitTest
                         request.Headers.IfModifiedSince = systemIO.FileGetLastWriteTimeUtc(destinationFilePath);
 
                     using var response = await httpClient.SendAsync(request);
-                    using var tmpFile = new TempFile();
 
                     if (response.StatusCode == System.Net.HttpStatusCode.NotModified)
                     {
@@ -101,6 +100,7 @@ namespace Duplicati.UnitTest
                     }
                     else
                     {
+                        using var tmpFile = new TempFile();
                         Console.WriteLine($"Downloading file from {url} to: {tmpFile}");
                         var contentStream = await response.Content.ReadAsStreamAsync();
                         var fileInfo = new FileInfo(tmpFile);
@@ -126,6 +126,13 @@ namespace Duplicati.UnitTest
                         throw;
 
                     await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
+                    try
+                    {
+                        System.Net.Dns.GetHostEntry(new System.Uri(url).Host);
+                    }
+                    catch (Exception)
+                    {
+                    }
                     Console.WriteLine($"Retrying download, {retries} retries left.");
                 }
             } while (retries > 0);

Duplicati/UnitTest/DisruptionTests.cs

@@ -65,8 +65,9 @@ namespace Duplicati.UnitTest
                 {
                     if (path.EndsWith(this.fileSizes[2] + "MB"))
                     {
-                        stopped.TrySetResult(true);
+                        Thread.Sleep(500);
                         controller.Stop();
+                        stopped.TrySetResult(true);
                     }
                 };
 #endif
@@ -102,7 +103,8 @@ namespace Duplicati.UnitTest
 
                 // This allows us to inspect the dlist files without needing the BackendManager (which is inaccessible here) to decrypt them.
                 ["no-encryption"] = "true",
-                ["disable-file-scanner"] = "true"
+                ["disable-file-scanner"] = "true",
+                ["concurrency-fileprocessors"] = "1",
             };
 
             // Run a full backup.
@@ -206,7 +208,8 @@ namespace Duplicati.UnitTest
             Dictionary<string, string> options = new Dictionary<string, string>(this.TestOptions)
             {
                 ["dblock-size"] = "10mb",
-                ["disable-file-scanner"] = "true"
+                ["disable-file-scanner"] = "true",
+                ["concurrency-fileprocessors"] = "1",
             };
 
             // First, run two complete backups followed by a partial backup. We will then set the keep-time
@@ -300,6 +303,7 @@ namespace Duplicati.UnitTest
             {
                 ["dblock-size"] = "10mb",
                 ["disable-file-scanner"] = "true",
+                ["concurrency-fileprocessors"] = "1",
             };
 
             // Run a full backup.
@@ -394,7 +398,8 @@ namespace Duplicati.UnitTest
             {
                 ["dblock-size"] = "10mb",
                 ["no-local-db"] = "true",
-                ["disable-file-scanner"] = "true"
+                ["disable-file-scanner"] = "true",
+                ["concurrency-fileprocessors"] = "1",
             };
 
             // Run a full backup.
@@ -435,7 +440,8 @@ namespace Duplicati.UnitTest
                 // This test assumes that we can perform 3 backups within 1 minute.
                 ["retention-policy"] = "1m:59s,U:1m",
                 ["no-backend-verification"] = "true",
-                ["disable-file-scanner"] = "true"
+                ["disable-file-scanner"] = "true",
+                ["concurrency-fileprocessors"] = "1"
             };
 
             DateTime firstBackupTime;
@@ -573,6 +579,7 @@ namespace Duplicati.UnitTest
                 // Choose a dblock size that is small enough so that more than one volume is needed.
                 ["dblock-size"] = "10mb",
                 ["disable-file-scanner"] = "true",
+                ["concurrency-fileprocessors"] = "1"
             };
 
             // Run a complete backup.
@@ -676,7 +683,8 @@ namespace Duplicati.UnitTest
             {
                 ["dblock-size"] = "10mb",
                 ["disable-synthetic-filelist"] = "true",
-                ["disable-file-scanner"] = "true"
+                ["disable-file-scanner"] = "true",
+                ["concurrency-fileprocessors"] = "1",
             };
 
             // Run a complete backup.

Duplicati/UnitTest/GeneralBlackBoxTesting.cs

@@ -1,30 +1,29 @@
-// Copyright (C) 2025, The Duplicati Team
-// https://duplicati.com, hello@duplicati.com
-// 
-// Permission is hereby granted, free of charge, to any person obtaining a 
-// copy of this software and associated documentation files (the "Software"), 
-// to deal in the Software without restriction, including without limitation 
-// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
-// and/or sell copies of the Software, and to permit persons to whom the 
-// Software is furnished to do so, subject to the following conditions:
-// 
-// The above copyright notice and this permission notice shall be included in 
-// all copies or substantial portions of the Software.
-// 
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+// Copyright (C) 2025, The Duplicati Team
+// https://duplicati.com, hello@duplicati.com
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a 
+// copy of this software and associated documentation files (the "Software"), 
+// to deal in the Software without restriction, including without limitation 
+// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+// and/or sell copies of the Software, and to permit persons to whom the 
+// Software is furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in 
+// all copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
 // DEALINGS IN THE SOFTWARE.
 
 using NUnit.Framework;
 using System.Linq;
 using System.IO;
-using System;
 using System.Collections.Generic;
-using System.Net;
+using Duplicati.Library.Utility;
 
 namespace Duplicati.UnitTest
 {
@@ -70,53 +69,7 @@ namespace Duplicati.UnitTest
         public void OneTimeSetUp()
         {
             this.OneTimeTearDown();
-            var url = $"https://testfiles.duplicati.com/{this.zipFilename}";
-            var destinationFilePath = this.zipFilepath;
-            var webRequest = (HttpWebRequest)WebRequest.Create(url);
-
-            using (var wr = (HttpWebResponse)webRequest.GetResponse())
-            using (WebClient client = new WebClient())
-            {
-                Console.WriteLine("downloading test file to: {0}, length: {1}", destinationFilePath, wr.ContentLength);
-                var maxAttempts = 5;
-                while (maxAttempts-- > 0)
-                {
-                    try
-                    {
-                        DateTime beginTime = DateTime.Now;
-                        client.DownloadFile(url, destinationFilePath);
-                        long length = new System.IO.FileInfo(destinationFilePath).Length;
-                        Console.WriteLine("downloaded test file: {0}: length {1}, duration {2}", destinationFilePath, length, (DateTime.Now - beginTime).TotalSeconds);
-                        if (length == wr.ContentLength)
-                        {
-                            maxAttempts = -1;
-                        }
-                        else
-                        {
-                            Console.WriteLine("invalid downloaded length {0}, should be {1}...", length, wr.ContentLength);
-                            System.Threading.Thread.Sleep(120000);
-                        }
-                    }
-                    catch (WebException ex)
-                    {
-                        if (ex.Response == null)
-                        {
-                            Console.WriteLine("ex.Response is null !");
-                        }
-                        else
-                        {
-                            Console.WriteLine("exception {0}", ex);
-                            throw;
-                        }
-                    }
-                }
-
-                if (maxAttempts == 0)
-                {
-                    throw new Exception(string.Format("Unable to download test file from {0}", url));
-                }
-            }
-
+            CommandLineOperationsTests.DownloadS3FileIfNewerAsync(this.zipFilepath, $"{CommandLineOperationsTests.S3_URL}{this.zipFilename}").Await();
             System.IO.Compression.ZipFile.ExtractToDirectory(this.zipFilepath, BASEFOLDER);
         }
 
@@ -127,10 +80,6 @@ namespace Duplicati.UnitTest
             {
                 Directory.Delete(SOURCE_FOLDERS, true);
             }
-            if (File.Exists(this.zipFilepath))
-            {
-                File.Delete(this.zipFilepath);
-            }
         }
 
         [Test]

LiveTests/Duplicati.Backend.Tests/FTP/FTPTests.cs

@@ -50,7 +50,7 @@ public sealed class FtpTests : BaseSftpgoTest
             .WithEnvironment("SFTPGO_LOADDATA_FROM", "/var/lib/sftpgo/users.json")
             .WithEnvironment("SFTPGO_LOADDATA_CLEAN", "0")
             .WithEnvironment("SFTPGO_FTPD__BINDINGS__0__PORT", "2121")
-            .WithEnvironment("SFTPGO_FTPD__PASSIVE_PORT_RANGE__END", "21000")
+            .WithEnvironment("SFTPGO_FTPD__PASSIVE_PORT_RANGE__END", "20999")
             .WithEnvironment("SFTPGO_FTPD__PASSIVE_PORT_RANGE__START", "20000")
             .WithEnvironment("SFTPGO_FTPD__BINDINGS__0__ACTIVE_CONNECTIONS_SECURITY", "1")
             .WithEnvironment("SFTPGO_FTPD__BINDINGS__0__CERTIFICATE_FILE", CERTIFICATE_FILE)
@@ -60,13 +60,13 @@ public sealed class FtpTests : BaseSftpgoTest
 
             .WithOutputConsumer(outputConsumer);
 
-        container = Enumerable.Range(20000, 1000)
+        container = Enumerable.Range(20000, 1001)
             .Aggregate(container, (current, port) => current.WithPortBinding(port, port));
 
         var builtContainer = container
             .WithOutputConsumer(outputConsumer)
             .Build();
-        
+
         Console.WriteLine("Starting container");
         await builtContainer.StartAsync();
 
@@ -88,7 +88,7 @@ public sealed class FtpTests : BaseSftpgoTest
             Console.WriteLine(await outputConsumer.GetStreamsOutput());
             Assert.Fail("BackendTester is returning non-zero exit code, check logs for details");
         }
-        
+
         await builtContainer.StopAsync();
     }
 
@@ -114,7 +114,7 @@ public sealed class FtpTests : BaseSftpgoTest
             .WithEnvironment("SFTPGO_LOADDATA_FROM", "/var/lib/sftpgo/users.json")
             .WithEnvironment("SFTPGO_LOADDATA_CLEAN", "0")
             .WithEnvironment("SFTPGO_FTPD__BINDINGS__0__PORT", "2121")
-            .WithEnvironment("SFTPGO_FTPD__PASSIVE_PORT_RANGE__END", "23000")
+            .WithEnvironment("SFTPGO_FTPD__PASSIVE_PORT_RANGE__END", "22999")
             .WithEnvironment("SFTPGO_FTPD__PASSIVE_PORT_RANGE__START", "22000")
             .WithEnvironment("SFTPGO_FTPD__BINDINGS__0__ACTIVE_CONNECTIONS_SECURITY", "1")
             .WithEnvironment("SFTPGO_FTPD__BINDINGS__0__CERTIFICATE_FILE", CERTIFICATE_FILE)
@@ -152,5 +152,5 @@ public sealed class FtpTests : BaseSftpgoTest
 
         await builtContainer.StopAsync();
     }
-    
+
 }