diff --git a/Duplicati/Library/AutoUpdater/UpdaterManager.cs b/Duplicati/Library/AutoUpdater/UpdaterManager.cs index 4936db2e6..a27866911 100644 --- a/Duplicati/Library/AutoUpdater/UpdaterManager.cs +++ b/Duplicati/Library/AutoUpdater/UpdaterManager.cs @@ -231,7 +231,7 @@ namespace Duplicati.Library.AutoUpdater /// /// The machine name /// - public static string MachineName => Environment.MachineName; + public static readonly string MachineName = MachineNameReader.GetMachineName(); /// /// The package type ID diff --git a/Duplicati/Library/Interface/QuotaInfo.cs b/Duplicati/Library/Interface/QuotaInfo.cs index 6aa7a330e..6ff8bd091 100644 --- a/Duplicati/Library/Interface/QuotaInfo.cs +++ b/Duplicati/Library/Interface/QuotaInfo.cs @@ -18,11 +18,6 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace Duplicati.Library.Interface { diff --git a/Duplicati/Library/Main/AsyncDownloader.cs b/Duplicati/Library/Main/AsyncDownloader.cs deleted file mode 100644 index 105d4344d..000000000 --- a/Duplicati/Library/Main/AsyncDownloader.cs +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using Duplicati.Library.Main.Database; - -namespace Duplicati.Library.Main -{ - internal interface IAsyncDownloadedFile : IRemoteVolume - { - Library.Utility.TempFile TempFile { get; } - } - - internal class AsyncDownloader : IEnumerable - { - private static readonly string LOGTAG = Logging.Log.LogTagFromType(); - - private class AsyncDownloaderEnumerator : IEnumerator - { - private class AsyncDownloadedFile : IAsyncDownloadedFile - { - private readonly Exception m_exception; - private Library.Utility.TempFile m_file; - - public string Name { get; private set; } - public string Hash { get; private set; } - public long Size { get; private set; } - - public void DisposeTempFile() - { - if (m_file != null) - try { m_file.Dispose(); } - finally { m_file = null; } - } - - - public Library.Utility.TempFile TempFile - { - get - { - if (m_exception != null) - throw m_exception; - - return m_file; - } - } - - public AsyncDownloadedFile(string name, string hash, long size, Library.Utility.TempFile tempfile, Exception exception) - { - this.Name = name; - this.Hash = hash; - this.Size = size; - m_exception = exception; - m_file = tempfile; - } - } - - private readonly IList m_volumes; - private BackendManager.IDownloadWaitHandle m_handle; - private readonly BackendManager m_backend; - private int m_index; - private AsyncDownloadedFile m_current; - - public AsyncDownloaderEnumerator(IList volumes, BackendManager backend) - { - m_volumes = volumes; - m_backend = backend; - m_index = 0; - } - - public IAsyncDownloadedFile Current - { - get { return m_current; } - } - - public void Dispose() - { - if (m_current != null) - { - m_current.DisposeTempFile(); - m_current = null; - } - } - - object System.Collections.IEnumerator.Current - { - get { return this.Current; } - } - - public bool MoveNext() - { - if (m_current != null) - { - m_current.DisposeTempFile(); - m_current = null; - } - - if (m_index >= m_volumes.Count) - return false; - - if (m_handle == null) - m_handle = m_backend.GetAsync(m_volumes[m_index].Name, m_volumes[m_index].Size, m_volumes[m_index].Hash); - - string hash = null; - long size = -1; - Library.Utility.TempFile file = null; - Exception exception = null; - try - { - file = m_handle.Wait(out hash, out size); - - } - catch (Exception ex) - { - Logging.Log.WriteErrorMessage(LOGTAG, "FailedToRetrieveFile", ex, "Failed to retrieve file {0}", m_volumes[m_index].Name); - exception = ex; - } - - m_current = new AsyncDownloadedFile(m_volumes[m_index].Name, hash, size, file, exception); - m_handle = null; - - m_index++; - if (m_index < m_volumes.Count) - m_handle = m_backend.GetAsync(m_volumes[m_index].Name, m_volumes[m_index].Size, m_volumes[m_index].Hash); - - return true; - } - - public void Reset() - { - throw new NotSupportedException("Cannot reset " + this.GetType().FullName); - } - } - - private readonly IList m_volumes; - private readonly BackendManager m_backend; - - public AsyncDownloader(IList volumes, BackendManager backend) - { - m_volumes = volumes; - m_backend = backend; - } - - public IEnumerator GetEnumerator() - { - return new AsyncDownloaderEnumerator(m_volumes, m_backend); - } - - System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() - { - return this.GetEnumerator(); - } - } -} diff --git a/Duplicati/Library/Main/Backend/BackendManager.DatabaseCollector.cs b/Duplicati/Library/Main/Backend/BackendManager.DatabaseCollector.cs new file mode 100644 index 000000000..9358885a1 --- /dev/null +++ b/Duplicati/Library/Main/Backend/BackendManager.DatabaseCollector.cs @@ -0,0 +1,179 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Duplicati.Library.Logging; +using Duplicati.Library.Main.Database; + +namespace Duplicati.Library.Main.Backend; + +#nullable enable + +partial class BackendManager +{ + /// + /// A class to collect operations performed on the remote destination. + /// This class is used to log operations so they can later be written to the database. + /// The goal is to keep the database state as close as possible to the remote operation state. + /// Since the backend communication is done in parallel with the actual operations, + /// we store the performed operations in a queue and flush them to the database when the database is available. + /// The main operations are responsible for flushing the messages when comitting a transaction. + /// If the operation fails, the logged messages here should still be flushed to the database, + /// as they have already been performed on the remote destination. + /// + private class DatabaseCollector + { + /// + /// The log tag for this class + /// + private static readonly string LOGTAG = Log.LogTagFromType(); + /// + /// The lock object for the database queue + /// + private readonly object m_dbqueuelock = new object(); + /// + /// The queue of database operations + /// + private List m_dbqueue = []; + + /// + /// Interface for database entries + /// + private interface IRemoteOperationEntry { } + + /// + /// Logs an operation performed on the remote destination + /// + /// The performed action + /// The file the operation is performed on + /// The result of the operation + private sealed record RemoteOperationLogEntry(string Action, string File, string? Result) : IRemoteOperationEntry; + + + /// + /// Logs a database update after a remote file operation + /// + /// The remote name of the volume + /// The new state of the volume + /// The new size of the volume + /// The new hash of the volume + private sealed record RemoteVolumeUpdate(string Remotename, RemoteVolumeState State, long Size, string? Hash) : IRemoteOperationEntry; + + /// + /// Logs a rename of a remote file + /// + /// The old name of the file + /// The new name of the file + private sealed record RenameRemoteVolume(string Oldname, string Newname) : IRemoteOperationEntry; + + /// + /// Logs an operation performed on the remote destination + /// + /// The action of the operation + /// The file of the operation + /// The result of the operation + public void LogRemoteOperation(string action, string file, string? result) + { + lock (m_dbqueuelock) + m_dbqueue.Add(new RemoteOperationLogEntry(action, file, result)); + } + + /// + /// Logs a database update after a remote file operation + /// + /// The remote name of the volume + /// The new state of the volume + /// The new size of the volume + /// The new hash of the volume + public void LogRemoteVolumeUpdated(string remotename, RemoteVolumeState state, long size, string? hash) + { + lock (m_dbqueuelock) + m_dbqueue.Add(new RemoteVolumeUpdate(remotename, state, size, hash)); + } + + /// + /// Logs a remote file rename + /// + /// The old name of the file + /// The new name of the file + public void LogRemoteVolumeRenamed(string oldname, string newname) + { + lock (m_dbqueuelock) + m_dbqueue.Add(new RenameRemoteVolume(oldname, newname)); + } + + /// + /// Drops all pending messages from the queue + /// + public void ClearPendingMessages() + { + lock (m_dbqueuelock) + m_dbqueue = []; + } + + /// + /// Flushes all messages to the database + /// + /// The database to write pending messages to + /// The transaction to use, if any + /// Whether any messages were flushed + public bool FlushPendingMessages(LocalDatabase db, System.Data.IDbTransaction? transaction) + { + List entries; + lock (m_dbqueuelock) + if (m_dbqueue.Count == 0) + return false; + else + { + entries = m_dbqueue; + m_dbqueue = []; + } + + // Collect removed volumes for final db cleanup. + var volsRemoved = new HashSet(); + + // As we replace the list, we can now freely access the elements without locking + foreach (var e in entries) + if (e is RemoteOperationLogEntry operation) + db.LogRemoteOperation(operation.Action, operation.File, operation.Result, transaction); + else if (e is RemoteVolumeUpdate update && update.State == RemoteVolumeState.Deleted) + { + db.UpdateRemoteVolume(update.Remotename, RemoteVolumeState.Deleted, update.Size, update.Hash, true, TimeSpan.FromHours(2), transaction); + volsRemoved.Add(update.Remotename); + } + else if (e is RemoteVolumeUpdate dbUpdate) + db.UpdateRemoteVolume(dbUpdate.Remotename, dbUpdate.State, dbUpdate.Size, dbUpdate.Hash, transaction); + else if (e is RenameRemoteVolume rename) + db.RenameRemoteFile(rename.Oldname, rename.Newname, transaction); + else if (e != null) + Log.WriteErrorMessage(LOGTAG, "InvalidQueueElement", null, "Queue had element of type: {0}, {1}", e.GetType(), e); + + // Finally remove volumes from DB. + if (volsRemoved.Count > 0) + db.RemoveRemoteVolumes(volsRemoved); + + return true; + } + + /// + /// Flushes all messages to the log after stopping the processing + /// + public void FlushMessagesToLog() + { + if (m_dbqueue.Count == 0) + return; + + string message; + lock (m_dbqueuelock) + message = string.Join("\n", m_dbqueue.Select(e => e switch + { + RemoteOperationLogEntry operation => $"Operation: {operation.Action} File: {operation.File} Result: {operation.Result}", + RemoteVolumeUpdate update => $"Update: {update.Remotename} State: {update.State} Size: {update.Size} Hash: {update.Hash}", + RenameRemoteVolume rename => $"Rename: {rename.Oldname} -> {rename.Newname}", + _ => $"InvalidQueueElement: {e.GetType()} {e}" + })); + + Log.WriteWarningMessage(LOGTAG, "FlushingMessagesToLog", null, message); + + } + } +} \ No newline at end of file diff --git a/Duplicati/Library/Main/Backend/BackendManager.DeleteOperation.cs b/Duplicati/Library/Main/Backend/BackendManager.DeleteOperation.cs new file mode 100644 index 000000000..a846328b3 --- /dev/null +++ b/Duplicati/Library/Main/Backend/BackendManager.DeleteOperation.cs @@ -0,0 +1,107 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Duplicati.Library.Interface; +using Duplicati.Library.Localization.Short; + +namespace Duplicati.Library.Main.Backend; + +#nullable enable + +partial class BackendManager +{ + /// + /// Represents a pending DELETE operation + /// + private class DeleteOperation : PendingOperation + { + /// + /// The log tag for this class + /// + private static readonly string LOGTAG = Logging.Log.LogTagFromType(); + + /// + /// The remote filename that is to be deleted + /// + public override string RemoteFilename { get; } + /// + /// The size of the remote file, or -1 if unknown + /// + public override long Size { get; } + + /// + /// The operation type + /// + public override BackendActionType Operation => BackendActionType.Delete; + + /// + /// Creates a new DeleteOperation + /// + /// The remote filename to delete + /// The size of the remote file, or -1 if unknown + /// The execution context + /// Whether to wait for the operation to complete + public DeleteOperation(string remoteName, long size, ExecuteContext context, bool waitForComplete, CancellationToken cancelToken) + : base(context, waitForComplete, cancelToken) + { + RemoteFilename = remoteName; + Size = size; + } + + /// + public override async Task ExecuteAsync(IBackend backend, CancellationToken cancelToken) + { + Context.Statwriter.SendEvent(BackendActionType.Delete, BackendEventType.Started, RemoteFilename, Size); + + string? result = null; + try + { + await backend.DeleteAsync(RemoteFilename, cancelToken).ConfigureAwait(false); + } + catch (Exception ex) + { + // Check if the file was not found, and if so, check if it was actually deleted + var isFileMissingException = ex is Library.Interface.FileMissingException || ex is System.IO.FileNotFoundException; + var wr = (ex as System.Net.WebException)?.Response as System.Net.HttpWebResponse; + bool recovered = false; + + if (isFileMissingException || (wr != null && wr.StatusCode == System.Net.HttpStatusCode.NotFound)) + { + Logging.Log.WriteInformationMessage(LOGTAG, "DeleteRemoteFileFailed", LC.L($"Delete operation failed for {RemoteFilename} with FileNotFound, listing contents")); + + try + { + recovered = !backend.List().Select(x => x.Name).Contains(RemoteFilename); + } + catch + { + } + + if (recovered) + Logging.Log.WriteInformationMessage(LOGTAG, "DeleteRemoteFileSuccess", LC.L($"Listing indicates file {RemoteFilename} was deleted correctly")); + else + Logging.Log.WriteWarningMessage(LOGTAG, "DeleteRemoteFileFailed", ex, LC.L($"Listing confirms file {RemoteFilename} was not deleted")); + } + + if (!recovered) + { + result = ex.ToString(); + throw; + } + } + finally + { + // Here, we do not know if the file was actually deleted or not + // We log that the operation was performed, and the result + Context.Database.LogRemoteOperation("delete", RemoteFilename, result); + + // We also log the new state of the file, so it will be attempted to be re-deleted on later listings + Context.Database.LogRemoteVolumeUpdated(RemoteFilename, RemoteVolumeState.Deleted, -1, null); + } + + Context.Statwriter.SendEvent(BackendActionType.Delete, BackendEventType.Completed, RemoteFilename, Size); + return true; + } + } +} \ No newline at end of file diff --git a/Duplicati/Library/Main/Backend/BackendManager.GetOperation.cs b/Duplicati/Library/Main/Backend/BackendManager.GetOperation.cs new file mode 100644 index 000000000..14bd9a2bf --- /dev/null +++ b/Duplicati/Library/Main/Backend/BackendManager.GetOperation.cs @@ -0,0 +1,250 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Duplicati.Library.Interface; +using Duplicati.Library.Utility; +using SharpAESCrypt; + +namespace Duplicati.Library.Main.Backend; + +#nullable enable + +partial class BackendManager +{ + /// + /// Represents a pending GET operation + /// + private class GetOperation : PendingOperation<(TempFile File, string Hash, long Size)> + { + /// + /// The log tag for this class + /// + private static readonly string LOGTAG = Logging.Log.LogTagFromType(); + + /// + /// The remote filename that is to be downloaded + /// + public override string RemoteFilename { get; } + /// + /// The size of the remote file, or -1 if unknown + /// + public override long Size { get; } + /// + /// The hash of the remote file, or null if unknown + /// + public required string? Hash { get; set; } + + /// + /// The operation type + /// + public override BackendActionType Operation => BackendActionType.Get; + + /// + /// Creates a new GetOperation + /// + /// The remote filename to download + /// The size of the remote file, or -1 if unknown + /// The execution context + /// The cancellation token + public GetOperation(string remotefilename, long size, ExecuteContext context, CancellationToken cancelToken) + : base(context, true, cancelToken) + { + RemoteFilename = remotefilename; + Size = size; + } + + /// + /// Executes the operation + /// + /// The backend to download from + /// The cancellation token + /// The file, hash and size of the downloaded file + public override async Task<(TempFile File, string Hash, long Size)> ExecuteAsync(IBackend backend, CancellationToken cancelToken) + { + TempFile? tmpfile = null; + Context.Statwriter.SendEvent(BackendActionType.Get, BackendEventType.Started, RemoteFilename, Size); + using var encryption = Context.Options.NoEncryption + ? null + : (DynamicLoader.EncryptionLoader.GetModule(Context.Options.EncryptionModule, Context.Options.Passphrase, Context.Options.RawOptions) + ?? throw new Exception(Strings.BackendMananger.EncryptionModuleNotFound(Context.Options.EncryptionModule)) + ); + + try + { + // Start and time the donwload + var begin = DateTime.Now; + + (tmpfile, var dataSizeDownloaded, var fileHash) = await DoGetFileAsync(backend, cancelToken).ConfigureAwait(false); + + var duration = DateTime.Now - begin; + Logging.Log.WriteProfilingMessage(LOGTAG, "DownloadSpeed", "Downloaded {0} in {1}, {2}/s", Library.Utility.Utility.FormatSizeString(dataSizeDownloaded), + duration, Library.Utility.Utility.FormatSizeString((long)(dataSizeDownloaded / duration.TotalSeconds))); + + Context.Database.LogRemoteOperation("get", RemoteFilename, System.Text.Json.JsonSerializer.Serialize(new { Size = dataSizeDownloaded, Hash = fileHash })); + Context.Statwriter.SendEvent(BackendActionType.Get, BackendEventType.Completed, RemoteFilename, dataSizeDownloaded); + + if (!Context.Options.SkipFileHashChecks) + { + if (Size >= 0 && dataSizeDownloaded != Size) + throw new Exception(Strings.Controller.DownloadedFileSizeError(RemoteFilename, dataSizeDownloaded, Size)); + + if (!string.IsNullOrEmpty(Hash) && fileHash != Hash) + throw new HashMismatchException(Strings.Controller.HashMismatchError(tmpfile, Hash, fileHash)); + } + + // Perform decryption after hash validation, if needed + tmpfile = DecryptFile(tmpfile, DetectEncryptionModule(encryption)); + + return (tmpfile, fileHash, dataSizeDownloaded); + } + catch + { + tmpfile?.Dispose(); + throw; + } + } + + /// + /// Downloads a file from the backend + /// + /// The backend to download from + /// The cancellation token + /// The downloaded file, the size of the file, and the hash of the file + private async Task<(TempFile tempFile, long downloadSize, string remotehash)> DoGetFileAsync(IBackend backend, CancellationToken cancelToken) + { + TempFile? dlTarget = null; + try + { + long retDownloadSize; + string retHashcode; + dlTarget = new TempFile(); + if (backend is IStreamingBackend streamingBackend && !Context.Options.DisableStreamingTransfers) + { + // extended to use stacked streams + using (var fs = System.IO.File.OpenWrite(dlTarget)) + using (var act = new StreamUtil.TimeoutObservingStream(fs) { WriteTimeout = backend is ITimeoutExemptBackend ? Timeout.Infinite : Context.Options.ReadWriteTimeout }) + using (var hasher = HashFactory.CreateHasher(Context.Options.FileHashAlgorithm)) + using (var hs = new HashCalculatingStream(act, hasher)) + using (var ss = new ShaderStream(hs, true)) + using (var linkedToken = CancellationTokenSource.CreateLinkedTokenSource(cancelToken, act.TimeoutToken)) + { + using (var ts = new ThrottledStream(ss, 0, Context.Options.MaxDownloadPrSecond)) + using (var pgs = new ProgressReportingStream(ts, pg => Context.HandleProgress(ts, pg, RemoteFilename))) + { + await streamingBackend.GetAsync(RemoteFilename, pgs, linkedToken.Token).ConfigureAwait(false); + } + ss.Flush(); + retDownloadSize = ss.TotalBytesWritten; + retHashcode = Convert.ToBase64String(hs.GetFinalHash()); + } + } + else + { + await backend.GetAsync(RemoteFilename, dlTarget, cancelToken).ConfigureAwait(false); + retDownloadSize = new System.IO.FileInfo(dlTarget).Length; + retHashcode = CalculateFileHash(dlTarget, Context.Options); + } + + var retTarget = dlTarget; + dlTarget = null; + return (retTarget, retDownloadSize, retHashcode); + } + finally + { + // Remove temp files on failure + dlTarget?.Dispose(); + } + } + + /// + /// Detects the encryption module to use, based on the filename. + /// This makes it possible to restore from a folder with mixed encryption modules. + /// + /// The default encryption module + /// The encryption module to use + private IEncryption? DetectEncryptionModule(IEncryption? encryption) + { + try + { + // Auto-guess the encryption module + var ext = (System.IO.Path.GetExtension(RemoteFilename) ?? "").TrimStart('.'); + if (!ext.Equals(encryption?.FilenameExtension, StringComparison.OrdinalIgnoreCase)) + { + // Check if the file is not encrypted + if (DynamicLoader.CompressionLoader.Keys.Contains(ext, StringComparer.OrdinalIgnoreCase)) + { + if (encryption != null) + Logging.Log.WriteVerboseMessage(LOGTAG, "AutomaticDecryptionDetection", "Filename extension \"{0}\" does not match encryption module \"{1}\", guessing that it is not encrypted", ext, Context.Options.EncryptionModule); + return null; + } + // Check if the file is encrypted with something else + else if (DynamicLoader.EncryptionLoader.Keys.Contains(ext, StringComparer.OrdinalIgnoreCase)) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "AutomaticDecryptionDetection", "Filename extension \"{0}\" does not match encryption module \"{1}\", attempting to use matching encryption module", ext, Context.Options.EncryptionModule); + + try + { + return DynamicLoader.EncryptionLoader.GetModule(ext, Context.Options.Passphrase, Context.Options.RawOptions) + ?? encryption; + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "AutomaticDecryptionDetection", ex, "Failed to load encryption module \"{0}\", using specified encryption module \"{1}\"", ext, Context.Options.EncryptionModule); + } + } + // Fallback, lets see what happens... + else + { + Logging.Log.WriteVerboseMessage(LOGTAG, "AutomaticDecryptionDetection", "Filename extension \"{0}\" does not match encryption module \"{1}\", attempting to use specified encryption module as no others match", ext, Context.Options.EncryptionModule); + } + } + + return encryption; + } + // If we fail here, make sure that we throw a crypto exception + catch (System.Security.Cryptography.CryptographicException) { throw; } + catch (Exception ex) { throw new System.Security.Cryptography.CryptographicException(ex.Message, ex); } + } + + /// + /// Performs decryption of a file. This could be more efficient if we decrypt while downloading, + /// but that would prevent us from verifying the hash of the encrypted file, before decrypting it. + /// Decrypting afterwards also ensures we can control the thrown exceptions. + /// + /// The encrypted file + /// Then encryption module to use, or null for no encryption + /// The decrypted file + private TempFile DecryptFile(TempFile tempFile, IEncryption? decrypter) + { + // Support no encryption + if (decrypter == null) + return tempFile; + + TempFile? decryptTarget = null; + + // Always dispose the source file + using (tempFile) + using (new Logging.Timer(LOGTAG, "DecryptFile", "Decrypting " + tempFile)) + { + try + { + decryptTarget = new TempFile(); + try { decrypter.Decrypt(tempFile, decryptTarget); } + // If we fail here, make sure that we throw a crypto exception + catch (System.Security.Cryptography.CryptographicException) { throw; } + catch (Exception ex) { throw new System.Security.Cryptography.CryptographicException(ex.Message, ex); } + + var result = decryptTarget; + decryptTarget = null; + return result; + } + finally + { + // Remove temp files on failure + decryptTarget?.Dispose(); + } + } + } + } +} \ No newline at end of file diff --git a/Duplicati/Library/Main/Backend/BackendManager.Handler.cs b/Duplicati/Library/Main/Backend/BackendManager.Handler.cs new file mode 100644 index 000000000..b059a0f1f --- /dev/null +++ b/Duplicati/Library/Main/Backend/BackendManager.Handler.cs @@ -0,0 +1,443 @@ +#nullable enable + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using CoCoL; +using Duplicati.Library.Interface; + +namespace Duplicati.Library.Main.Backend; + +partial class BackendManager +{ + /// + /// Wrapper class for making a backend disposable and reclaimable + /// + private sealed class ReclaimableBackend : IDisposable + { + /// + /// The tag used for logging + /// + private static readonly string LOGTAG = Logging.Log.LogTagFromType(); + + /// + /// The backend being wrapped + /// + public IBackend Backend { get; } + /// + /// The pool where the backend should be returned to + /// + private readonly ConcurrentQueue pool; + /// + /// Whether the backend should be reused + /// + private bool reuse; + /// + /// Whether the backend wrapper has been disposed + /// + private bool disposed; + + /// + /// Creates a new instance of the class + /// + /// The backend to wrap + /// The pool where the backend should be returned to + /// Whether the backend should be reused or disposed + public ReclaimableBackend(IBackend backend, ConcurrentQueue pool, bool reuse) + { + Backend = backend; + this.pool = pool; + this.reuse = reuse; + } + + /// + /// Prevents the backend from being reclaimed + /// + public void PreventReuse() + { + reuse = false; + } + + /// + /// Disposes the backend wrapper + /// + public void Dispose() + { + if (disposed) + return; + disposed = true; + + if (reuse) + pool.Enqueue(Backend); + else + try { Backend.Dispose(); } + catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "BackendDisposeError", ex, "Failed to dispose backend instance: {0}", ex.Message); } + } + } + + /// + /// The handler for processing backend operations + /// + private class Handler + { + /// + /// The tag used for logging + /// + private static readonly string LOGTAG = Logging.Log.LogTagFromType(); + + /// + /// The list of active uploads + /// + private readonly List activeUploads = new(); + /// + /// The pool of backends currently created + /// + private readonly ConcurrentQueue backendPool = new(); + /// + /// The URL of the backend + /// + private readonly string backendUrl; + /// + /// The context for the handler + /// + private readonly ExecuteContext context; + /// + /// The maximum number of parallel uploads + /// + private readonly int maxParallelUploads; + /// + /// The maximum number of retries + /// + private readonly int maxRetries; + /// + /// The delay between retries + /// + private readonly TimeSpan retryDelay; + /// + /// Whether to retry with exponential backoff + /// + private readonly bool retryWithExponentialBackoff; + /// + /// Whether to allow backend reuse + /// + private readonly bool allowBackendReuse; + /// + /// Whether any files have been uploaded + /// + private bool anyUploaded; + /// + /// Whether any files have been downloaded + /// + private bool anyDownloaded; + + /// + /// Creates and runs with an instance of the class + /// + /// The channel for pending operations + /// The URL of the backend + /// The execution context + /// An awaitable task + public static Task RunHandlerAsync(IReadChannel requestChannel, string backendUrl, ExecuteContext context) + => AutomationExtensions.RunTask(new { requestChannel }, + self => new Handler(backendUrl, context).Run(self.requestChannel) + ); + + /// + /// Creates a new instance of the class + /// + /// The URL of the backend + /// The execution context + private Handler(string backendUrl, ExecuteContext context) + { + this.backendUrl = backendUrl; + this.context = context; + + maxParallelUploads = Math.Max(1, context.Options.AsynchronousConcurrentUploadLimit); + maxRetries = context.Options.NumberOfRetries; + retryDelay = context.Options.RetryDelay; + retryWithExponentialBackoff = context.Options.RetryWithExponentialBackoff; + allowBackendReuse = !context.Options.NoConnectionReuse; + + } + + /// + /// Creates a new backend instance or reuses an existing one + /// + /// The backend instance + private ReclaimableBackend CreateBackend() + { + backendPool.TryDequeue(out var backend); + if (backend == null) + backend = DynamicLoader.BackendLoader.GetBackend(backendUrl, context.Options.RawOptions); + + return new ReclaimableBackend( + backend, + backendPool, + allowBackendReuse + ); + } + + /// + /// Reclaims completed uploads + /// + /// An awaitable task + private async Task ReclaimCompletedUploads() + { + for (int i = activeUploads.Count - 1; i >= 0; i--) + { + if (activeUploads[i].IsCompleted) + { + // Make sure the task is awaited so we capture any exceptions + await activeUploads[i].ConfigureAwait(false); + activeUploads.RemoveAt(i); + } + } + } + + /// + /// Ensures that there are at most N - 1 pending uploads + /// + /// The maximum number of pending uploads + /// An awaitable task + private async Task EnsureAtMostNPendingUploads(int n) + { + while (activeUploads.Count >= n) + { + await Task.WhenAny(activeUploads).ConfigureAwait(false); + await ReclaimCompletedUploads().ConfigureAwait(false); + } + } + + /// + /// Runs the handler + /// + /// The channel for pending operations + /// An awaitable task + private async Task Run(IReadChannel requestChannel) + { + while (true) + { + // Get next operation + var op = await requestChannel.ReadAsync().ConfigureAwait(false); + + try + { + // Clean up completed uploads, if any + await ReclaimCompletedUploads().ConfigureAwait(false); + + // Allow PUT operations to be queued, if requested + if (op is PutOperation putOp && !putOp.WaitForComplete) + { + await EnsureAtMostNPendingUploads(maxParallelUploads).ConfigureAwait(false); + + // Operation is accepted into queue, so we can signal completion + putOp.SetComplete(true); + activeUploads.Add(ExecuteWithRetry(putOp)); + } + else + { + // Wait for all pending uploads to complete + await EnsureAtMostNPendingUploads(1).ConfigureAwait(false); + // Execute the operation + await ExecuteWithRetry(op).ConfigureAwait(false); + } + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "HandlerError", ex, "Error in handler: {0}", ex.Message); + // If we fail, the task may "hang", so we ensure it is completed here + op.SetFailed(ex); + throw; + } + } + } + + /// + /// Tries to create a folder, handling errors + /// + /// true if the folder was created, false otherwise + private async Task TryCreateFolder() + { + using var backend = CreateBackend(); + try + { + // If we successfully create the folder, we can re-use the connection + await backend.Backend.CreateFolderAsync(context.TaskReader.TransferToken).ConfigureAwait(false); + return true; + } + catch (Exception ex) + { + // Failure should not reuse the backend + backend.PreventReuse(); + Logging.Log.WriteWarningMessage(LOGTAG, "FolderCreateError", ex, "Failed to create folder: {0}", ex.Message); + } + + return false; + } + + /// + /// Executes an operation with retries and error handling + /// + /// The operation to execute + /// An awaitable task + private async Task ExecuteWithRetry(PendingOperationBase op) + { + // Once in this method, we MUST set the op result, + // or the program will hang waiting for the operation to complete + + int retries = 0; + Exception? lastException = null; + + do + { + try + { + // Happy case is execute and return + await Execute(op).ConfigureAwait(false); + return; + } + catch (Exception ex) + { + retries++; + lastException = ex; + Logging.Log.WriteRetryMessage(LOGTAG, $"Retry{op.Operation}", ex, "Operation {0} with file {1} attempt {2} of {3} failed with message: {4}", op.Operation, op.RemoteFilename, retries, maxRetries, ex.Message); + + // If we are cancelled, stop retrying + if (op.CancelToken.IsCancellationRequested || context.TaskReader.ProgressToken.IsCancellationRequested || context.TaskReader.TransferToken.IsCancellationRequested) + { + op.SetCancelled(); + return; + } + + // Refresh DNS name if we fail to connect in order to prevent issues with incorrect DNS entries + var dnsFailure = Library.Utility.Utility.FlattenException(ex).Any(x => x is System.Net.WebException wex && wex.Status == System.Net.WebExceptionStatus.NameResolutionFailure); + if (dnsFailure) + { + try + { + using (var backend = CreateBackend()) + foreach (var name in await backend.Backend.GetDNSNamesAsync(context.TaskReader.TransferToken).ConfigureAwait(false) ?? []) + if (!string.IsNullOrWhiteSpace(name)) + System.Net.Dns.GetHostEntry(name); + } + catch + { + } + } + + context.Statwriter.SendEvent(op.Operation, retries < maxRetries ? BackendEventType.Retrying : BackendEventType.Failed, op.RemoteFilename, op.Size); + + // Check if we can recover from the error + var recovered = false; + + // Check if this was a folder missing exception and we are allowed to autocreate folders + if (!(anyDownloaded || anyUploaded) && context.Options.AutocreateFolders && Library.Utility.Utility.FlattenException(ex).Any(x => x is FolderMissingException)) + { + if (await TryCreateFolder().ConfigureAwait(false)) + recovered = true; + } + + // We did not recover, so wait or give up + if (!recovered && retries < maxRetries && retryDelay.Ticks != 0) + { + var delay = Library.Utility.Utility.GetRetryDelay(retryDelay, retries, retryWithExponentialBackoff); + await Task.Delay(delay, context.TaskReader.ProgressToken).ConfigureAwait(false); + } + } + + } while (retries < maxRetries); + + // If we have a last exception, we failed + if (lastException != null) + { + op.SetFailed(lastException); + (op as IDisposable)?.Dispose(); + + // Stop processing tasks if the operation failed and is not being waited for + // Delete operations can be retried later, so we don't stop processing + if (!op.WaitForComplete && op is not DeleteOperation) + throw lastException; + } + } + + /// + /// Fan-out for executing operations. + /// This method requires manual updates when new operation types are added, + /// but avoids a reflection-based dispatch. + /// + /// The operation to execute + /// An awaitable task + private async Task Execute(PendingOperationBase op) + { + await context.TaskReader.ProgressRendevouz().ConfigureAwait(false); + using (new Logging.Timer(LOGTAG, $"RemoteOperation{op.Operation}", $"RemoteOperation{op.Operation}")) + switch (op) + { + case PutOperation putOp: + await Execute(putOp).ConfigureAwait(false); + anyUploaded = true; + return; + case GetOperation getOp: + await Execute(getOp).ConfigureAwait(false); + anyDownloaded = true; + return; + case DeleteOperation deleteOp: + await Execute(deleteOp).ConfigureAwait(false); + return; + case ListOperation listOp: + await Execute(listOp).ConfigureAwait(false); + return; + case QuotaInfoOperation quotaOp: + await Execute(quotaOp).ConfigureAwait(false); + return; + case WaitForEmptyOperation waitOp: + waitOp.SetComplete(true); + return; + default: + throw new NotImplementedException($"Operation type {op.GetType()} is not supported"); + } + } + + /// + /// Executes a specific operation + /// + /// The return value type of the operation + /// The operation to execute + /// An awaitable task + private async Task Execute(PendingOperation op) + { + using var backend = CreateBackend(); + using var token = CancellationTokenSource.CreateLinkedTokenSource(op.CancelToken, context.TaskReader.TransferToken); + + try + { + // Start processing the operation + var task = op.ExecuteAsync(backend.Backend, token.Token); + + if (typeof(TResult) == typeof(bool) && !op.WaitForComplete) + { + // Operation is accepted into queue, so we can signal completion + op.SetComplete((TResult)(object)true); + await task.ConfigureAwait(false); + } + else + { + if (!op.WaitForComplete) + throw new NotImplementedException($"WaitForComplete is required for operations returning a value: {op.GetType().FullName}"); + + // Wait for the operation to complete + op.SetComplete(await task.ConfigureAwait(false)); + } + } + catch + { + // If the operation fails, we prevent reuse of the backend + backend.PreventReuse(); + throw; + } + } + } +} diff --git a/Duplicati/Library/Main/Backend/BackendManager.ListOperation.cs b/Duplicati/Library/Main/Backend/BackendManager.ListOperation.cs new file mode 100644 index 000000000..c3389ff31 --- /dev/null +++ b/Duplicati/Library/Main/Backend/BackendManager.ListOperation.cs @@ -0,0 +1,63 @@ +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Duplicati.Library.Interface; + +namespace Duplicati.Library.Main.Backend; + +#nullable enable + +partial class BackendManager +{ + /// + /// Represents a pending LIST operation + /// + /// + /// Creates a new ListOperation + /// + /// The execution context + /// The cancellation token + private class ListOperation(ExecuteContext context, CancellationToken cancelToken) + : PendingOperation>(context, true, cancelToken) + { + /// + /// The operation type + /// + public override BackendActionType Operation => BackendActionType.List; + + /// + /// Executes the operation + /// + /// The backend to list + /// The cancellation token + /// An awaitable task + public override Task> ExecuteAsync(IBackend backend, CancellationToken cancelToken) + { + Context.Statwriter.SendEvent(BackendActionType.List, BackendEventType.Started, null, -1); + + var r = backend.List().ToList(); + + // TODO: Investigate better way to solve this so we do not use memory for large lists + var sb = new StringBuilder(); + sb.AppendLine("["); + long count = 0; + foreach (var e in r) + { + if (count != 0) + sb.AppendLine(","); + count++; + sb.Append(System.Text.Json.JsonSerializer.Serialize(e)); + } + + sb.AppendLine(); + sb.Append("]"); + Context.Database.LogRemoteOperation("list", "", sb.ToString()); + + Context.Statwriter.SendEvent(BackendActionType.List, BackendEventType.Completed, null, r.Count); + + return Task.FromResult(r); + } + } +} \ No newline at end of file diff --git a/Duplicati/Library/Main/Backend/BackendManager.PendingOperation.cs b/Duplicati/Library/Main/Backend/BackendManager.PendingOperation.cs new file mode 100644 index 000000000..42fd7e040 --- /dev/null +++ b/Duplicati/Library/Main/Backend/BackendManager.PendingOperation.cs @@ -0,0 +1,138 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Duplicati.Library.Interface; +using Duplicati.Library.Main.Operation.Common; +using Duplicati.Library.Utility; + +namespace Duplicati.Library.Main.Backend; + +#nullable enable + +partial class BackendManager +{ + /// + /// Execution context for the backend manager + /// + /// A progress handler + /// The stat writer + /// The database collector + /// The task reader + /// The options + private sealed record ExecuteContext( + Action HandleProgress, + IBackendWriter Statwriter, + DatabaseCollector Database, + ITaskReader TaskReader, + Options Options); + + /// + /// A base non-generic pending operation + /// + private abstract class PendingOperationBase + { + /// + /// The execution context + /// + public ExecuteContext Context { get; } + /// + /// Whether to wait for the operation to complete. + /// If false, the operation is queued and the task is returned after the operation is queued. + /// + public bool WaitForComplete { get; } + /// + /// The cancellation token + /// + public CancellationToken CancelToken { get; set; } + /// + /// The type of operation the pending operation represents + /// + public abstract BackendActionType Operation { get; } + /// + /// The remote filename, if any + /// + public virtual string RemoteFilename => string.Empty; + /// + /// The remote size, if any + /// + public virtual long Size => -1L; + + /// + /// Sets the operation as cancelled + /// + public abstract void SetCancelled(); + /// + /// Sets the operation as failed + /// + /// The exception + public abstract void SetFailed(Exception ex); + + /// + /// Creates a new pending operation + /// + /// The execution context + /// Whether to wait for the operation to complete + /// The cancellation token + public PendingOperationBase(ExecuteContext context, bool waitForComplete, CancellationToken cancelToken) + { + Context = context; + WaitForComplete = waitForComplete; + CancelToken = cancelToken; + } + } + + /// + /// The basic queued backend operation + /// + /// + /// Creates a new pending operation + /// + /// The execution context + /// Whether to wait for the operation to complete + /// The cancellation token + private abstract class PendingOperation(ExecuteContext context, bool waitForComplete, CancellationToken cancelToken) : PendingOperationBase(context, waitForComplete, cancelToken) + { + /// + /// A signal for the task completion + /// + private readonly TaskCompletionSource taskCompleteSignal = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + /// + /// The task that is completed when the operation is done + /// + public Task GetResult() => taskCompleteSignal.Task; + + /// + /// Sets the operation as complete + /// + public void SetComplete(TResult result) + { + taskCompleteSignal.TrySetResult(result); + } + + /// + /// Sets the operation as failed + /// + /// The exception + public override void SetFailed(Exception ex) + { + taskCompleteSignal.TrySetException(ex); + } + + /// + /// Sets the operation as cancelled + /// + public override void SetCancelled() + { + taskCompleteSignal.TrySetCanceled(); + } + + /// + /// Executes the operation + /// + /// The backend to execute the operation on + /// The cancellation token + /// An awaitable task + public abstract Task ExecuteAsync(IBackend backend, CancellationToken cancelToken); + } + +} \ No newline at end of file diff --git a/Duplicati/Library/Main/Backend/BackendManager.ProgressHandler.cs b/Duplicati/Library/Main/Backend/BackendManager.ProgressHandler.cs new file mode 100644 index 000000000..67b77b10b --- /dev/null +++ b/Duplicati/Library/Main/Backend/BackendManager.ProgressHandler.cs @@ -0,0 +1,54 @@ +using Duplicati.Library.Main.Operation.Common; +using Duplicati.Library.Utility; + +namespace Duplicati.Library.Main.Backend; + +#nullable enable + +partial class BackendManager +{ + /// + /// Represents a pending QuotaInfo operation + /// + private sealed record ProgressHandler(Options Options, IBackendWriter Stats, ITaskReader TaskReader) + { + /// + /// Stores the last throttle values + /// + /// The upload throttle value + /// The download throttle value + private sealed record ThrottleValues(string? Upload, string? Download); + + /// + /// The last throttle values + /// + private ThrottleValues lastThrottleValue = new(string.Empty, string.Empty); + + /// + /// Handles the progress of a throttled stream + /// + /// The throttled stream + /// The progress + /// The filename + public void HandleProgress(ThrottledStream ts, long pg, string filename) + { + // This pauses and throws on cancellation, but ignores stop + TaskReader.TransferRendevouz().Await(); + + var prev = lastThrottleValue; + + // Update the throttle speeds if they have changed + Options.RawOptions.TryGetValue("throttle-upload", out var tmpUpload); + Options.RawOptions.TryGetValue("throttle-download", out var tmpDownload); + + if (tmpUpload != prev.Upload || tmpDownload != prev.Download) + { + ts.WriteSpeed = Options.MaxUploadPrSecond; + ts.ReadSpeed = Options.MaxDownloadPrSecond; + lastThrottleValue = new(tmpUpload, tmpDownload); + } + + Stats.BackendProgressUpdater.UpdateProgress(pg); + } + } +} \ No newline at end of file diff --git a/Duplicati/Library/Main/Backend/BackendManager.PutOperation.cs b/Duplicati/Library/Main/Backend/BackendManager.PutOperation.cs new file mode 100644 index 000000000..f9733c775 --- /dev/null +++ b/Duplicati/Library/Main/Backend/BackendManager.PutOperation.cs @@ -0,0 +1,345 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Duplicati.Library.Interface; +using Duplicati.Library.Main.Volumes; +using Duplicati.Library.Utility; +using Newtonsoft.Json; + +namespace Duplicati.Library.Main.Backend; + +#nullable enable + +partial class BackendManager +{ + /// + /// Represents a pending PUT operation + /// + private class PutOperation : PendingOperation + { + /// + /// Different states of the operation + /// + private enum OperationState + { + /// + /// The operation is not started + /// + None, + /// + /// The upload has started + /// + Uploading, + /// + /// The upload has completed + /// + Uploaded, + /// + /// The operation has completed + /// + Completed + } + + /// + /// The log tag for this class + /// + private static readonly string LOGTAG = Logging.Log.LogTagFromType(); + + /// + /// The operation type + /// + public override BackendActionType Operation => BackendActionType.Put; + /// + /// The remote filename backing field + /// + private string remoteFilename; + /// + /// The remote filename + /// + public override string RemoteFilename => remoteFilename; + /// + /// The local temporary file + /// + public required TempFile? LocalTempfile { get; set; } + /// + /// Flag indicating if the file is not to be encrypted + /// + public required bool Unencrypted { get; set; } + /// + /// The local filename as a string + /// + public string LocalFilename => LocalTempfile; + /// + /// Flag to indicate if the file is not tracked in the database + /// + public required bool TrackedInDb { get; init; } + /// + /// The encryption and hashing task, the return value indicates if the hash and size was updated + /// + private Task<(string Hash, long Size)>? encryptionAndHashingTask; + /// + /// The state of the operation + /// + private OperationState operationState = OperationState.None; + /// + /// The pending index operation, if any + /// + private (IndexVolumeWriter Writer, PutOperation Operation)? indexOperation; + /// + /// The original index file, if any + /// + public required IndexVolumeWriter? OriginalIndexFile { get; init; } + /// + /// A callback that is invoked when the index volume is finished + /// + public required Action? IndexVolumeFinishedCallback { get; init; } + + /// + /// Creates a new put operation + /// + /// The execution context + /// True if the operation should wait for completion + /// The cancellation token + public PutOperation(string remotefilename, ExecuteContext context, bool waitForComplete, CancellationToken cancelToken) + : base(context, waitForComplete, cancelToken) + { + remoteFilename = remotefilename; + } + + /// + /// Starts the encryption of the file + /// + /// The options to use + public void StartEncryptionAndHashing() + { + if (encryptionAndHashingTask != null) + throw new Exception("Encryption already started"); + + // Run detached to allow encrypting while waiting in upload queue + encryptionAndHashingTask = Task.Run(() => + { + if (!Context.Options.NoEncryption && !Unencrypted) + { + using var enc = DynamicLoader.EncryptionLoader.GetModule(Context.Options.EncryptionModule, Context.Options.Passphrase, Context.Options.RawOptions); + if (enc == null) + throw new Exception(Strings.BackendMananger.EncryptionModuleNotFound(Context.Options.EncryptionModule)); + + var tempfile = new TempFile(); + enc.Encrypt(LocalFilename, tempfile); + DeleteLocalFile(); + LocalTempfile = tempfile; + } + + if (!TrackedInDb) + return (string.Empty, -1L); + + return CalculateHashAndSize(); + + }); + } + + /// + /// Deletes the local temporary file + /// + private void DeleteLocalFile() + { + try { LocalTempfile?.Dispose(); } + catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "DeleteTemporaryFileError", ex, $"Failed to dispose temporary file: {LocalTempfile}"); } + finally { LocalTempfile = null; } + } + + /// + /// Calculates the hash and size of the file + /// + /// The hash and size of the file + private (string Hash, long Size) CalculateHashAndSize() + { + var hash = CalculateFileHash(LocalFilename, Context.Options); + var size = new System.IO.FileInfo(LocalFilename).Length; + + return (hash, size); + + } + + /// + /// Executes the operation + /// + /// The backend to use + /// The cancellation token + /// An awaitable task + public override async Task ExecuteAsync(IBackend backend, CancellationToken cancelToken) + { + // This operation is slightly more complex than the others, as it involves two files. + // To keep the rest of the code simpler, the upload treats the pair as a single operation + // for the caller, but internally it is two separate operations. + + // If there is a retry, this function needs to handle a retry in various states + // And, if the block file has been renamed, the index file needs to be rewritten + + if (operationState == OperationState.Completed) + throw new Exception("Operation already completed"); + + // If this is a retry after the block has uploaded, pass the operating to the index file upload + if (operationState == OperationState.Uploaded && indexOperation != null) + { + await indexOperation.Value.Operation.ExecuteAsync(backend, cancelToken).ConfigureAwait(false); + indexOperation.Value.Writer.Dispose(); + operationState = OperationState.Completed; + return true; + } + + // Check if the operation is in a valid state + if (operationState == OperationState.Uploaded) + throw new Exception("Operation already uploaded"); + + if (this.encryptionAndHashingTask == null) + throw new Exception("Encryption not started"); + + // On retry attempts, we get the same value, without recalculating + (var hash, var size) = await encryptionAndHashingTask.ConfigureAwait(false); + + // On first upload attempt, calculate the hash and size + if (operationState == OperationState.None && TrackedInDb) + Context.Database.LogRemoteVolumeUpdated(RemoteFilename, RemoteVolumeState.Uploading, size, hash); + + // First attempt here, finish the index file now that all information is known + if (OriginalIndexFile != null && indexOperation == null) + { + Context.Database.LogRemoteVolumeUpdated(OriginalIndexFile.RemoteFilename, RemoteVolumeState.Uploading, -1, null); + + // Prepare an upload operation for the index file + var req2 = new PutOperation(OriginalIndexFile.RemoteFilename, Context, false, cancelToken) + { + LocalTempfile = OriginalIndexFile.TempFile, + TrackedInDb = TrackedInDb, + Unencrypted = Unencrypted, + IndexVolumeFinishedCallback = null, + OriginalIndexFile = null + }; + + OriginalIndexFile.FinishVolume(hash, size); + IndexVolumeFinishedCallback?.Invoke(); + OriginalIndexFile.Close(); + + indexOperation = (OriginalIndexFile, req2); + } + + // If we have previously attempted to upload, we need to rename the file + if (operationState == OperationState.Uploading) + RenameFileAfterError(size); + + // Flag for next attempt, if any + operationState = OperationState.Uploading; + + await PerformUpload(backend, hash, size, cancelToken).ConfigureAwait(false); + + operationState = OperationState.Uploaded; + + // Upload completed, prepare the index file if any + if (indexOperation != null) + { + // TODO: It would be better if we encrypt the index file while uploading the block file + // since most operations work correctly. But if the upload of the block file fails + // we need to deal with decryption, or keep the unencrypted file around + indexOperation.Value.Operation.StartEncryptionAndHashing(); + await indexOperation.Value.Operation.ExecuteAsync(backend, cancelToken).ConfigureAwait(false); + indexOperation.Value.Writer.Dispose(); + } + + operationState = OperationState.Completed; + return true; + } + + /// + /// Performs the actual upload of a file + /// + /// The backend to upload to + /// The hash of the file + /// The size of the file + /// The cancellation token + /// An awaitable task + private async Task PerformUpload(IBackend backend, string Hash, long Size, CancellationToken cancelToken) + { + Context.Database.LogRemoteOperation("put", RemoteFilename, JsonConvert.SerializeObject(new { Size = Size, Hash = Hash })); + Context.Statwriter.SendEvent(BackendActionType.Put, BackendEventType.Started, RemoteFilename, Size); + + var begin = DateTime.Now; + + if (backend is IStreamingBackend streamingBackend && !Context.Options.DisableStreamingTransfers) + { + using (var fs = System.IO.File.OpenRead(LocalFilename)) + using (var act = new StreamUtil.TimeoutObservingStream(fs) { ReadTimeout = backend is ITimeoutExemptBackend ? Timeout.Infinite : Context.Options.ReadWriteTimeout }) + using (var ts = new ThrottledStream(act, Context.Options.MaxUploadPrSecond, 0)) + using (var pgs = new ProgressReportingStream(ts, pg => Context.HandleProgress(ts, pg, RemoteFilename))) + using (var linkedToken = CancellationTokenSource.CreateLinkedTokenSource(cancelToken, act.TimeoutToken)) + await streamingBackend.PutAsync(RemoteFilename, pgs, linkedToken.Token).ConfigureAwait(false); + } + else + await backend.PutAsync(RemoteFilename, LocalFilename, cancelToken).ConfigureAwait(false); + + var duration = DateTime.Now - begin; + Logging.Log.WriteProfilingMessage(LOGTAG, "UploadSpeed", "Uploaded {0} in {1}, {2}/s", Library.Utility.Utility.FormatSizeString(Size), duration, Library.Utility.Utility.FormatSizeString((long)(Size / duration.TotalSeconds))); + + if (TrackedInDb) + Context.Database.LogRemoteVolumeUpdated(RemoteFilename, RemoteVolumeState.Uploaded, Size, Hash); + + Context.Statwriter.SendEvent(BackendActionType.Put, BackendEventType.Completed, RemoteFilename, Size); + + if (Context.Options.ListVerifyUploads) + { + var f = backend.List().FirstOrDefault(n => n.Name.Equals(RemoteFilename, StringComparison.OrdinalIgnoreCase)); + if (f == null) + throw new Exception(string.Format($"List verify failed, file was not found after upload: {RemoteFilename}")); + else if (f.Size != Size && f.Size >= 0) + throw new Exception(string.Format($"List verify failed for file: {f.Name}, size was {f.Size} but expected to be {Size}")); + } + + DeleteLocalFile(); + } + + /// + /// Renames the remote target file after an error, and updates the index file, if any + /// + /// The size of the file + private void RenameFileAfterError(long Size) + { + var p = VolumeBase.ParseFilename(RemoteFilename); + var guid = VolumeWriterBase.GenerateGuid(); + var time = p.Time.Ticks == 0 ? p.Time : p.Time.AddSeconds(1); + var newname = VolumeBase.GenerateFilename(p.FileType, p.Prefix, guid, time, p.CompressionModule, p.EncryptionModule); + var oldname = RemoteFilename; + + Context.Statwriter.SendEvent(BackendActionType.Put, BackendEventType.Rename, oldname, Size); + Context.Statwriter.SendEvent(BackendActionType.Put, BackendEventType.Rename, newname, Size); + Logging.Log.WriteInformationMessage(LOGTAG, "RenameRemoteTargetFile", "Renaming \"{0}\" to \"{1}\"", oldname, newname); + Context.Database.LogRemoteVolumeRenamed(oldname, newname); + remoteFilename = newname; + + // If there is an index file attached to the block file, + // it references the block filename, so we create a new index file + // which is a copy of the current, but with the new name + if (indexOperation != null) + { + IndexVolumeWriter? wr = null; + try + { + var hashsize = HashFactory.HashSizeBytes(Context.Options.BlockHashAlgorithm); + wr = new IndexVolumeWriter(Context.Options); + using (var rd = new IndexVolumeReader(p.CompressionModule, indexOperation.Value.Operation.LocalFilename, Context.Options, hashsize)) + wr.CopyFrom(rd, x => x == oldname ? newname : x); + indexOperation.Value.Writer.Dispose(); + indexOperation = (wr, indexOperation.Value.Operation); + indexOperation.Value.Operation.LocalTempfile?.Dispose(); + indexOperation.Value.Operation.LocalTempfile = wr.TempFile; + wr.Close(); + } + catch + { + wr?.Dispose(); + throw; + } + } + } + } +} \ No newline at end of file diff --git a/Duplicati/Library/Main/Backend/BackendManager.QuotaInfoOperation.cs b/Duplicati/Library/Main/Backend/BackendManager.QuotaInfoOperation.cs new file mode 100644 index 000000000..7ce537dfa --- /dev/null +++ b/Duplicati/Library/Main/Backend/BackendManager.QuotaInfoOperation.cs @@ -0,0 +1,46 @@ +using System.Threading; +using System.Threading.Tasks; +using Duplicati.Library.Interface; + +namespace Duplicati.Library.Main.Backend; + +#nullable enable + +partial class BackendManager +{ + /// + /// Represents a pending QuotaInfo operation + /// + /// + /// Creates a new QuotaInfoOperation + /// + /// The execution context + /// The cancellation token + private class QuotaInfoOperation(ExecuteContext context, CancellationToken cancelToken) + : PendingOperation(context, true, cancelToken) + { + /// + /// The operation type + /// + public override BackendActionType Operation => BackendActionType.QuotaInfo; + + /// + /// Executes the operation + /// + /// The backend to get the quota info from + /// The cancellation token + /// An awaitable task + public override async Task ExecuteAsync(IBackend backend, CancellationToken cancelToken) + { + IQuotaInfo? quotaInfo = null; + if (backend is IQuotaEnabledBackend quotaEnabledBackend && !Context.Options.QuotaDisable) + { + Context.Statwriter.SendEvent(BackendActionType.QuotaInfo, BackendEventType.Started, null, 0); + quotaInfo = await quotaEnabledBackend.GetQuotaInfoAsync(cancelToken).ConfigureAwait(false); + } + + return quotaInfo; + } + + } +} \ No newline at end of file diff --git a/Duplicati/Library/Main/Backend/BackendManager.WaitForEmptyOperation.cs b/Duplicati/Library/Main/Backend/BackendManager.WaitForEmptyOperation.cs new file mode 100644 index 000000000..be792191d --- /dev/null +++ b/Duplicati/Library/Main/Backend/BackendManager.WaitForEmptyOperation.cs @@ -0,0 +1,37 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Duplicati.Library.Interface; + +namespace Duplicati.Library.Main.Backend; + +#nullable enable + +partial class BackendManager +{ + /// + /// Represents a pending QuotaInfo operation + /// + /// + /// Creates a new WaitForEmptyOperation + /// + /// The execution context + /// The cancellation token + private class WaitForEmptyOperation(ExecuteContext context, CancellationToken cancelToken) + : PendingOperation(context, true, cancelToken) + { + /// + /// The operation type + /// + public override BackendActionType Operation => BackendActionType.WaitForEmpty; + + /// + /// Executes the operation + /// + /// The backend to get the quota info from + /// The cancellation token + /// An awaitable task + public override Task ExecuteAsync(IBackend backend, CancellationToken cancelToken) + => Task.FromResult(true); + } +} \ No newline at end of file diff --git a/Duplicati/Library/Main/Backend/BackendManager.cs b/Duplicati/Library/Main/Backend/BackendManager.cs new file mode 100644 index 000000000..2e8cedf1d --- /dev/null +++ b/Duplicati/Library/Main/Backend/BackendManager.cs @@ -0,0 +1,363 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using CoCoL; +using Duplicati.Library.Interface; +using Duplicati.Library.Main.Database; +using Duplicati.Library.Main.Operation.Common; +using Duplicati.Library.Main.Volumes; +using Duplicati.Library.Utility; + +namespace Duplicati.Library.Main.Backend; + +#nullable enable + +/// +/// The backend manager +/// +internal partial class BackendManager : IBackendManager +{ + /// + /// The log tag for the class + /// + private static readonly string LOGTAG = Logging.Log.LogTagFromType(); + /// + /// List of backend instances that are currently in use + /// + private readonly List backendPool = []; + + /// + /// The channel for issuing and handling requests + /// + private readonly IChannel requestChannel = ChannelManager.CreateChannel(name: "BackendManager"); + + /// + /// The queue runner task + /// + private readonly Task queueRunner; + + /// + /// The last file read size + /// + public long LastReadSize { get; private set; } + /// + /// The last file write size + /// + public long LastWriteSize { get; private set; } + + /// + /// The execution context + /// + private readonly ExecuteContext context; + + /// + /// Flag keeping track of whether the object has been disposed + /// + private bool isDisposed = false; + + /// + /// Initializes a new instance of the class. + /// + /// The backend URL + /// The options + /// The backend writer + /// The task reader + public BackendManager(string backendUrl, Options options, IBackendWriter backendWriter, ITaskReader taskReader) + { + if (string.IsNullOrWhiteSpace(backendUrl)) + throw new ArgumentNullException(nameof(backendUrl)); + + // To avoid excessive parameter passing, the context is captured here + context = new ExecuteContext( + new ProgressHandler(options, backendWriter, taskReader).HandleProgress, + backendWriter ?? throw new ArgumentNullException(nameof(backendWriter)), + new DatabaseCollector(), + taskReader ?? throw new ArgumentNullException(nameof(taskReader)), + options ?? throw new ArgumentNullException(nameof(options)) + ); + + // The BackendManager class is a wrapper that essentially sends + // requests into a queue and processes them in order. + // The Handler class is the one that actually processes the requests. + queueRunner = Handler.RunHandlerAsync( + requestChannel, + backendUrl, + context); + } + + /// + /// Enters a task into the queue for processing. + /// + /// The operation to queue + /// An awaitable task + private async Task QueueTask(PendingOperationBase op) + { + if (queueRunner.IsCompleted) + { + if (queueRunner.IsFaulted) + await queueRunner.ConfigureAwait(false); + if (queueRunner.IsCanceled) + throw new OperationCanceledException("Backend manager is stopped", queueRunner.Exception); + throw new InvalidOperationException("Backend manager is stopped"); + } + + try + { + await requestChannel.WriteAsync(op).ConfigureAwait(false); + } + catch (RetiredException ex) + { + // Try to get a better error message + if (queueRunner.IsFaulted) + await queueRunner.ConfigureAwait(false); + + throw new InvalidOperationException("Backend manager is stopped", ex); + } + } + + /// + /// Calculates the hash of a file + /// + /// The filename + /// The options + /// The hash + protected static string CalculateFileHash(string filename, Options options) + { + using (var fs = System.IO.File.OpenRead(filename)) + using (var hasher = HashFactory.CreateHasher(options.FileHashAlgorithm)) + return Convert.ToBase64String(hasher.ComputeHash(fs)); + } + + /// + /// Deletes a remote file + /// + /// The name of the remote file + /// The size of the remote file, for statistics + /// True if the operation should wait for the file to actually be deleted. If this argument is false, the task will complete once the operation is queued + /// The cancellation token + /// An awaitable task + public async Task DeleteAsync(string remotename, long size, bool waitForComplete, CancellationToken cancelToken) + { + var op = new DeleteOperation(remotename, size, context, waitForComplete, cancelToken); + await QueueTask(op).ConfigureAwait(false); + await op.GetResult().ConfigureAwait(false); + } + + /// + /// Gets a file from the remote location + /// + /// The name of the remote file + /// The hash of the remote file, for verification + /// The size of the remote file, for verification + /// The cancellation token + /// A temporary file with the contents of the remote file + public async Task GetAsync(string remotename, string hash, long size, CancellationToken cancelToken) + { + var op = new GetOperation(remotename, size, context, cancelToken) + { + Hash = hash + }; + await QueueTask(op).ConfigureAwait(false); + (var file, var _, var downloadSize) = await op.GetResult().ConfigureAwait(false); + LastReadSize = downloadSize; + return file; + } + + /// + /// Gets quota information from the backend + /// + /// The cancellation token + /// The quota information + public async Task GetQuotaInfoAsync(CancellationToken cancelToken) + { + var op = new QuotaInfoOperation(context, cancelToken); + await QueueTask(op).ConfigureAwait(false); + return await op.GetResult().ConfigureAwait(false); + } + + /// + /// Gets a file from the remote location, along with the hash and size of the file + /// + /// The name of the remote file + /// The hash of the remote file, or null if not known + /// The size of the remote file, or -1 if not known + /// The cancellation token + /// A tuple containing the temporary file, the hash of the file, and the size of the file + public async Task<(TempFile File, string Hash, long Size)> GetWithInfoAsync(string remotename, string hash, long size, CancellationToken cancelToken) + { + var op = new GetOperation(remotename, size, context, cancelToken) + { + Hash = hash + }; + await QueueTask(op).ConfigureAwait(false); + (var file, var downloadHash, var downloadSize) = await op.GetResult().ConfigureAwait(false); + LastReadSize = downloadSize; + return (file, downloadHash, downloadSize); + } + + /// + /// Lists files on the remote destination + /// + /// The cancellation token + /// The list of files + public async Task> ListAsync(CancellationToken cancelToken) + { + var op = new ListOperation(context, cancelToken); + await QueueTask(op).ConfigureAwait(false); + return await op.GetResult().ConfigureAwait(false); + } + + /// + /// Uploads a volume to the remote location + /// + /// The volume to upload + /// The index volume to upload, if any + /// The callback to call when the index volume is finished + /// True if the operation should wait for the file to actually be uploaded. If this argument is false, the task will complete once the operation is queued + /// The cancellation token + /// An awaitable task + public async Task PutAsync(VolumeWriterBase volume, IndexVolumeWriter? indexVolume, Action? indexVolumeFinished, bool waitForComplete, CancellationToken cancelToken) + { + volume.Close(); + + var op = new PutOperation(volume.RemoteFilename, context, waitForComplete, cancelToken) + { + LocalTempfile = volume.TempFile, + OriginalIndexFile = indexVolume, + Unencrypted = false, + TrackedInDb = true, + IndexVolumeFinishedCallback = indexVolumeFinished + }; + + // Prepare encryption + op.StartEncryptionAndHashing(); + await QueueTask(op).ConfigureAwait(false); + await op.GetResult().ConfigureAwait(false); + } + + /// + /// Uploads a verification file to the remote location without encryption + /// + /// The name of the remote file + /// The temporary file to upload + /// The cancellation token + /// An awaitable task + public async Task PutVerificationFileAsync(string remotename, TempFile tempFile, CancellationToken cancelToken) + { + var op = new PutOperation(remotename, context, true, cancelToken) + { + LocalTempfile = tempFile, + Unencrypted = true, // Avoid encrypting + TrackedInDb = false, // Not tracked + OriginalIndexFile = null, + IndexVolumeFinishedCallback = null + }; + + // Sets the task as already completed + op.StartEncryptionAndHashing(); + await QueueTask(op).ConfigureAwait(false); + await op.GetResult().ConfigureAwait(false); + } + + /// + /// Waits for the backend queue to be empty and flushes the database messages + /// + /// The cancellation token + /// An awaitable task + public async Task WaitForEmptyAsync(LocalDatabase database, IDbTransaction? transaction, CancellationToken cancellationToken) + { + context.Database.FlushPendingMessages(database, transaction); + + var op = new WaitForEmptyOperation(context, cancellationToken); + await QueueTask(op).ConfigureAwait(false); + await op.GetResult().ConfigureAwait(false); + + context.Database.FlushPendingMessages(database, transaction); + } + + /// + /// Stops the backend manager and flushes any pending messages to the database + /// + /// The database to write pending messages to + /// The transaction to use, if any + public async Task StopRunnerAndFlushMessages(LocalDatabase database, IDbTransaction? transaction) + { + await requestChannel.RetireAsync().ConfigureAwait(false); + context.Database.FlushPendingMessages(database, transaction); + if (queueRunner.IsFaulted) + Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerShutdown", queueRunner.Exception, "Backend manager queue runner crashed"); + } + + /// + /// Stops the backend manager and discards any pending messages + /// + public void StopRunnerAndDiscardMessages() + { + requestChannel.RetireAsync().Await(); + if (queueRunner.IsFaulted) + Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerShutdown", queueRunner.Exception, "Backend manager queue runner crashed"); + context.Database.ClearPendingMessages(); + } + + /// + /// Performs a download of the files specified, with pre-fetch to overlap the download and processing + /// + /// The volumes to download + /// The cancellation token + /// The downloaded files and the volume they came from + public async IAsyncEnumerable<(TempFile File, string Hash, long Size, string Name)> GetFilesOverlappedAsync(IEnumerable volumes, [EnumeratorCancellation] CancellationToken cancelToken) + { + var prevVolume = volumes.FirstOrDefault(); + if (prevVolume == null) + yield break; + + // Get the first volume, so we do not have pending parallel transfers + var prevResult = await GetWithInfoAsync(prevVolume.Name, prevVolume.Hash, prevVolume.Size, cancelToken); + + foreach (var volume in volumes.Skip(1)) + { + // Prepare the next volume, while processing the previous one + var nextTask = GetWithInfoAsync(volume.Name, volume.Hash, volume.Size, cancelToken); + + // Assuming we do not throw while yielding, otherwise we would need to dispose nextTask + yield return (prevResult.File, prevResult.Hash, prevResult.Size, prevVolume.Name); + prevResult.File.Dispose(); + + // Set up for next iteration + prevVolume = volume; + prevResult = await nextTask; + } + + // Return the last result + yield return (prevResult.File, prevResult.Hash, prevResult.Size, prevVolume.Name); + prevResult.File.Dispose(); + } + + /// + /// Disposes the backend manager + /// + public void Dispose() + { + if (isDisposed) + return; + + isDisposed = true; + requestChannel.RetireAsync().Await(); + context.Database.FlushMessagesToLog(); + + if (!queueRunner.IsCompleted) + { + Task.WhenAny(queueRunner, Task.Delay(1000)).Await(); + + if (!queueRunner.IsCompleted) + Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerShutdown", null, "Backend manager queue runner did not stop"); + if (queueRunner.IsFaulted) + Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerShutdown", queueRunner.Exception, "Backend manager queue runner crashed"); + + } + } +} diff --git a/Duplicati/Library/Main/BackendManager.cs b/Duplicati/Library/Main/BackendManager.cs deleted file mode 100644 index 8ecc6e692..000000000 --- a/Duplicati/Library/Main/BackendManager.cs +++ /dev/null @@ -1,1355 +0,0 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using Duplicati.Library.Utility; -using Duplicati.Library.Main.Database; -using Duplicati.Library.Main.Volumes; -using Newtonsoft.Json; -using Duplicati.Library.Localization.Short; -using System.Threading; -using System.Net; -using Duplicati.Library.Interface; -using System.Threading.Tasks; -using Duplicati.Library.Main.Operation.Common; - -namespace Duplicati.Library.Main -{ - internal class BackendManager : IDisposable - { - /// - /// The tag used for logging - /// - private static readonly string LOGTAG = Logging.Log.LogTagFromType(); - - /// - /// Class to represent hash failures - /// - [Serializable] - public class HashMismatchException : Exception - { - /// - /// Default constructor, sets a generic string as the message - /// - public HashMismatchException() : base() { } - - /// - /// Constructor with non-default message - /// - /// The exception message - public HashMismatchException(string message) : base(message) { } - - /// - /// Constructor with non-default message and inner exception details - /// - /// The exception message - /// The exception that caused this exception - public HashMismatchException(string message, Exception innerException) : base(message, innerException) { } - } - - private enum OperationType - { - Get, - Put, - List, - Delete, - CreateFolder, - Terminate, - Nothing - } - - public interface IDownloadWaitHandle - { - TempFile Wait(); - TempFile Wait(out string hash, out long size); - } - - private class FileEntryItem : IDownloadWaitHandle - { - /// - /// The current operation this entry represents - /// - public readonly OperationType Operation; - /// - /// The name of the remote file - /// - public string RemoteFilename; - /// - /// The name of the local file - /// - public string LocalFilename { get { return LocalTempfile; } } - /// - /// A reference to a temporary file that is disposed upon - /// failure or completion of the item - /// - public TempFile LocalTempfile; - /// - /// True if the item has been encrypted - /// - public bool Encrypted; - /// - /// The result object - /// - public object Result; - /// - /// The expected hash value of the file - /// - public string Hash; - /// - /// The expected size of the file - /// - public long Size; - /// - /// Reference to the index file entry that is updated if this entry changes - /// - public Tuple Indexfile; - /// - /// A flag indicating if the final hash and size of the block volume has been written to the index file - /// - public bool IndexfileUpdated; - /// - /// An exception that this item has caused - /// - public Exception Exception; - /// - /// True if an exception ultimately kills the handler, - /// false if the item is returned with an exception - /// - public readonly bool ExceptionKillsHandler; - /// - /// A flag indicating if the file is a extra metadata file - /// that has no entry in the database - /// - public bool NotTrackedInDb; - /// - /// A flag that indicates that the download is only checked for the hash and the file is not decrypted or returned - /// - public bool VerifyHashOnly; - - /// - /// The event that is signaled once the operation is complete or has failed - /// - private readonly System.Threading.ManualResetEvent DoneEvent; - - public FileEntryItem(OperationType operation, string remotefilename, Tuple indexfile = null) - { - Operation = operation; - RemoteFilename = remotefilename; - Indexfile = indexfile; - ExceptionKillsHandler = operation != OperationType.Get; - Size = -1; - - DoneEvent = new System.Threading.ManualResetEvent(false); - } - - public FileEntryItem(OperationType operation, string remotefilename, long size, string hash, Tuple indexfile = null) - : this(operation, remotefilename, indexfile) - { - Size = size; - Hash = hash; - } - - public void SetLocalfilename(string name) - { - this.LocalTempfile = Library.Utility.TempFile.WrapExistingFile(name); - this.LocalTempfile.Protected = true; - } - - public void SignalComplete() - { - DoneEvent.Set(); - } - - public void WaitForComplete() - { - DoneEvent.WaitOne(); - } - - TempFile IDownloadWaitHandle.Wait() - { - this.WaitForComplete(); - if (Exception != null) - throw Exception; - - return (TempFile)this.Result; - } - - TempFile IDownloadWaitHandle.Wait(out string hash, out long size) - { - this.WaitForComplete(); - - if (Exception != null) - throw Exception; - - hash = this.Hash; - size = this.Size; - - return (TempFile)this.Result; - } - - public void Encrypt(Library.Interface.IEncryption encryption, IBackendWriter stat) - { - if (encryption != null && !this.Encrypted) - { - var tempfile = new Library.Utility.TempFile(); - encryption.Encrypt(this.LocalFilename, tempfile); - this.DeleteLocalFile(stat); - this.LocalTempfile = tempfile; - this.Hash = null; - this.Size = 0; - this.Encrypted = true; - } - } - - public bool UpdateHashAndSize(Options options) - { - if (Hash == null || Size < 0) - { - Hash = CalculateFileHash(this.LocalFilename, options); - Size = new System.IO.FileInfo(this.LocalFilename).Length; - return true; - } - - return false; - } - - public void DeleteLocalFile(IBackendWriter stat) - { - if (this.LocalTempfile != null) - try { this.LocalTempfile.Dispose(); } - catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "DeleteTemporaryFileError", ex, "Failed to dispose temporary file: {0}", this.LocalTempfile); } - finally { this.LocalTempfile = null; } - } - - public BackendActionType BackendActionType - { - get - { - switch (this.Operation) - { - case OperationType.Get: - return BackendActionType.Get; - case OperationType.Put: - return BackendActionType.Put; - case OperationType.Delete: - return BackendActionType.Delete; - case OperationType.List: - return BackendActionType.List; - case OperationType.CreateFolder: - return BackendActionType.CreateFolder; - default: - throw new Exception(string.Format("Unexpected operation type: {0}", this.Operation)); - } - } - } - - } - - private class DatabaseCollector - { - private readonly object m_dbqueuelock = new object(); - private readonly LocalDatabase m_database; - private readonly System.Threading.Thread m_callerThread; - private List m_dbqueue; - - private interface IDbEntry { } - - private class DbOperation : IDbEntry - { - public string Action; - public string File; - public string Result; - } - - private class DbUpdate : IDbEntry - { - public string Remotename; - public RemoteVolumeState State; - public long Size; - public string Hash; - } - - private class DbRename : IDbEntry - { - public string Oldname; - public string Newname; - } - - public DatabaseCollector(LocalDatabase database) - { - m_database = database; - m_dbqueue = new List(); - if (m_database != null) - m_callerThread = System.Threading.Thread.CurrentThread; - } - - - public void LogDbOperation(string action, string file, string result) - { - lock (m_dbqueuelock) - m_dbqueue.Add(new DbOperation() { Action = action, File = file, Result = result }); - } - - public void LogDbUpdate(string remotename, RemoteVolumeState state, long size, string hash) - { - lock (m_dbqueuelock) - m_dbqueue.Add(new DbUpdate() { Remotename = remotename, State = state, Size = size, Hash = hash }); - } - - public void LogDbRename(string oldname, string newname) - { - lock (m_dbqueuelock) - m_dbqueue.Add(new DbRename() { Oldname = oldname, Newname = newname }); - } - - public bool FlushDbMessages(bool checkThread = false) - { - if (m_database != null && (checkThread == false || m_callerThread == System.Threading.Thread.CurrentThread)) - return FlushDbMessages(m_database, null); - - return false; - } - - public bool FlushDbMessages(LocalDatabase db, System.Data.IDbTransaction transaction) - { - List entries; - lock (m_dbqueuelock) - if (m_dbqueue.Count == 0) - return false; - else - { - entries = m_dbqueue; - m_dbqueue = new List(); - } - - // collect removed volumes for final db cleanup. - HashSet volsRemoved = new HashSet(); - - //As we replace the list, we can now freely access the elements without locking - foreach (var e in entries) - if (e is DbOperation operation) - db.LogRemoteOperation(operation.Action, operation.File, operation.Result, transaction); - else if (e is DbUpdate update && update.State == RemoteVolumeState.Deleted) - { - db.UpdateRemoteVolume(update.Remotename, RemoteVolumeState.Deleted, update.Size, update.Hash, true, TimeSpan.FromHours(2), transaction); - volsRemoved.Add(update.Remotename); - } - else if (e is DbUpdate dbUpdate) - db.UpdateRemoteVolume(dbUpdate.Remotename, dbUpdate.State, dbUpdate.Size, dbUpdate.Hash, transaction); - else if (e is DbRename rename) - db.RenameRemoteFile(rename.Oldname, rename.Newname, transaction); - else if (e != null) - Logging.Log.WriteErrorMessage(LOGTAG, "InvalidQueueElement", null, "Queue had element of type: {0}, {1}", e.GetType(), e); - - // Finally remove volumes from DB. - if (volsRemoved.Count > 0) - db.RemoveRemoteVolumes(volsRemoved); - - return true; - } - } - - private readonly BlockingQueue m_queue; - private readonly Options m_options; - private volatile Exception m_lastException; - private readonly Library.Interface.IEncryption m_encryption; - private readonly object m_encryptionLock = new object(); - private Library.Interface.IBackend m_backend; - private readonly string m_backendurl; - private readonly IBackendWriter m_statwriter; - private System.Threading.Thread m_thread; - private readonly ITaskReader m_taskReader; - private readonly DatabaseCollector m_db; - - // Cache these - private readonly int m_numberofretries; - private readonly TimeSpan m_retrydelay; - private readonly Boolean m_retrywithexponentialbackoff; - - public string BackendUrl { get { return m_backendurl; } } - public IBackend BorrowBackend() - { - var backend = m_backend; - m_backend = null; - return backend; - } - - public void ResetBackend(IBackend instance) - { - m_backend = instance ?? DynamicLoader.BackendLoader.GetBackend(m_backendurl, m_options.RawOptions); - } - - - public BackendManager(string backendurl, Options options, IBackendWriter statwriter, LocalDatabase database) - { - m_options = options; - m_backendurl = backendurl; - m_statwriter = statwriter; - m_taskReader = (statwriter as BasicResults).TaskControl; - m_numberofretries = options.NumberOfRetries; - m_retrydelay = options.RetryDelay; - m_retrywithexponentialbackoff = options.RetryWithExponentialBackoff; - - m_db = new DatabaseCollector(database); - - m_backend = DynamicLoader.BackendLoader.GetBackend(m_backendurl, m_options.RawOptions); - if (m_backend == null) - { - string shortname = m_backendurl; - - // Try not to leak hostnames or other information in the error messages - try { shortname = new Library.Utility.Uri(shortname).Scheme; } - catch { } - - throw new Duplicati.Library.Interface.UserInformationException(string.Format("Backend not supported: {0}", shortname), "BackendNotSupported"); - } - - if (!m_options.NoEncryption) - { - m_encryption = DynamicLoader.EncryptionLoader.GetModule(m_options.EncryptionModule, m_options.Passphrase, m_options.RawOptions); - if (m_encryption == null) - throw new Duplicati.Library.Interface.UserInformationException(string.Format("Encryption method not supported: {0}", m_options.EncryptionModule), "EncryptionMethodNotSupported"); - } - - m_queue = new BlockingQueue(options.SynchronousUpload ? 1 : (options.AsynchronousUploadLimit == 0 ? int.MaxValue : options.AsynchronousUploadLimit)); - m_thread = new System.Threading.Thread(this.ThreadRun); - m_thread.Name = "Backend Async Worker"; - m_thread.IsBackground = true; - m_thread.Start(); - } - - public static string CalculateFileHash(string filename, Options options) - { - using (System.IO.FileStream fs = System.IO.File.OpenRead(filename)) - return CalculateFileHash(fs, options); - } - - /// Calculate file hash directly on stream object (for piping) - public static string CalculateFileHash(System.IO.Stream stream, Options options) - { - using (var hasher = HashFactory.CreateHasher(options.FileHashAlgorithm)) - return Convert.ToBase64String(hasher.ComputeHash(stream)); - } - - - private void ThreadRun() - { - var uploadSuccess = false; - while (!m_queue.Completed) - { - var item = m_queue.Dequeue(); - if (item != null) - { - int retries = 0; - Exception lastException = null; - - do - { - try - { - m_taskReader?.ProgressRendevouz().Await(); - - if (m_options.NoConnectionReuse && m_backend != null) - { - m_backend.Dispose(); - m_backend = null; - } - - if (m_backend == null) - m_backend = DynamicLoader.BackendLoader.GetBackend(m_backendurl, m_options.RawOptions); - if (m_backend == null) - throw new Exception("Backend failed to re-load"); - - using (new Logging.Timer(LOGTAG, string.Format("RemoteOperation{0}", item.Operation), string.Format("RemoteOperation{0}", item.Operation))) - switch (item.Operation) - { - case OperationType.Put: - DoPutAsync(item).Await(); - // We do not auto create folders, - // because we know the folder exists - uploadSuccess = true; - break; - case OperationType.Get: - DoGetAsync(item).Await(); - break; - case OperationType.List: - DoList(item); - break; - case OperationType.Delete: - DoDeleteAsync(item).Await(); - break; - case OperationType.CreateFolder: - DoCreateFolderAsync(item).Await(); - break; - case OperationType.Terminate: - m_queue.SetCompleted(); - break; - case OperationType.Nothing: - item.SignalComplete(); - break; - } - - lastException = null; - retries = m_numberofretries; - } - catch (Exception ex) - { - retries++; - lastException = ex; - Logging.Log.WriteRetryMessage(LOGTAG, $"Retry{item.Operation}", ex, "Operation {0} with file {1} attempt {2} of {3} failed with message: {4}", item.Operation, item.RemoteFilename, retries, m_numberofretries, ex.Message); - - // If the thread is aborted, we exit here - if (ex is System.Threading.ThreadAbortException) - { - m_queue.SetCompleted(); - item.Exception = ex; - item.SignalComplete(); - throw; - } - - if (ex is WebException exception) - { - // Refresh DNS name if we fail to connect in order to prevent issues with incorrect DNS entries - if (exception.Status == System.Net.WebExceptionStatus.NameResolutionFailure) - { - try - { - var names = m_backend.GetDNSNamesAsync(m_taskReader.TransferToken).Await() ?? new string[0]; - foreach (var name in names) - if (!string.IsNullOrWhiteSpace(name)) - System.Net.Dns.GetHostEntry(name); - } - catch - { - } - } - } - - m_statwriter.SendEvent(item.BackendActionType, retries < m_numberofretries ? BackendEventType.Retrying : BackendEventType.Failed, item.RemoteFilename, item.Size); - - bool recovered = false; - if (!uploadSuccess && ex is Duplicati.Library.Interface.FolderMissingException && m_options.AutocreateFolders) - { - try - { - // If we successfully create the folder, we can re-use the connection - m_backend.CreateFolderAsync(m_taskReader.TransferToken).Await(); - recovered = true; - } - catch (Exception dex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "FolderCreateError", dex, "Failed to create folder: {0}", ex.Message); - } - } - - // To work around the Apache WEBDAV issue, we rename the file here - if (item.Operation == OperationType.Put && retries < m_numberofretries && !item.NotTrackedInDb) - RenameFileAfterError(item); - - if (!recovered) - { - try { m_backend.Dispose(); } - catch (Exception dex) { Logging.Log.WriteWarningMessage(LOGTAG, "BackendDisposeError", dex, "Failed to dispose backend instance: {0}", ex.Message); } - - m_backend = null; - - if (retries < m_numberofretries && m_retrydelay.Ticks != 0) - { - var delay = Library.Utility.Utility.GetRetryDelay(m_retrydelay, retries, m_retrywithexponentialbackoff); - var target = DateTime.Now.Add(delay); - - while (target > DateTime.Now) - { - if (m_taskReader?.ProgressToken.IsCancellationRequested ?? false) - break; - - System.Threading.Thread.Sleep(500); - } - } - } - } - - - } while (retries < m_numberofretries); - - if (lastException != null && !(lastException is Duplicati.Library.Interface.FileMissingException) && item.Operation == OperationType.Delete) - { - Logging.Log.WriteInformationMessage(LOGTAG, "DeleteFileFailed", LC.L("Failed to delete file {0}, testing if file exists", item.RemoteFilename)); - try - { - if (!m_backend.List().Select(x => x.Name).Contains(item.RemoteFilename)) - { - lastException = null; - Logging.Log.WriteInformationMessage(LOGTAG, "DeleteFileFailureRecovered", LC.L("Recovered from problem with attempting to delete non-existing file {0}", item.RemoteFilename)); - } - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "DeleteFileFailure", ex, LC.L("Failed to recover from error deleting file {0}", item.RemoteFilename), ex); - } - } - - if (lastException != null) - { - item.Exception = lastException; - if (item.Operation == OperationType.Put) - item.DeleteLocalFile(m_statwriter); - - if (item.ExceptionKillsHandler) - { - m_lastException = lastException; - - //TODO: If there are temp files in the queue, we must delete them - m_queue.SetCompleted(); - } - - } - - item.SignalComplete(); - } - } - - //Make sure everything in the queue is signalled - FileEntryItem i; - while ((i = m_queue.Dequeue()) != null) - i.SignalComplete(); - } - - private void RenameFileAfterError(FileEntryItem item) - { - var p = VolumeBase.ParseFilename(item.RemoteFilename); - var guid = VolumeWriterBase.GenerateGuid(); - var time = p.Time.Ticks == 0 ? p.Time : p.Time.AddSeconds(1); - var newname = VolumeBase.GenerateFilename(p.FileType, p.Prefix, guid, time, p.CompressionModule, p.EncryptionModule); - var oldname = item.RemoteFilename; - - m_statwriter.SendEvent(item.BackendActionType, BackendEventType.Rename, oldname, item.Size); - m_statwriter.SendEvent(item.BackendActionType, BackendEventType.Rename, newname, item.Size); - Logging.Log.WriteInformationMessage(LOGTAG, "RenameRemoteTargetFile", "Renaming \"{0}\" to \"{1}\"", oldname, newname); - m_db.LogDbRename(oldname, newname); - item.RemoteFilename = newname; - - // If there is an index file attached to the block file, - // it references the block filename, so we create a new index file - // which is a copy of the current, but with the new name - if (item.Indexfile != null) - { - if (!item.IndexfileUpdated) - { - item.Indexfile.Item1.FinishVolume(item.Hash, item.Size); - item.Indexfile.Item1.Close(); - item.IndexfileUpdated = true; - } - - IndexVolumeWriter wr = null; - try - { - var hashsize = HashFactory.HashSizeBytes(m_options.BlockHashAlgorithm); - wr = new IndexVolumeWriter(m_options); - using (var rd = new IndexVolumeReader(p.CompressionModule, item.Indexfile.Item2.LocalFilename, m_options, hashsize)) - wr.CopyFrom(rd, x => x == oldname ? newname : x); - item.Indexfile.Item1.Dispose(); - item.Indexfile = new Tuple(wr, item.Indexfile.Item2); - item.Indexfile.Item2.LocalTempfile.Dispose(); - item.Indexfile.Item2.LocalTempfile = wr.TempFile; - wr.Close(); - } - catch - { - wr?.Dispose(); - throw; - } - } - } - - private string m_lastThrottleUploadValue = null; - private string m_lastThrottleDownloadValue = null; - - private void HandleProgress(ThrottledStream ts, long pg) - { - // This pauses and throws on cancellation, but ignores stop - m_taskReader?.TransferRendevouz(); - - // Update the throttle speeds if they have changed - string tmp; - m_options.RawOptions.TryGetValue("throttle-upload", out tmp); - if (tmp != m_lastThrottleUploadValue) - { - ts.WriteSpeed = m_options.MaxUploadPrSecond; - m_lastThrottleUploadValue = tmp; - } - - m_options.RawOptions.TryGetValue("throttle-download", out tmp); - if (tmp != m_lastThrottleDownloadValue) - { - ts.ReadSpeed = m_options.MaxDownloadPrSecond; - m_lastThrottleDownloadValue = tmp; - } - - m_statwriter.BackendProgressUpdater.UpdateProgress(pg); - } - - private async Task DoPutAsync(FileEntryItem item) - { - if (m_encryption != null) - lock (m_encryptionLock) - item.Encrypt(m_encryption, m_statwriter); - - if (item.UpdateHashAndSize(m_options) && !item.NotTrackedInDb) - m_db.LogDbUpdate(item.RemoteFilename, RemoteVolumeState.Uploading, item.Size, item.Hash); - - if (item.Indexfile != null && !item.IndexfileUpdated) - { - item.Indexfile.Item1.FinishVolume(item.Hash, item.Size); - item.Indexfile.Item1.Close(); - item.IndexfileUpdated = true; - } - - m_db.LogDbOperation("put", item.RemoteFilename, JsonConvert.SerializeObject(new { Size = item.Size, Hash = item.Hash })); - m_statwriter.SendEvent(BackendActionType.Put, BackendEventType.Started, item.RemoteFilename, item.Size); - - var begin = DateTime.Now; - - if (m_backend is Library.Interface.IStreamingBackend streamingBackend && !m_options.DisableStreamingTransfers) - { - using (var fs = System.IO.File.OpenRead(item.LocalFilename)) - using (var act = new Duplicati.StreamUtil.TimeoutObservingStream(fs) { ReadTimeout = m_backend is ITimeoutExemptBackend ? Timeout.Infinite : m_options.ReadWriteTimeout }) - using (var ts = new ThrottledStream(act, m_options.MaxUploadPrSecond, 0)) - using (var pgs = new Library.Utility.ProgressReportingStream(ts, pg => HandleProgress(ts, pg))) - using (var linkedToken = System.Threading.CancellationTokenSource.CreateLinkedTokenSource(m_taskReader.TransferToken, act.TimeoutToken)) - await streamingBackend.PutAsync(item.RemoteFilename, pgs, linkedToken.Token); - } - else - await m_backend.PutAsync(item.RemoteFilename, item.LocalFilename, m_taskReader.TransferToken); - - var duration = DateTime.Now - begin; - Logging.Log.WriteProfilingMessage(LOGTAG, "UploadSpeed", "Uploaded {0} in {1}, {2}/s", Library.Utility.Utility.FormatSizeString(item.Size), duration, Library.Utility.Utility.FormatSizeString((long)(item.Size / duration.TotalSeconds))); - - if (!item.NotTrackedInDb) - m_db.LogDbUpdate(item.RemoteFilename, RemoteVolumeState.Uploaded, item.Size, item.Hash); - - m_statwriter.SendEvent(BackendActionType.Put, BackendEventType.Completed, item.RemoteFilename, item.Size); - - if (m_options.ListVerifyUploads) - { - var f = m_backend.List().FirstOrDefault(n => n.Name.Equals(item.RemoteFilename, StringComparison.OrdinalIgnoreCase)); - if (f == null) - throw new Exception(string.Format("List verify failed, file was not found after upload: {0}", item.RemoteFilename)); - else if (f.Size != item.Size && f.Size >= 0) - throw new Exception(string.Format("List verify failed for file: {0}, size was {1} but expected to be {2}", f.Name, f.Size, item.Size)); - } - - item.DeleteLocalFile(m_statwriter); - } - - private async Task<(TempFile tempFile, long downloadSize, string remotehash)> DoGetFileAsync(FileEntryItem item, IEncryption useDecrypter) - { - TempFile retTarget, dlTarget = null, decryptTarget = null; - long retDownloadSize; - string retHashcode; - try - { - dlTarget = new Library.Utility.TempFile(); - if (m_backend is Library.Interface.IStreamingBackend streamingBackend && !m_options.DisableStreamingTransfers) - { - // extended to use stacked streams - using (var fs = System.IO.File.OpenWrite(dlTarget)) - using (var act = new Duplicati.StreamUtil.TimeoutObservingStream(fs) { WriteTimeout = m_backend is ITimeoutExemptBackend ? Timeout.Infinite : m_options.ReadWriteTimeout }) - using (var hasher = HashFactory.CreateHasher(m_options.FileHashAlgorithm)) - using (var hs = new HashCalculatingStream(act, hasher)) - using (var ss = new ShaderStream(hs, true)) - using (var linkedToken = System.Threading.CancellationTokenSource.CreateLinkedTokenSource(m_taskReader.TransferToken, act.TimeoutToken)) - { - // NOTE: It is possible to hash the file in parallel with download - // but this requires some careful handling of buffers and threads/tasks - // to avoid adding more overhead than what is gained. - - using (var ts = new ThrottledStream(ss, 0, m_options.MaxDownloadPrSecond)) - using (var pgs = new Library.Utility.ProgressReportingStream(ts, pg => HandleProgress(ts, pg))) - { await streamingBackend.GetAsync(item.RemoteFilename, pgs, linkedToken.Token); } - ss.Flush(); - retDownloadSize = ss.TotalBytesWritten; - retHashcode = Convert.ToBase64String(hs.GetFinalHash()); - } - } - else - { - await m_backend.GetAsync(item.RemoteFilename, dlTarget, m_taskReader.TransferToken); - retDownloadSize = new System.IO.FileInfo(dlTarget).Length; - retHashcode = CalculateFileHash(dlTarget, m_options); - } - - // Decryption is not placed in the stream stack because there seemed to be an effort - // to throw a CryptographicException on fail. If in main stack, we cannot differentiate - // in which part of the stack the source of an exception resides. - if (useDecrypter != null) - { - decryptTarget = new Library.Utility.TempFile(); - lock (m_encryptionLock) - { - try { useDecrypter.Decrypt(dlTarget, decryptTarget); } - // If we fail here, make sure that we throw a crypto exception - catch (System.Security.Cryptography.CryptographicException) { throw; } - catch (Exception ex) { throw new System.Security.Cryptography.CryptographicException(ex.Message, ex); } - } - retTarget = decryptTarget; - decryptTarget = null; - } - else - { - retTarget = dlTarget; - dlTarget = null; - } - } - finally - { - if (dlTarget != null) dlTarget.Dispose(); - if (decryptTarget != null) decryptTarget.Dispose(); - } - - return (retTarget, retDownloadSize, retHashcode); - } - - private async Task DoGetAsync(FileEntryItem item) - { - Library.Utility.TempFile tmpfile = null; - m_statwriter.SendEvent(BackendActionType.Get, BackendEventType.Started, item.RemoteFilename, item.Size); - - try - { - var begin = DateTime.Now; - - // We already know the filename, so we put the decision about if and which decryptor to - // use prior to download. This allows to set up stacked streams or a pipe doing decryption - Interface.IEncryption useDecrypter = null; - if (!item.VerifyHashOnly && !m_options.NoEncryption) - { - useDecrypter = m_encryption; - { - lock (m_encryptionLock) - { - try - { - // Auto-guess the encryption module - var ext = (System.IO.Path.GetExtension(item.RemoteFilename) ?? "").TrimStart('.'); - if (!m_encryption.FilenameExtension.Equals(ext, StringComparison.OrdinalIgnoreCase)) - { - // Check if the file is encrypted with something else - if (DynamicLoader.EncryptionLoader.Keys.Contains(ext, StringComparer.OrdinalIgnoreCase)) - { - Logging.Log.WriteVerboseMessage(LOGTAG, "AutomaticDecryptionDetection", "Filename extension \"{0}\" does not match encryption module \"{1}\", using matching encryption module", ext, m_options.EncryptionModule); - useDecrypter = DynamicLoader.EncryptionLoader.GetModule(ext, m_options.Passphrase, m_options.RawOptions); - useDecrypter = useDecrypter ?? m_encryption; - } - // Check if the file is not encrypted - else if (DynamicLoader.CompressionLoader.Keys.Contains(ext, StringComparer.OrdinalIgnoreCase)) - { - Logging.Log.WriteVerboseMessage(LOGTAG, "AutomaticDecryptionDetection", "Filename extension \"{0}\" does not match encryption module \"{1}\", guessing that it is not encrypted", ext, m_options.EncryptionModule); - useDecrypter = null; - } - // Fallback, lets see what happens... - else - { - Logging.Log.WriteVerboseMessage(LOGTAG, "AutomaticDecryptionDetection", "Filename extension \"{0}\" does not match encryption module \"{1}\", attempting to use specified encryption module as no others match", ext, m_options.EncryptionModule); - } - } - } - // If we fail here, make sure that we throw a crypto exception - catch (System.Security.Cryptography.CryptographicException) { throw; } - catch (Exception ex) { throw new System.Security.Cryptography.CryptographicException(ex.Message, ex); } - } - } - } - - (tmpfile, var dataSizeDownloaded, var fileHash) = await DoGetFileAsync(item, useDecrypter); - - var duration = DateTime.Now - begin; - Logging.Log.WriteProfilingMessage(LOGTAG, "DownloadSpeed", "Downloaded {3}{0} in {1}, {2}/s", Library.Utility.Utility.FormatSizeString(dataSizeDownloaded), - duration, Library.Utility.Utility.FormatSizeString((long)(dataSizeDownloaded / duration.TotalSeconds)), - useDecrypter == null ? "" : "and decrypted "); - - m_db.LogDbOperation("get", item.RemoteFilename, JsonConvert.SerializeObject(new { Size = dataSizeDownloaded, Hash = fileHash })); - m_statwriter.SendEvent(BackendActionType.Get, BackendEventType.Completed, item.RemoteFilename, dataSizeDownloaded); - - if (!m_options.SkipFileHashChecks) - { - if (item.Size >= 0) - { - if (dataSizeDownloaded != item.Size) - throw new Exception(Strings.Controller.DownloadedFileSizeError(item.RemoteFilename, dataSizeDownloaded, item.Size)); - } - else - item.Size = dataSizeDownloaded; - - if (!string.IsNullOrEmpty(item.Hash)) - { - if (fileHash != item.Hash) - throw new HashMismatchException(Strings.Controller.HashMismatchError(tmpfile, item.Hash, fileHash)); - } - else - item.Hash = fileHash; - } - - if (item.VerifyHashOnly) - { - tmpfile.Dispose(); - } - else - { - item.Result = tmpfile; - tmpfile = null; - } - - } - catch - { - if (tmpfile != null) - tmpfile.Dispose(); - - throw; - } - } - - private void DoList(FileEntryItem item) - { - m_statwriter.SendEvent(BackendActionType.List, BackendEventType.Started, null, -1); - - var r = m_backend.List().ToList(); - - StringBuilder sb = new StringBuilder(); - sb.AppendLine("["); - long count = 0; - foreach (var e in r) - { - if (count != 0) - sb.AppendLine(","); - count++; - sb.Append(JsonConvert.SerializeObject(e)); - } - - sb.AppendLine(); - sb.Append("]"); - m_db.LogDbOperation("list", "", sb.ToString()); - item.Result = r; - - m_statwriter.SendEvent(BackendActionType.List, BackendEventType.Completed, null, r.Count); - } - - private async Task DoDeleteAsync(FileEntryItem item) - { - m_statwriter.SendEvent(BackendActionType.Delete, BackendEventType.Started, item.RemoteFilename, item.Size); - - string result = null; - try - { - await m_backend.DeleteAsync(item.RemoteFilename, m_taskReader.TransferToken).ConfigureAwait(false); - } - catch (Exception ex) - { - var isFileMissingException = ex is Library.Interface.FileMissingException || ex is System.IO.FileNotFoundException; - var wr = ex as System.Net.WebException == null ? null : (ex as System.Net.WebException).Response as System.Net.HttpWebResponse; - - if (isFileMissingException || (wr != null && wr.StatusCode == System.Net.HttpStatusCode.NotFound)) - { - Logging.Log.WriteInformationMessage(LOGTAG, "DeleteRemoteFileFailed", LC.L("Delete operation failed for {0} with FileNotFound, listing contents", item.RemoteFilename)); - bool success = false; - - try - { - success = !m_backend.List().Select(x => x.Name).Contains(item.RemoteFilename); - } - catch - { - } - - if (success) - { - Logging.Log.WriteInformationMessage(LOGTAG, "DeleteRemoteFileSuccess", LC.L("Listing indicates file {0} was deleted correctly", item.RemoteFilename)); - return; - } - else - { - Logging.Log.WriteWarningMessage(LOGTAG, "DeleteRemoteFileFailed", ex, LC.L("Listing confirms file {0} was not deleted", item.RemoteFilename)); - } - } - - result = ex.ToString(); - throw; - } - finally - { - m_db.LogDbOperation("delete", item.RemoteFilename, result); - } - - m_db.LogDbUpdate(item.RemoteFilename, RemoteVolumeState.Deleted, -1, null); - m_statwriter.SendEvent(BackendActionType.Delete, BackendEventType.Completed, item.RemoteFilename, item.Size); - } - - private async Task DoCreateFolderAsync(FileEntryItem item) - { - m_statwriter.SendEvent(BackendActionType.CreateFolder, BackendEventType.Started, null, -1); - - string result = null; - try - { - await m_backend.CreateFolderAsync(m_taskReader.TransferToken).ConfigureAwait(false); - } - catch (Exception ex) - { - result = ex.ToString(); - throw; - } - finally - { - m_db.LogDbOperation("createfolder", item.RemoteFilename, result); - } - - m_statwriter.SendEvent(BackendActionType.CreateFolder, BackendEventType.Completed, null, -1); - } - - public void PutUnencrypted(string remotename, string localpath) - { - if (m_lastException != null) - throw m_lastException; - - var req = new FileEntryItem(OperationType.Put, remotename, null); - req.SetLocalfilename(localpath); - req.Encrypted = true; //Prevent encryption - req.NotTrackedInDb = true; //Prevent Db updates - - try - { - m_statwriter.BackendProgressUpdater.SetBlocking(true); - if (m_queue.Enqueue(req) && m_options.SynchronousUpload) - { - req.WaitForComplete(); - if (req.Exception != null) - throw req.Exception; - } - } - finally - { - m_statwriter.BackendProgressUpdater.SetBlocking(false); - } - - if (m_lastException != null) - throw m_lastException; - } - - public void Put(VolumeWriterBase item, IndexVolumeWriter indexfile = null, Action indexVolumeFinishedCallback = null, bool synchronous = false) - { - if (m_lastException != null) - throw m_lastException; - - item.Close(); - var req = new FileEntryItem(OperationType.Put, item.RemoteFilename, null); - req.LocalTempfile = item.TempFile; - - if (m_lastException != null) - throw m_lastException; - - FileEntryItem req2 = null; - - // As the network link is the bottleneck, - // we encrypt the dblock volume before the - // upload is enqueued (i.e. on the worker thread) - if (m_encryption != null) - lock (m_encryptionLock) - req.Encrypt(m_encryption, m_statwriter); - - req.UpdateHashAndSize(m_options); - m_db.LogDbUpdate(item.RemoteFilename, RemoteVolumeState.Uploading, req.Size, req.Hash); - - // We do not encrypt the dindex volume, because it is small, - // and may need to be re-written if the dblock upload is retried - if (indexfile != null) - { - m_db.LogDbUpdate(indexfile.RemoteFilename, RemoteVolumeState.Uploading, -1, null); - req2 = new FileEntryItem(OperationType.Put, indexfile.RemoteFilename); - req2.LocalTempfile = indexfile.TempFile; - req.Indexfile = new Tuple(indexfile, req2); - - indexfile.FinishVolume(req.Hash, req.Size); - indexVolumeFinishedCallback?.Invoke(); - indexfile.Close(); - req.IndexfileUpdated = true; - } - - try - { - m_statwriter.BackendProgressUpdater.SetBlocking(true); - m_db.FlushDbMessages(true); - - if (m_queue.Enqueue(req) && (m_options.SynchronousUpload || synchronous)) - { - req.WaitForComplete(); - if (req.Exception != null) - throw req.Exception; - } - - if (req2 != null && m_queue.Enqueue(req2) && (m_options.SynchronousUpload || synchronous)) - { - req2.WaitForComplete(); - if (req2.Exception != null) - throw req2.Exception; - } - } - finally - { - m_statwriter.BackendProgressUpdater.SetBlocking(false); - } - - if (m_lastException != null) - throw m_lastException; - } - - public Library.Utility.TempFile GetWithInfo(string remotename, out long size, out string hash) - { - if (m_lastException != null) - throw m_lastException; - - hash = null; size = -1; - var req = new FileEntryItem(OperationType.Get, remotename, -1, null); - try - { - m_statwriter.BackendProgressUpdater.SetBlocking(true); - if (m_queue.Enqueue(req)) - ((IDownloadWaitHandle)req).Wait(out hash, out size); - } - finally - { - m_statwriter.BackendProgressUpdater.SetBlocking(false); - } - - if (m_lastException != null) - throw m_lastException; - - return (Library.Utility.TempFile)req.Result; - } - - public Library.Utility.TempFile Get(string remotename, long size, string hash) - { - if (m_lastException != null) - throw m_lastException; - - var req = new FileEntryItem(OperationType.Get, remotename, size, hash); - try - { - m_statwriter.BackendProgressUpdater.SetBlocking(true); - if (m_queue.Enqueue(req)) - ((IDownloadWaitHandle)req).Wait(); - } - finally - { - m_statwriter.BackendProgressUpdater.SetBlocking(false); - } - - if (m_lastException != null) - throw m_lastException; - - return (Library.Utility.TempFile)req.Result; - } - - public IDownloadWaitHandle GetAsync(string remotename, long size, string hash) - { - if (m_lastException != null) - throw m_lastException; - - var req = new FileEntryItem(OperationType.Get, remotename, size, hash); - try - { - m_statwriter.BackendProgressUpdater.SetBlocking(true); - if (m_queue.Enqueue(req)) - return req; - } - finally - { - m_statwriter.BackendProgressUpdater.SetBlocking(false); - } - - if (m_lastException != null) - throw m_lastException; - else - throw new InvalidOperationException("GetAsync called after backend is shut down"); - } - - public void GetForTesting(string remotename, long size, string hash) - { - if (m_lastException != null) - throw m_lastException; - - if (string.IsNullOrWhiteSpace(hash)) - throw new InvalidOperationException("Cannot test a file without the hash"); - - var req = new FileEntryItem(OperationType.Get, remotename, size, hash); - req.VerifyHashOnly = true; - try - { - m_statwriter.BackendProgressUpdater.SetBlocking(true); - if (m_queue.Enqueue(req)) - { - req.WaitForComplete(); - if (req.Exception != null) - throw req.Exception; - } - } - finally - { - m_statwriter.BackendProgressUpdater.SetBlocking(false); - } - - if (m_lastException != null) - throw m_lastException; - } - - public IList List() - { - if (m_lastException != null) - throw m_lastException; - - var req = new FileEntryItem(OperationType.List, null); - try - { - m_statwriter.BackendProgressUpdater.SetBlocking(true); - if (m_queue.Enqueue(req)) - { - req.WaitForComplete(); - if (req.Exception != null) - throw req.Exception; - } - } - finally - { - m_statwriter.BackendProgressUpdater.SetBlocking(false); - } - - if (m_lastException != null) - throw m_lastException; - - return (IList)req.Result; - } - - public void WaitForComplete(LocalDatabase db, System.Data.IDbTransaction transation) - { - m_statwriter.BackendProgressUpdater.SetBlocking(true); - m_db.FlushDbMessages(db, transation); - if (m_lastException != null) - throw m_lastException; - - var item = new FileEntryItem(OperationType.Terminate, null); - if (m_queue.Enqueue(item)) - item.WaitForComplete(); - - m_db.FlushDbMessages(db, transation); - - if (m_lastException != null) - throw m_lastException; - } - - public void WaitForEmpty(LocalDatabase db, System.Data.IDbTransaction transation) - { - try - { - m_statwriter.BackendProgressUpdater.SetBlocking(true); - m_db.FlushDbMessages(db, transation); - if (m_lastException != null) - throw m_lastException; - - var item = new FileEntryItem(OperationType.Nothing, null); - if (m_queue.Enqueue(item)) - item.WaitForComplete(); - - m_db.FlushDbMessages(db, transation); - - if (m_lastException != null) - throw m_lastException; - } - finally - { - m_statwriter.BackendProgressUpdater.SetBlocking(false); - } - } - - public void Delete(string remotename, long size, bool synchronous = false) - { - if (m_lastException != null) - throw m_lastException; - - m_db.LogDbUpdate(remotename, RemoteVolumeState.Deleting, size, null); - var req = new FileEntryItem(OperationType.Delete, remotename, size, null); - try - { - m_statwriter.BackendProgressUpdater.SetBlocking(true); - if (m_queue.Enqueue(req) && synchronous) - { - req.WaitForComplete(); - if (req.Exception != null) - throw req.Exception; - } - } - finally - { - m_statwriter.BackendProgressUpdater.SetBlocking(false); - } - - if (m_lastException != null) - throw m_lastException; - } - - public Task GetQuotaInfoAsync(CancellationToken cancelToken) - => (m_backend as IQuotaEnabledBackend)?.GetQuotaInfoAsync(cancelToken) ?? Task.FromResult(null); - - public bool FlushDbMessages() - { - return m_db.FlushDbMessages(false); - } - - public void Dispose() - { - if (m_queue != null && !m_queue.Completed) - m_queue.SetCompleted(); - - if (m_thread != null) - { - if (!m_thread.Join(TimeSpan.FromSeconds(10))) - { - m_thread.Interrupt(); - m_thread.Join(TimeSpan.FromSeconds(10)); - } - - m_thread = null; - } - - //TODO: We cannot null this, because it will be recreated - //Should we wait for queue completion or abort immediately? - if (m_backend != null) - { - m_backend.Dispose(); - m_backend = null; - } - - try { m_db.FlushDbMessages(true); } - catch (Exception ex) { Logging.Log.WriteErrorMessage(LOGTAG, "ShutdownError", ex, "Backend Shutdown error: {0}", ex.Message); } - } - } -} diff --git a/Duplicati/Library/Main/Controller.cs b/Duplicati/Library/Main/Controller.cs index 4f2336110..beff76f0c 100644 --- a/Duplicati/Library/Main/Controller.cs +++ b/Duplicati/Library/Main/Controller.cs @@ -29,6 +29,7 @@ using Duplicati.Library.Interface; using Duplicati.Library.Main.Database; using System.Threading.Tasks; using Duplicati.Library.Main.Operation.Common; +using System.IO; namespace Duplicati.Library.Main { @@ -41,7 +42,7 @@ namespace Duplicati.Library.Main /// /// The backend url /// - private string m_backend; + private string m_backendUrl; /// /// The parsed type-safe version of the commandline options /// @@ -83,11 +84,11 @@ namespace Duplicati.Library.Main /// /// Constructs a new interface for performing backup and restore operations /// - /// The url for the backend to use + /// The url for the backend to use /// All required options - public Controller(string backend, Dictionary options, IMessageSink messageSink) + public Controller(string backendUrl, Dictionary options, IMessageSink messageSink) { - m_backend = backend; + m_backendUrl = backendUrl; m_options = new Options(options); m_messageSink = messageSink; } @@ -115,19 +116,19 @@ namespace Duplicati.Library.Main public Duplicati.Library.Interface.IBackupResults Backup(string[] inputsources, IFilter filter = null) { - Library.UsageReporter.Reporter.Report("USE_BACKEND", new Library.Utility.Uri(m_backend).Scheme); + Library.UsageReporter.Reporter.Report("USE_BACKEND", new Library.Utility.Uri(m_backendUrl).Scheme); Library.UsageReporter.Reporter.Report("USE_COMPRESSION", m_options.CompressionModule); Library.UsageReporter.Reporter.Report("USE_ENCRYPTION", m_options.EncryptionModule); CheckAutoCompactInterval(); CheckAutoVacuumInterval(); - return RunAction(new BackupResults(), ref inputsources, ref filter, (result) => + return RunAction(new BackupResults(), ref inputsources, ref filter, (result, backendManager) => { - using (var h = new Operation.BackupHandler(m_backend, m_options, result)) + using (var h = new Operation.BackupHandler(m_backendUrl, m_options, result)) { - h.RunAsync(ExpandInputSources(inputsources, filter), filter).Await(); + h.RunAsync(ExpandInputSources(inputsources, filter), backendManager, filter).Await(); } Library.UsageReporter.Reporter.Report("BACKUP_FILECOUNT", result.ExaminedFiles); @@ -138,9 +139,9 @@ namespace Duplicati.Library.Main public Library.Interface.IRestoreResults Restore(string[] paths, Library.Utility.IFilter filter = null) { - return RunAction(new RestoreResults(), ref paths, ref filter, (result) => + return RunAction(new RestoreResults(), ref paths, ref filter, (result, backendManager) => { - new Operation.RestoreHandler(m_backend, m_options, result).Run(paths, filter); + new Operation.RestoreHandler(m_options, result).Run(paths, backendManager, filter); Library.UsageReporter.Reporter.Report("RESTORE_FILECOUNT", result.RestoredFiles); Library.UsageReporter.Reporter.Report("RESTORE_FILESIZE", result.SizeOfRestoredFiles); @@ -150,25 +151,25 @@ namespace Duplicati.Library.Main public Duplicati.Library.Interface.IRestoreControlFilesResults RestoreControlFiles(IEnumerable files = null, Library.Utility.IFilter filter = null) { - return RunAction(new RestoreControlFilesResults(), ref filter, (result) => + return RunAction(new RestoreControlFilesResults(), ref filter, (result, backendManager) => { - new Operation.RestoreControlFilesHandler(m_backend, m_options, result).Run(files, filter); + new Operation.RestoreControlFilesHandler(m_options, result).Run(files, backendManager, filter); }); } public Duplicati.Library.Interface.IDeleteResults Delete() { - return RunAction(new DeleteResults(), (result) => + return RunAction(new DeleteResults(), (result, backendManager) => { - new Operation.DeleteHandler(m_backend, m_options, result).Run(); + new Operation.DeleteHandler(m_options, result).Run(backendManager); }); } public Duplicati.Library.Interface.IRepairResults Repair(Library.Utility.IFilter filter = null) { - return RunAction(new RepairResults(), ref filter, (result) => + return RunAction(new RepairResults(), ref filter, (result, backendManager) => { - new Operation.RepairHandler(m_backend, m_options, result).Run(filter); + new Operation.RepairHandler(m_options, result).Run(backendManager, filter); }); } @@ -184,40 +185,39 @@ namespace Duplicati.Library.Main public Duplicati.Library.Interface.IListResults List(IEnumerable filterstrings, Library.Utility.IFilter filter) { - return RunAction(new ListResults(), ref filter, (result) => + return RunAction(new ListResults(), ref filter, (result, backendManager) => { - new Operation.ListFilesHandler(m_backend, m_options, result).Run(filterstrings, filter); + new Operation.ListFilesHandler(m_options, result).Run(backendManager, filterstrings, filter).Await(); }); } public Duplicati.Library.Interface.IListResults ListControlFiles(IEnumerable filterstrings, Library.Utility.IFilter filter) { - return RunAction(new ListResults(), ref filter, (result) => + return RunAction(new ListResults(), ref filter, (result, backendManager) => { - new Operation.ListControlFilesHandler(m_backend, m_options, result).Run(filterstrings, filter); + new Operation.ListControlFilesHandler(m_options, result).Run(backendManager, filterstrings, filter); }); } public Duplicati.Library.Interface.IListRemoteResults ListRemote() { - return RunAction(new ListRemoteResults(), (result) => + return RunAction(new ListRemoteResults(), (result, backendManager) => { using (var tf = System.IO.File.Exists(m_options.Dbpath) ? null : new Library.Utility.TempFile()) using (var db = new Database.LocalDatabase(((string)tf) ?? m_options.Dbpath, "list-remote", true)) - using (var bk = new BackendManager(m_backend, m_options, result.BackendWriter, null)) - result.SetResult(bk.List()); + result.SetResult(backendManager.ListAsync(CancellationToken.None).Await()); }); } public Duplicati.Library.Interface.IListRemoteResults DeleteAllRemoteFiles() { - return RunAction(new ListRemoteResults(), (result) => + return RunAction(new ListRemoteResults(), (result, backendManager) => { + var cancelToken = CancellationToken.None; result.OperationProgressUpdater.UpdatePhase(OperationPhase.Delete_Listing); - using (var bk = new BackendManager(m_backend, m_options, result.BackendWriter, null)) { // Only delete files that match the expected pattern and prefix - var list = bk.List() + var list = backendManager.ListAsync(cancelToken).Await() .Select(x => Volumes.VolumeBase.ParseFilename(x)) .Where(x => x != null) .Where(x => x.Prefix == m_options.Prefix) @@ -243,7 +243,7 @@ namespace Duplicati.Library.Main { try { - bk.Delete(list[i].File.Name, list[i].File.Size, true); + backendManager.DeleteAsync(list[i].File.Name, list[i].File.Size, true, cancelToken).Await(); } catch (Exception ex) { @@ -260,9 +260,9 @@ namespace Duplicati.Library.Main { CheckAutoVacuumInterval(); - return RunAction(new CompactResults(), (result) => + return RunAction(new CompactResults(), (result, backendManager) => { - new Operation.CompactHandler(m_backend, m_options, result).Run(); + new Operation.CompactHandler(m_options, result).Run(backendManager).Await(); }); } @@ -270,10 +270,10 @@ namespace Duplicati.Library.Main { var filelistfilter = Operation.RestoreHandler.FilterNumberedFilelist(m_options.Time, m_options.Version, singleTimeMatch: true); - return RunAction(new RecreateDatabaseResults(), ref filter, (result) => + return RunAction(new RecreateDatabaseResults(), ref filter, (result, backendManager) => { - using (var h = new Operation.RecreateDatabaseHandler(m_backend, m_options, result)) - h.RunUpdate(filter, filelistfilter); + using (var h = new Operation.RecreateDatabaseHandler(m_options, result)) + h.RunUpdate(backendManager, filter, filelistfilter, null); }); } @@ -281,7 +281,7 @@ namespace Duplicati.Library.Main { var t = new string[] { targetpath }; - return RunAction(new CreateLogDatabaseResults(), ref t, (result) => + return RunAction(new CreateLogDatabaseResults(), ref t, (result, backendManager) => { new Operation.CreateBugReportHandler(t[0], m_options, result).Run(); }); @@ -291,15 +291,15 @@ namespace Duplicati.Library.Main { var t = new string[] { baseVersion, targetVersion }; - return RunAction(new ListChangesResults(), ref t, ref filter, (result) => + return RunAction(new ListChangesResults(), ref t, ref filter, (result, backendManager) => { - new Operation.ListChangesHandler(m_backend, m_options, result).Run(t[0], t[1], filterstrings, filter, callback); + new Operation.ListChangesHandler(m_options, result).Run(t[0], t[1], backendManager, filterstrings, filter, callback); }); } public Duplicati.Library.Interface.IListAffectedResults ListAffected(List args, Action callback = null) { - return RunAction(new ListAffectedResults(), (result) => + return RunAction(new ListAffectedResults(), (result, backendManager) => { new Operation.ListAffected(m_options, result).Run(args, callback); }); @@ -310,9 +310,9 @@ namespace Duplicati.Library.Main if (!m_options.RawOptions.ContainsKey("full-remote-verification")) m_options.RawOptions["full-remote-verification"] = "true"; - return RunAction(new TestResults(), (result) => + return RunAction(new TestResults(), (result, backendManager) => { - new Operation.TestHandler(m_backend, m_options, result).Run(samples); + new Operation.TestHandler(m_options, result).Run(samples, backendManager); }); } @@ -325,7 +325,7 @@ namespace Duplicati.Library.Main var filtertag = Logging.Log.LogTagFromType(typeof(Operation.Backup.FileEnumerationProcess)); using (Logging.Log.StartScope(m_messageSink.WriteMessage, x => x.FilterTag.Contains(filtertag))) { - return RunAction(new TestFilterResults(), ref paths, ref filter, (result) => + return RunAction(new TestFilterResults(), ref paths, ref filter, (result, backendManager) => { new Operation.TestFilterHandler(m_options, result).RunAsync(ExpandInputSources(paths, filter), filter).Await(); }); @@ -334,7 +334,7 @@ namespace Duplicati.Library.Main public Library.Interface.ISystemInfoResults SystemInfo() { - return RunAction(new SystemInfoResults(), result => + return RunAction(new SystemInfoResults(), (result, backendManager) => { Operation.SystemInfoHandler.Run(result); }); @@ -342,25 +342,25 @@ namespace Duplicati.Library.Main public Library.Interface.IPurgeFilesResults PurgeFiles(Library.Utility.IFilter filter) { - return RunAction(new PurgeFilesResults(), result => + return RunAction(new PurgeFilesResults(), (result, backendManager) => { - new Operation.PurgeFilesHandler(m_backend, m_options, result).Run(filter); + new Operation.PurgeFilesHandler(m_options, result).Run(backendManager, filter); }); } public Library.Interface.IListBrokenFilesResults ListBrokenFiles(Library.Utility.IFilter filter, Func callbackhandler = null) { - return RunAction(new ListBrokenFilesResults(), result => + return RunAction(new ListBrokenFilesResults(), (result, backendManager) => { - new Operation.ListBrokenFilesHandler(m_backend, m_options, result).Run(filter, callbackhandler); + new Operation.ListBrokenFilesHandler(m_options, result).Run(backendManager, filter, callbackhandler); }); } public Library.Interface.IPurgeBrokenFilesResults PurgeBrokenFiles(Library.Utility.IFilter filter) { - return RunAction(new PurgeBrokenFilesResults(), result => + return RunAction(new PurgeBrokenFilesResults(), (result, backendManager) => { - new Operation.PurgeBrokenFilesHandler(m_backend, m_options, result).Run(filter); + new Operation.PurgeBrokenFilesHandler(m_options, result).Run(backendManager, filter); }); } @@ -386,7 +386,7 @@ namespace Duplicati.Library.Main var filtertag = Logging.Log.LogTagFromType(); using (Logging.Log.StartScope(m_messageSink.WriteMessage, x => x.FilterTag.Contains(filtertag))) { - return RunAction(new SendMailResults(), result => + return RunAction(new SendMailResults(), (result, backendManager) => { result.Lines = new string[0]; System.Threading.Thread.Sleep(5); @@ -396,36 +396,36 @@ namespace Duplicati.Library.Main public Library.Interface.IVacuumResults Vacuum() { - return RunAction(new VacuumResults(), result => + return RunAction(new VacuumResults(), (result, backendManager) => { new Operation.VacuumHandler(m_options, result).Run(); }); } - private T RunAction(T result, Action method) - where T : ISetCommonOptions, ITaskControlProvider, Logging.ILogDestination, IBasicResults + private T RunAction(T result, Action method) + where T : ISetCommonOptions, ITaskControlProvider, Logging.ILogDestination, IBasicResults, IBackendWriterProvider { var tmp = new string[0]; IFilter tempfilter = null; return RunAction(result, ref tmp, ref tempfilter, method); } - private T RunAction(T result, ref string[] paths, Action method) - where T : ISetCommonOptions, ITaskControlProvider, Logging.ILogDestination, IBasicResults + private T RunAction(T result, ref string[] paths, Action method) + where T : ISetCommonOptions, ITaskControlProvider, Logging.ILogDestination, IBasicResults, IBackendWriterProvider { IFilter tempfilter = null; return RunAction(result, ref paths, ref tempfilter, method); } - private T RunAction(T result, ref IFilter filter, Action method) - where T : ISetCommonOptions, ITaskControlProvider, Logging.ILogDestination, IBasicResults + private T RunAction(T result, ref IFilter filter, Action method) + where T : ISetCommonOptions, ITaskControlProvider, Logging.ILogDestination, IBasicResults, IBackendWriterProvider { var tmp = new string[0]; return RunAction(result, ref tmp, ref filter, method); } - private T RunAction(T result, ref string[] paths, ref IFilter filter, Action method) - where T : ISetCommonOptions, ITaskControlProvider, Logging.ILogDestination, IBasicResults + private T RunAction(T result, ref string[] paths, ref IFilter filter, Action method) + where T : ISetCommonOptions, ITaskControlProvider, Logging.ILogDestination, IBasicResults, IBackendWriterProvider { OnOperationStarted?.Invoke(result); var resultSetter = result as ISetCommonOptions; @@ -447,7 +447,25 @@ namespace Duplicati.Library.Main using (new Logging.Timer(LOGTAG, string.Format("Run{0}", result.MainOperation), string.Format("Running {0}", result.MainOperation))) using (new CoCoL.IsolatedChannelScope()) using (m_options.ConcurrencyMaxThreads <= 0 ? null : new CoCoL.CappedThreadedThreadPool(m_options.ConcurrencyMaxThreads)) - method(result); + using (var backend = new Backend.BackendManager(m_backendUrl, m_options, result.BackendWriter, result.TaskControl)) + { + method(result, backend); + + // TODO: Should also have a single shared database connection for all operations + // The transactions should be managed inside the connection, and not passed around + + // This would allow us to pass the database instance to the backend manager + // And safeguard against remote operations not being logged in the database + if (File.Exists(m_options.Dbpath)) + { + using (var db = new LocalDatabase(m_options.Dbpath, result.MainOperation.ToString(), true)) + backend.StopRunnerAndFlushMessages(db, null).Await(); + } + else + { + backend.StopRunnerAndDiscardMessages(); + } + } if (resultSetter.EndTime.Ticks == 0) resultSetter.EndTime = DateTime.UtcNow; @@ -621,7 +639,7 @@ namespace Duplicati.Library.Main // Store the URL connection options separately, as these should only be visible to modules implementing IConnectionModule var conopts = new Dictionary(m_options.RawOptions); - var qp = new Library.Utility.Uri(m_backend).QueryParameters; + var qp = new Library.Utility.Uri(m_backendUrl).QueryParameters; foreach (var k in qp.Keys) conopts[(string)k] = qp[(string)k]; @@ -657,7 +675,7 @@ namespace Duplicati.Library.Main } if (mx.Value is IGenericCallbackModule module) - module.OnStart(result.MainOperation.ToString(), ref m_backend, ref paths); + module.OnStart(result.MainOperation.ToString(), ref m_backendUrl, ref paths); } // If the filters were changed by a module, read them back in @@ -700,17 +718,17 @@ namespace Duplicati.Library.Main } if (string.IsNullOrEmpty(m_options.Dbpath)) - m_options.Dbpath = CLIDatabaseLocator.GetDatabasePathForCLI(m_backend, m_options); + m_options.Dbpath = CLIDatabaseLocator.GetDatabasePathForCLI(m_backendUrl, m_options); ValidateOptions(); } private async Task ApplySecretProvider(CancellationToken cancellationToken) { - var args = new[] { new Library.Utility.Uri(m_backend) }; + var args = new[] { new Library.Utility.Uri(m_backendUrl) }; await SecretProviderHelper.ApplySecretProviderAsync([], args, m_options.RawOptions, TempFolder.SystemTempPath, SecretProvider, cancellationToken); // Write back the backend argument, if it was modified by the secret provider - m_backend = args[0].ToString(); + m_backendUrl = args[0].ToString(); } /// @@ -794,14 +812,14 @@ namespace Duplicati.Library.Main // Throw url-encoded options into the mix //TODO: This can hide values if both commandline and url-parameters supply the same key - var ext = new Library.Utility.Uri(m_backend).QueryParameters; + var ext = new Library.Utility.Uri(m_backendUrl).QueryParameters; foreach (var k in ext.AllKeys) ropts[k] = ext[k]; //Now run through all supported options, and look for deprecated options foreach (var l in new IEnumerable[] { m_options.SupportedCommands, - DynamicLoader.BackendLoader.GetSupportedCommands(m_backend), + DynamicLoader.BackendLoader.GetSupportedCommands(m_backendUrl), m_options.NoEncryption ? null : DynamicLoader.EncryptionLoader.GetSupportedCommands(m_options.EncryptionModule), moduleOptions, DynamicLoader.CompressionLoader.GetSupportedCommands(m_options.CompressionModule) }) @@ -865,7 +883,7 @@ namespace Duplicati.Library.Main } //Inform the user about the deprecated Tardigrade-Backend. They should switch to Storj DCS instead. - if (string.Equals(new Library.Utility.Uri(m_backend).Scheme, "tardigrade", StringComparison.OrdinalIgnoreCase)) + if (string.Equals(new Library.Utility.Uri(m_backendUrl).Scheme, "tardigrade", StringComparison.OrdinalIgnoreCase)) Logging.Log.WriteWarningMessage(LOGTAG, "TardigradeRename", null, "The Tardigrade-backend got renamed to Storj DCS - please migrate your backups to the new configuration by changing the destination storage type to Storj DCS."); //TODO: Based on the action, see if all options are relevant diff --git a/Duplicati/Library/Main/Database/IRemoteVolume.cs b/Duplicati/Library/Main/Database/IRemoteVolume.cs index fffb11ace..b5047e2dc 100644 --- a/Duplicati/Library/Main/Database/IRemoteVolume.cs +++ b/Duplicati/Library/Main/Database/IRemoteVolume.cs @@ -18,17 +18,12 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; - -namespace Duplicati.Library.Main.Database -{ - public interface IRemoteVolume - { - string Name { get; } - string Hash { get; } - long Size { get; } - } -} +namespace Duplicati.Library.Main.Database +{ + public interface IRemoteVolume + { + string Name { get; } + string Hash { get; } + long Size { get; } + } +} diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 77529d345..18003cc06 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -1467,14 +1467,20 @@ ORDER BY { if (m_connection.State == System.Data.ConnectionState.Open && !m_hasExecutedVacuum) { - using (IDbTransaction transaction = m_connection.BeginTransaction()) + using (var transaction = m_connection.BeginTransaction()) + using (var command = m_connection.CreateCommand(transaction)) { - using (IDbCommand command = m_connection.CreateCommand(transaction)) + // SQLite recommends that PRAGMA optimize is run just before closing each database connection. + command.ExecuteNonQuery("PRAGMA optimize"); + + try { - // SQLite recommends that PRAGMA optimize is run just before closing each database connection. - command.ExecuteNonQuery("PRAGMA optimize"); transaction.Commit(); } + catch (System.Data.SQLite.SQLiteException ex) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "FailedToCommitTransaction", ex, "Failed to commit transaction after pragma optimize, usually caused by the a no-op transaction"); + } } m_connection.Close(); diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index 787a74eae..fde1e870f 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; @@ -198,7 +198,7 @@ namespace Duplicati.Library.Main.Database { m_fileprogtable = null; m_totalprogtable = null; - Logging.Log.WriteWarningMessage(LOGTAG, "ProgressTrackerSetupError", ex, "Failed to set up progress tracking tables"); + Logging.Log.WriteWarningMessage(LOGTAG, "ProgressTrackerSetupError", ex, "Failed to set up progress tracking tables"); throw; } } @@ -211,10 +211,10 @@ namespace Duplicati.Library.Main.Database m_tempfiletable = "Fileset-" + m_temptabsetguid; m_tempblocktable = "Blocks-" + m_temptabsetguid; - using(var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand()) { var filesetIds = GetFilesetIDs(Library.Utility.Utility.NormalizeDateTime(restoretime), versions).ToList(); - while(filesetIds.Count > 0) + while (filesetIds.Count > 0) { var filesetId = filesetIds[0]; filesetIds.RemoveAt(0); @@ -249,7 +249,7 @@ namespace Duplicati.Library.Main.Database // If we get a list of filenames, the lookup table is faster // unfortunately we cannot do this if the filesystem is case sensitive as // SQLite only supports ASCII compares - using(var tr = m_connection.BeginTransaction()) + using (var tr = m_connection.BeginTransaction()) { var p = expression.GetSimpleList(); var m_filenamestable = "Filenames-" + m_temptabsetguid; @@ -258,7 +258,7 @@ namespace Duplicati.Library.Main.Database cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"") VALUES (?)", m_filenamestable); cmd.AddParameter(); - foreach(var s in p) + foreach (var s in p) { cmd.SetParameterValue(0, s); cmd.ExecuteNonQuery(); @@ -275,7 +275,7 @@ namespace Duplicati.Library.Main.Database var sb = new StringBuilder(); sb.AppendLine(); - using(var rd = cmd.ExecuteReader(string.Format(@"SELECT ""Path"" FROM ""{0}"" WHERE ""Path"" NOT IN (SELECT ""Path"" FROM ""{1}"")", m_filenamestable, m_tempfiletable))) + using (var rd = cmd.ExecuteReader(string.Format(@"SELECT ""Path"" FROM ""{0}"" WHERE ""Path"" NOT IN (SELECT ""Path"" FROM ""{1}"")", m_filenamestable, m_tempfiletable))) while (rd.Read()) sb.AppendLine(rd.GetValue(0).ToString()); @@ -286,7 +286,7 @@ namespace Duplicati.Library.Main.Database cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", m_filenamestable)); - using(new Logging.Timer(LOGTAG, "CommitPrepareFileset", "CommitPrepareFileset")) + using (new Logging.Timer(LOGTAG, "CommitPrepareFileset", "CommitPrepareFileset")) tr.Commit(); } } @@ -298,14 +298,14 @@ namespace Duplicati.Library.Main.Database cmd.AddParameter(filesetId); object[] values = new object[3]; - using(var cmd2 = m_connection.CreateCommand()) + using (var cmd2 = m_connection.CreateCommand()) { cmd2.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") VALUES (?,?,?,0)", m_tempfiletable); cmd2.AddParameter(); cmd2.AddParameter(); cmd2.AddParameter(); - using(var rd = cmd.ExecuteReader()) + using (var rd = cmd.ExecuteReader()) while (rd.Read()) { rd.GetValues(values); @@ -407,7 +407,7 @@ namespace Duplicati.Library.Main.Database { var dirsep = Util.GuessDirSeparator(string.IsNullOrWhiteSpace(largest_prefix) ? GetFirstPath() : largest_prefix); - using(var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand()) { if (string.IsNullOrEmpty(destination)) { @@ -474,7 +474,7 @@ namespace Duplicati.Library.Main.Database public void FindMissingBlocks(bool skipMetadata) { - using(var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand()) { cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""FileID"", ""Index"", ""Hash"", ""Size"", ""Restored"", ""Metadata"", ""VolumeId"", ""BlockId"") SELECT DISTINCT ""{1}"".""ID"", ""BlocksetEntry"".""Index"", ""Block"".""Hash"", ""Block"".""Size"", 0, 0, ""Block"".""VolumeID"", ""Block"".""ID"" FROM ""{1}"", ""BlocksetEntry"", ""Block"" WHERE ""{1}"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" ", m_tempblocktable, m_tempfiletable); var p1 = cmd.ExecuteNonQuery(); @@ -576,7 +576,7 @@ namespace Duplicati.Library.Main.Database public string TargetPath { get { return m_reader.ConvertValueToString(0); } } public string TargetHash { get { return m_reader.ConvertValueToString(1); } } - public long TargetFileID { get { return m_reader.ConvertValueToInt64(2); } } + public long TargetFileID { get { return m_reader.ConvertValueToInt64(2); } } public long Length { get { return m_reader.ConvertValueToInt64(3); } } public bool HasMore { get; private set; } @@ -607,14 +607,14 @@ namespace Duplicati.Library.Main.Database public static IEnumerable GetExistingFilesWithBlocks(System.Data.IDbConnection connection, string tablename) { - using(var cmd = connection.CreateCommand()) + using (var cmd = connection.CreateCommand()) { cmd.CommandText = string.Format(@"SELECT ""{0}"".""TargetPath"", ""Blockset"".""FullHash"", ""{0}"".""ID"", ""Blockset"".""Length"", ""Block"".""Hash"", ""BlocksetEntry"".""Index"", ""Block"".""Size"" FROM ""{0}"", ""Blockset"", ""BlocksetEntry"", ""Block"" WHERE ""{0}"".""BlocksetID"" = ""Blockset"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""{0}"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" ORDER BY ""{0}"".""TargetPath"", ""BlocksetEntry"".""Index""", tablename); - using(var rd = cmd.ExecuteReader()) + using (var rd = cmd.ExecuteReader()) if (rd.Read()) { var more = true; - while(more) + while (more) { var f = new ExistingFile(rd); string current = f.TargetPath; @@ -712,12 +712,12 @@ namespace Duplicati.Library.Main.Database { // TODO: Skip metadata as required cmd.CommandText = string.Format(@"SELECT DISTINCT ""A"".""TargetPath"", ""A"".""ID"", ""B"".""Hash"", (""B"".""Index"" * {2}), ""B"".""Index"", ""B"".""Size"", ""C"".""Path"", (""D"".""Index"" * {2}), ""E"".""Size"", ""B"".""Metadata"" FROM ""{0}"" ""A"", ""{1}"" ""B"", ""File"" ""C"", ""BlocksetEntry"" ""D"", ""Block"" E WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""C"".""BlocksetID"" = ""D"".""BlocksetID"" AND ""D"".""BlockID"" = ""E"".""ID"" AND ""B"".""Hash"" = ""E"".""Hash"" AND ""B"".""Size"" = ""E"".""Size"" AND ""B"".""Restored"" = 0", filetablename, blocktablename, blocksize); - using(var rd = cmd.ExecuteReader()) + using (var rd = cmd.ExecuteReader()) { if (rd.Read()) { var more = true; - while(more) + while (more) { var f = new LocalBlockSource(rd); string current = f.TargetPath; @@ -869,7 +869,7 @@ namespace Duplicati.Library.Main.Database { get { - using(var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand()) { // The IN-clause with subquery enables SQLite to use indexes better. Three way join (A,B,C) is slow here! cmd.CommandText = string.Format( @@ -879,12 +879,12 @@ namespace Duplicati.Library.Main.Database + @" AND ""BB"".""ID"" IN (SELECT ""B"".""ID"" FROM ""{1}"" ""B"", ""{2}"" ""C"" WHERE ""B"".""Hash"" = ""C"".""Hash"" AND ""B"".""Size"" = ""C"".""Size"") " + @" ORDER BY ""A"".""TargetPath"", ""BB"".""Index""" , m_filetablename, m_blocktablename, m_tmptable, m_blocksize, "0"); - using(var rd = cmd.ExecuteReader()) + using (var rd = cmd.ExecuteReader()) { if (rd.Read()) { var more = true; - while(more) + while (more) { var f = new VolumePatch(rd); string current = f.Path; @@ -904,7 +904,7 @@ namespace Duplicati.Library.Main.Database { get { - using(var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand()) { // The IN-clause with subquery enables SQLite to use indexes better. Three way join (A,B,C) is slow here! cmd.CommandText = string.Format( @@ -914,7 +914,7 @@ namespace Duplicati.Library.Main.Database + @" AND ""BB"".""ID"" IN (SELECT ""B"".""ID"" FROM ""{1}"" ""B"", ""{2}"" ""C"" WHERE ""B"".""Hash"" = ""C"".""Hash"" AND ""B"".""Size"" = ""C"".""Size"") " + @" ORDER BY ""A"".""TargetPath"", ""BB"".""Index""" , m_filetablename, m_blocktablename, m_tmptable, m_blocksize, "1"); - using(var rd = cmd.ExecuteReader()) + using (var rd = cmd.ExecuteReader()) { if (rd.Read()) { @@ -1029,7 +1029,7 @@ namespace Duplicati.Library.Main.Database using var reader = cmd.ExecuteReader(); for (long i = 0; reader.Read(); i++) { - yield return new BlockRequest(reader.GetInt64(0), i, reader.GetString(1), reader.GetInt64(2), reader.GetInt64(3), false); + yield return new BlockRequest(reader.ConvertValueToInt64(0), i, reader.ConvertValueToString(1), reader.ConvertValueToInt64(2), reader.ConvertValueToInt64(3), false); } } @@ -1054,7 +1054,7 @@ namespace Duplicati.Library.Main.Database using var reader = cmd.ExecuteReader(); for (long i = 0; reader.Read(); i++) { - yield return new BlockRequest(reader.GetInt64(0), i, reader.GetString(1), reader.GetInt64(2), reader.GetInt64(3), false); + yield return new BlockRequest(reader.ConvertValueToInt64(0), i, reader.ConvertValueToString(1), reader.ConvertValueToInt64(2), reader.ConvertValueToInt64(3), false); } } @@ -1063,7 +1063,7 @@ namespace Duplicati.Library.Main.Database /// /// The ID of the volume. /// A tuple containing the name, size, and hash of the volume. - public IEnumerable<(string,long,string)> GetVolumeInfo(long VolumeID) + public IEnumerable<(string, long, string)> GetVolumeInfo(long VolumeID) { using var cmd = m_connection.CreateCommand(); cmd.CommandText = "SELECT Name, Size, Hash FROM RemoteVolume WHERE ID = ?"; @@ -1071,7 +1071,7 @@ namespace Duplicati.Library.Main.Database cmd.SetParameterValue(0, VolumeID); using var reader = cmd.ExecuteReader(); while (reader.Read()) - yield return (reader.GetString(0), reader.GetInt64(1), reader.GetString(2)); + yield return (reader.ConvertValueToString(0), reader.ConvertValueToInt64(1), reader.ConvertValueToString(2)); } public void DropRestoreTable() @@ -1084,7 +1084,7 @@ namespace Duplicati.Library.Main.Database cmd.CommandText = string.Format(@"DROP TABLE IF EXISTS ""{0}""", m_tempfiletable); cmd.ExecuteNonQuery(); } - catch(Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } + catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } finally { m_tempfiletable = null; } if (m_tempblocktable != null) @@ -1328,8 +1328,8 @@ namespace Duplicati.Library.Main.Database { using (var cmd = m_connection.CreateCommand()) using (var rd = cmd.ExecuteReader(string.Format(@"SELECT ""TargetPath"" FROM ""{0}"" WHERE ""BlocksetID"" == ?", m_tempfiletable), FOLDER_BLOCKSET_ID)) - while(rd.Read()) - yield return rd.GetValue(0).ToString(); + while (rd.Read()) + yield return rd.ConvertValueToString(0); } public interface IFastSource @@ -1355,19 +1355,19 @@ namespace Duplicati.Library.Main.Database private readonly System.Data.IDataReader m_rd; private readonly long m_blocksize; public BlockEntry(System.Data.IDataReader rd, long blocksize) { m_rd = rd; m_blocksize = blocksize; } - public long Offset { get { return m_rd.GetInt64(3) * m_blocksize; } } - public long Index { get { return m_rd.GetInt64(3); } } - public long Size { get { return m_rd.GetInt64(5); } } - public string Hash { get { return m_rd.GetString(4); } } + public long Offset { get { return m_rd.ConvertValueToInt64(3) * m_blocksize; } } + public long Index { get { return m_rd.ConvertValueToInt64(3); } } + public long Size { get { return m_rd.ConvertValueToInt64(5); } } + public string Hash { get { return m_rd.ConvertValueToString(4); } } } private readonly System.Data.IDataReader m_rd; private readonly long m_blocksize; public FastSource(System.Data.IDataReader rd, long blocksize) { m_rd = rd; m_blocksize = blocksize; MoreData = true; } public bool MoreData { get; private set; } - public string TargetPath { get { return m_rd.GetValue(0).ToString(); } } - public long TargetFileID { get { return m_rd.GetInt64(2); } } - public string SourcePath { get { return m_rd.GetValue(1).ToString(); } } + public string TargetPath { get { return m_rd.ConvertValueToString(0); } } + public long TargetFileID { get { return m_rd.ConvertValueToInt64(2); } } + public string SourcePath { get { return m_rd.ConvertValueToString(1); } } public IEnumerable Blocks { @@ -1378,7 +1378,7 @@ namespace Duplicati.Library.Main.Database do { yield return new BlockEntry(m_rd, m_blocksize); - } while((MoreData = m_rd.Read()) && tid == this.TargetFileID); + } while ((MoreData = m_rd.Read()) && tid == this.TargetFileID); } } @@ -1483,7 +1483,7 @@ namespace Duplicati.Library.Main.Database @" ORDER BY ""{0}"".""ID"", ""{1}"".""Index"" ", m_tempfiletable, m_tempblocktable); using (var cmd = m_connection.CreateCommand()) - using(var rd = cmd.ExecuteReader(sources)) + using (var rd = cmd.ExecuteReader(sources)) { if (rd.Read()) { @@ -1495,7 +1495,7 @@ namespace Duplicati.Library.Main.Database yield return n; more = n.MoreData; - while(more && n.TargetFileID == tid) + while (more && n.TargetFileID == tid) more = rd.Read(); } while (more); diff --git a/Duplicati/Library/Main/Duplicati.Library.Main.csproj b/Duplicati/Library/Main/Duplicati.Library.Main.csproj index 75cd77830..dd7f67065 100644 --- a/Duplicati/Library/Main/Duplicati.Library.Main.csproj +++ b/Duplicati/Library/Main/Duplicati.Library.Main.csproj @@ -4,6 +4,7 @@ net8.0 Library Copyright © 2025 Team Duplicati, MIT license + Duplicati.Library.Main diff --git a/Duplicati/Library/Main/Enums.cs b/Duplicati/Library/Main/Enums.cs index e535d3829..3486c098e 100644 --- a/Duplicati/Library/Main/Enums.cs +++ b/Duplicati/Library/Main/Enums.cs @@ -1,27 +1,23 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. namespace Duplicati.Library.Main { @@ -32,9 +28,11 @@ namespace Duplicati.Library.Main Get, Put, Delete, - CreateFolder + CreateFolder, + QuotaInfo, + WaitForEmpty } - + public enum BackendEventType { Started, @@ -150,5 +148,5 @@ namespace Duplicati.Library.Main /// The entry is a symbolic link /// Symlink - } + } } diff --git a/Duplicati/Library/Main/IBackendManager.cs b/Duplicati/Library/Main/IBackendManager.cs new file mode 100644 index 000000000..270f6c946 --- /dev/null +++ b/Duplicati/Library/Main/IBackendManager.cs @@ -0,0 +1,112 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.Threading; +using System.Threading.Tasks; +using Duplicati.Library.Interface; +using Duplicati.Library.Main.Database; +using Duplicati.Library.Main.Volumes; +using Duplicati.Library.Utility; +using IFileEntry = Duplicati.Library.Interface.IFileEntry; + +#nullable enable + +namespace Duplicati.Library.Main; + +/// +/// Interface for the backend manager +/// +internal interface IBackendManager : IDisposable +{ + /// + /// Uploads a block volume to the backend, including an optional index volume + /// + /// The block volume to upload + /// The index volume to upload, if any + /// The action to call when the index volume is finished + /// Whether to wait for the upload to complete + /// The cancellation token + /// An awaitable task + Task PutAsync(VolumeWriterBase blockVolume, IndexVolumeWriter? indexVolume, Action? indexVolumeFinished, bool waitForComplete, CancellationToken cancelToken); + + /// + /// Uploads a file to the backend without encryption + /// + /// The name of the file to upload to + /// The file to upload + /// The cancellation token + /// An awaitable task + Task PutVerificationFileAsync(string remotename, TempFile tempFile, CancellationToken cancelToken); + + /// + /// Waits for the backend queue to be empty and flushes any pending messages to the database + /// + /// The database to write pending messages to + /// The transaction to use, if any + /// The cancellation token + /// An awaitable task + Task WaitForEmptyAsync(LocalDatabase database, IDbTransaction? transaction, CancellationToken cancellationToken); + + /// + /// Lists the files on the backend + /// + /// The cancellation token + /// An enumerable of file entries + Task> ListAsync(CancellationToken cancelToken); + + /// + /// Deletes a file on the backend + /// + /// The name of the file to delete + /// The size of the file to delete, or -1 if not known + /// Whether to wait for the delete to complete + /// The cancellation token + /// An awaitable task + Task DeleteAsync(string remotename, long size, bool waitForComplete, CancellationToken cancelToken); + + /// + /// Gets the quota information for the backend + /// + /// The cancellation token + /// The quota information, or null if not available or disabled + Task GetQuotaInfoAsync(CancellationToken cancelToken); + + /// + /// Gets a file with the hash and size + /// + /// The name of the file to get + /// The hash of the file to get, or null if not known + /// The size of the file to get, or -1 if not known + /// The cancellation token + /// The file, hash, and size + Task<(TempFile File, string Hash, long Size)> GetWithInfoAsync(string remotename, string hash, long size, CancellationToken cancelToken); + + /// + /// Gets a file from the backend + /// + /// The name of the file to get + /// The hash of the file to get, or null if not known + /// The size of the file to get, or -1 if not known + /// The cancellation token + /// + Task GetAsync(string remotename, string hash, long size, CancellationToken cancelToken); + + /// + /// Performs a download of the files specified, with pre-fetch to overlap the download and processing + /// + /// The volumes to download + /// The cancellation token + /// The downloaded files, hash, size, and name + IAsyncEnumerable<(TempFile File, string Hash, long Size, string Name)> GetFilesOverlappedAsync(IEnumerable volumes, CancellationToken cancelToken); + + /// + /// Gets the size of the last read operation + /// + long LastReadSize { get; } + + /// + /// Gets the size of the last write operation + /// + long LastWriteSize { get; } + +} diff --git a/Duplicati/Library/Main/Operation/Backup/BackendUploader.cs b/Duplicati/Library/Main/Operation/Backup/BackendUploader.cs deleted file mode 100644 index 4bd43812f..000000000 --- a/Duplicati/Library/Main/Operation/Backup/BackendUploader.cs +++ /dev/null @@ -1,551 +0,0 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - -using CoCoL; -using Duplicati.Library.Interface; -using Duplicati.Library.Main.Operation.Common; -using Duplicati.Library.Main.Volumes; -using Duplicati.Library.Utility; -using Newtonsoft.Json; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Runtime.ExceptionServices; -using System.Threading; -using System.Threading.Tasks; -using static Duplicati.Library.Main.Operation.Common.BackendHandler; - -namespace Duplicati.Library.Main.Operation.Backup -{ - internal interface IUploadRequest - { - } - - internal class FlushRequest : IUploadRequest - { - public Task LastWriteSizeAsync { get { return m_tcs.Task; } } - private readonly TaskCompletionSource m_tcs = new TaskCompletionSource(); - - public void SetFlushed(long size) - { - m_tcs.TrySetResult(size); - } - - public void TrySetCanceled() - { - m_tcs.TrySetCanceled(); - } - } - - internal class IndexVolumeUploadRequest : IUploadRequest - { - public IndexVolumeWriter IndexVolume { get; private set; } - - public IndexVolumeUploadRequest(IndexVolumeWriter indexVolume) - { - IndexVolume = indexVolume; - } - } - - internal class FilesetUploadRequest : IUploadRequest - { - public FilesetVolumeWriter Fileset { get; private set; } - - public FilesetUploadRequest(FilesetVolumeWriter fileset) - { - Fileset = fileset; - } - } - - internal class VolumeUploadRequest : IUploadRequest - { - public FileEntryItem BlockEntry { get; } - public BlockVolumeWriter BlockVolume { get; } - public TemporaryIndexVolume IndexVolume { get; } - public Options Options { get; } - public BackupDatabase Database { get; } - - public VolumeUploadRequest(BlockVolumeWriter blockVolume, FileEntryItem blockEntry, TemporaryIndexVolume indexVolume, Options options, BackupDatabase database) - { - BlockVolume = blockVolume; - BlockEntry = blockEntry; - IndexVolume = indexVolume; - Options = options; - Database = database; - } - } - - /// - /// This class encapsulates all requests to the backend - /// and ensures that the AsynchronousUploadLimit is honored - /// - internal class BackendUploader - { - private static readonly string LOGTAG = Logging.Log.LogTagFromType(); - private readonly Func m_backendFactory; - private readonly IBackend m_backend; - private int m_backendTaken = 0; - private bool m_backendDisposed = false; - private CancellationTokenSource m_cancelTokenSource; - private readonly DatabaseCommon m_database; - private long m_initialUploadThrottleSpeed; - private int m_lastThrottleCheckTime; - private long m_lastUploadThrottleSpeed; - private int m_maxConcurrentUploads; - private readonly Options m_options; - private readonly FileProgressThrottler m_progressUpdater; - private readonly StatsCollector m_stats; - private readonly ITaskReader m_taskReader; - - public IBackend Backend { get { return m_backendDisposed ? null : m_backend; } } - - public BackendUploader(IBackend backend, Func backendFactory, Options options, DatabaseCommon database, ITaskReader taskReader, StatsCollector stats) - { - m_backend = backend; - m_backendFactory = () => - { - if (Interlocked.Exchange(ref m_backendTaken, 1) == 0) - return m_backend; - - return backendFactory(); - }; - m_options = options; - m_taskReader = taskReader; - m_stats = stats; - m_database = database; - m_progressUpdater = new FileProgressThrottler(stats, options.MaxUploadPrSecond); - } - - public Task Run(Channels channels) - { - return AutomationExtensions.RunTask(new - { - Input = channels.BackendRequest.AsRead() - }, - - async self => - { - var workers = new List(); - m_maxConcurrentUploads = m_options.AsynchronousConcurrentUploadLimit <= 0 ? int.MaxValue : m_options.AsynchronousConcurrentUploadLimit; - m_initialUploadThrottleSpeed = m_options.AsynchronousConcurrentUploadLimit <= 0 ? int.MaxValue : m_options.MaxUploadPrSecond / m_maxConcurrentUploads; - var lastSize = -1L; - var uploadsInProgress = 0; - m_cancelTokenSource = new CancellationTokenSource(); - m_progressUpdater.Run(m_cancelTokenSource.Token); - var filesetsToUpload = new List(); - - try - { - while (true) - { - var req = await self.Input.ReadAsync(); - - // Listen to pause/resume and abort, but ignore stop requests - await m_taskReader.ProgressRendevouz().ConfigureAwait(false); - - Worker worker = GetWorker(workers); - - if (req is VolumeUploadRequest volumeUpload) - { - if (volumeUpload.IndexVolume == null) - worker.Task = Task.Run(() => UploadFileAsync(volumeUpload.BlockEntry, worker, m_cancelTokenSource.Token)); - else - worker.Task = Task.Run(() => UploadBlockAndIndexAsync(volumeUpload, worker, m_cancelTokenSource.Token)); - - lastSize = volumeUpload.BlockVolume.SourceSize; - uploadsInProgress++; - } - else if (req is FilesetUploadRequest filesetUpload) - { - filesetsToUpload.Add(filesetUpload); - } - else if (req is IndexVolumeUploadRequest indexUpload) - { - worker.Task = Task.Run(() => UploadVolumeWriter(indexUpload.IndexVolume, worker, m_cancelTokenSource.Token)); - uploadsInProgress++; - } - else if (req is FlushRequest flush) - { - await WaitForWorkers(workers, flush).ConfigureAwait(false); - UploadFilesets(workers, filesetsToUpload); - await WaitForWorkers(workers, flush).ConfigureAwait(false); - - flush.SetFlushed(lastSize); - uploadsInProgress = 0; - break; - } - - if (uploadsInProgress >= m_maxConcurrentUploads) - { - await Task.WhenAny(workers.Select(w => w.Task)).ConfigureAwait(false); - uploadsInProgress--; - - var failedUploads = workers.Where(w => w.Task.IsFaulted).Select(w => GetInnerMostException(w.Task.Exception)).ToList(); - if (failedUploads.Any()) - { - if (failedUploads.Count == 1) - ExceptionDispatchInfo.Capture(failedUploads.First()).Throw(); - else - throw new AggregateException(failedUploads); - } - } - } - } - catch (Exception ex) when (!ex.IsRetiredException()) - { - m_cancelTokenSource.Cancel(); - try - { - await Task.WhenAll(workers.Select(w => w.Task)); - } - catch - { - // As we are cancelling all threads we do not need to alert the user to any of these exceptions. - } - finally - { - workers.ForEach(w => - { - if (w.Backend == m_backend) - m_backendDisposed = true; - w.Dispose(); - }); - } - throw; - } - - try - { - m_stats.SetBlocking(true); - await Task.WhenAll(workers.Select(w => w.Task)); - } - finally - { - m_stats.SetBlocking(false); - workers.ForEach(w => - { - if (w.Backend == m_backend) - { - Interlocked.Exchange(ref m_backendTaken, 0); - w.Backend = null; - } - w.Dispose(); - }); - } - }); - } - - private Worker GetWorker(List workers) - { - var worker = workers.FirstOrDefault(w => w.Task.IsCompleted && !w.Task.IsFaulted); - if (worker == null) - { - worker = new Worker(m_backendFactory()); - workers.Add(worker); - } - - return worker; - } - - private async Task WaitForWorkers(List workers, FlushRequest flush) - { - while (workers.Any()) - { - Task finishedTask = await Task.WhenAny(workers.Select(w => w.Task)).ConfigureAwait(false); - if (finishedTask.IsFaulted) - { - flush.TrySetCanceled(); - ExceptionDispatchInfo.Capture(finishedTask.Exception).Throw(); - } - - Worker finishedWorker = workers.Single(w => w.Task == finishedTask); - workers.Remove(finishedWorker); - if (finishedWorker.Backend == m_backend && !m_backendDisposed) - { - finishedWorker.Backend = null; - Interlocked.Exchange(ref m_backendTaken, 0); - } - - finishedWorker.Dispose(); - } - } - - private void UploadFilesets(List workers, List filesets) - { - foreach (var fileset in filesets) - { - Worker worker = GetWorker(workers); - worker.Task = Task.Run(() => UploadVolumeWriter(fileset.Fileset, worker, m_cancelTokenSource.Token)); - } - - filesets.Clear(); - } - - private static Exception GetInnerMostException(Exception ex) - { - while (ex.InnerException != null) - ex = ex.InnerException; - return ex; - } - - private async Task UploadBlockAndIndexAsync(VolumeUploadRequest upload, Worker worker, CancellationToken cancelToken) - { - if (await UploadFileAsync(upload.BlockEntry, worker, cancelToken).ConfigureAwait(false)) - { - // We must use the BlockEntry's RemoteFilename and not the BlockVolume's since the - // BlockEntry's' RemoteFilename reflects renamed files due to retries after errors. - IndexVolumeWriter indexVolumeWriter = await upload.IndexVolume.CreateVolume(upload.BlockEntry.RemoteFilename, upload.BlockEntry.Hash, upload.BlockEntry.Size, upload.Options, upload.Database); - // Create link before upload is started, it will be removed later if upload fails - await m_database.AddIndexBlockLinkAsync(indexVolumeWriter.VolumeID, upload.BlockVolume.VolumeID).ConfigureAwait(false); - - FileEntryItem indexEntry = indexVolumeWriter.CreateFileEntryForUpload(upload.Options); - await UploadFileAsync(indexEntry, worker, cancelToken).ConfigureAwait(false); - } - } - - private async Task UploadVolumeWriter(VolumeWriterBase volumeWriter, Worker worker, CancellationToken cancelToken) - { - var fileEntry = new FileEntryItem(BackendActionType.Put, volumeWriter.RemoteFilename); - fileEntry.SetLocalfilename(volumeWriter.LocalFilename); - fileEntry.Encrypt(m_options); - fileEntry.UpdateHashAndSize(m_options); - - await UploadFileAsync(fileEntry, worker, cancelToken).ConfigureAwait(false); - } - - private async Task UploadFileAsync(FileEntryItem item, Worker worker, CancellationToken cancelToken) - { - if (cancelToken.IsCancellationRequested) - return false; - - return await DoWithRetry(async () => - { - if (item.IsRetry) - await RenameFileAfterErrorAsync(item).ConfigureAwait(false); - await DoPut(item, worker.Backend, cancelToken).ConfigureAwait(false); - }, - item, worker, cancelToken).ConfigureAwait(false); - } - - private async Task DoWithRetry(Func method, FileEntryItem item, Worker worker, CancellationToken cancelToken) - { - item.IsRetry = false; - var retryCount = 0; - - for (retryCount = 0; retryCount <= m_options.NumberOfRetries; retryCount++) - { - if (m_options.RetryDelay.Ticks != 0 && retryCount != 0) - { - var delay = Library.Utility.Utility.GetRetryDelay(m_options.RetryDelay, retryCount, m_options.RetryWithExponentialBackoff); - await Task.Delay(delay).ConfigureAwait(false); - } - - if (cancelToken.IsCancellationRequested) - return false; - - try - { - if (worker.Backend == null) - worker.Backend = m_backendFactory(); - await method().ConfigureAwait(false); - return true; - } - catch (Exception ex) - { - item.IsRetry = true; - Logging.Log.WriteRetryMessage(LOGTAG, $"Retry{item.Operation}", ex, "Operation {0} with file {1} attempt {2} of {3} failed with message: {4}", item.Operation, item.RemoteFilename, retryCount + 1, m_options.NumberOfRetries + 1, ex.Message); - if (ex is ThreadAbortException || ex is OperationCanceledException) - break; - - await m_stats.SendEventAsync(item.Operation, retryCount < m_options.NumberOfRetries ? BackendEventType.Retrying : BackendEventType.Failed, item.RemoteFilename, item.Size); - - bool recovered = false; - if (m_options.AutocreateFolders && ex is FolderMissingException) - { - try - { - // If we successfully create the folder, we can re-use the connection - await worker.Backend.CreateFolderAsync(cancelToken).ConfigureAwait(false); - recovered = true; - } - catch (Exception dex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "FolderCreateError", dex, "Failed to create folder: {0}", ex.Message); - } - } - - if (!recovered) - ResetBackend(ex, worker); - - if (retryCount == m_options.NumberOfRetries) - throw; - } - finally - { - if (m_options.NoConnectionReuse) - ResetBackend(null, worker); - } - } - - return false; - } - - private void ResetBackend(Exception ex, Worker worker) - { - try - { - if (worker.Backend == m_backend) - m_backendDisposed = true; - worker.Backend?.Dispose(); - } - catch (Exception dex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "BackendDisposeError", dex, "Failed to dispose backend instance: {0}", ex?.Message); - } - finally - { - worker.Backend = null; - } - } - - private async Task RenameFileAfterErrorAsync(FileEntryItem item) - { - var p = VolumeBase.ParseFilename(item.RemoteFilename); - var guid = VolumeWriterBase.GenerateGuid(); - var time = p.Time.Ticks == 0 ? p.Time : p.Time.AddSeconds(1); - var newname = VolumeBase.GenerateFilename(p.FileType, p.Prefix, guid, time, p.CompressionModule, p.EncryptionModule); - var oldname = item.RemoteFilename; - - await m_stats.SendEventAsync(item.Operation, BackendEventType.Rename, oldname, item.Size); - await m_stats.SendEventAsync(item.Operation, BackendEventType.Rename, newname, item.Size); - Logging.Log.WriteInformationMessage(LOGTAG, "RenameRemoteTargetFile", "Renaming \"{0}\" to \"{1}\"", oldname, newname); - await m_database.RenameRemoteFileAsync(oldname, newname); - item.RemoteFilename = newname; - } - - private async Task DoPut(FileEntryItem item, IBackend backend, CancellationToken cancelToken) - { - if (cancelToken.IsCancellationRequested) - return; - - if (item.TrackedInDb) - await m_database.UpdateRemoteVolumeAsync(item.RemoteFilename, RemoteVolumeState.Uploading, item.Size, item.Hash); - - if (m_options.Dryrun) - { - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldUploadVolume", "Would upload volume: {0}, size: {1}", item.RemoteFilename, Library.Utility.Utility.FormatSizeString(new FileInfo(item.LocalFilename).Length)); - item.DeleteLocalFile(); - return; - } - - await m_database.LogRemoteOperationAsync("put", item.RemoteFilename, JsonConvert.SerializeObject(new { Size = item.Size, Hash = item.Hash })); - await m_stats.SendEventAsync(BackendActionType.Put, BackendEventType.Started, item.RemoteFilename, item.Size, updateProgress: false); - m_progressUpdater.StartFileProgress(item.RemoteFilename, item.Size); - - var begin = DateTime.Now; - - if (!m_options.DisableStreamingTransfers && backend is IStreamingBackend streamingBackend) - { - // A download throttle speed is not given to the ThrottledStream as we are only uploading data here - using (var fs = File.OpenRead(item.LocalFilename)) - using (var act = new StreamUtil.TimeoutObservingStream(fs) { ReadTimeout = backend is ITimeoutExemptBackend ? System.Threading.Timeout.Infinite : m_options.ReadWriteTimeout }) - using (var ts = new ThrottledStream(act, m_initialUploadThrottleSpeed, 0)) - using (var pgs = new ProgressReportingStream(ts, pg => HandleProgress(ts, pg, item.RemoteFilename))) - using (var linkedToken = CancellationTokenSource.CreateLinkedTokenSource(m_taskReader.TransferToken, act.TimeoutToken)) - await streamingBackend.PutAsync(item.RemoteFilename, pgs, linkedToken.Token).ConfigureAwait(false); - } - else - await backend.PutAsync(item.RemoteFilename, item.LocalFilename, cancelToken).ConfigureAwait(false); - - var duration = DateTime.Now - begin; - m_progressUpdater.EndFileProgress(item.RemoteFilename); - Logging.Log.WriteProfilingMessage(LOGTAG, "UploadSpeed", "Uploaded {0} in {1}, {2}/s", Library.Utility.Utility.FormatSizeString(item.Size), duration, Library.Utility.Utility.FormatSizeString((long)(item.Size / duration.TotalSeconds))); - - if (item.TrackedInDb) - await m_database.UpdateRemoteVolumeAsync(item.RemoteFilename, RemoteVolumeState.Uploaded, item.Size, item.Hash); - - await m_stats.SendEventAsync(BackendActionType.Put, BackendEventType.Completed, item.RemoteFilename, item.Size); - - if (m_options.ListVerifyUploads) - { - var f = backend.List().FirstOrDefault(n => n.Name.Equals(item.RemoteFilename, StringComparison.OrdinalIgnoreCase)); - if (f == null) - throw new Exception(string.Format("List verify failed, file was not found after upload: {0}", item.RemoteFilename)); - else if (f.Size != item.Size && f.Size >= 0) - throw new Exception(string.Format("List verify failed for file: {0}, size was {1} but expected to be {2}", f.Name, f.Size, item.Size)); - } - - item.DeleteLocalFile(); - await m_database.CommitTransactionAsync("CommitAfterUpload"); - } - - private void HandleProgress(ThrottledStream stream, long progress, string path) - { - UpdateThrottleSpeed(); - m_progressUpdater.UpdateFileProgress(path, progress, stream); - } - - private void UpdateThrottleSpeed() - { - var updateThrottleSpeed = false; - - lock (m_progressUpdater) - { - var currentTick = Environment.TickCount; - if (currentTick - m_lastThrottleCheckTime > 2000) - { - m_lastThrottleCheckTime = currentTick; - updateThrottleSpeed = true; - } - } - - if (!updateThrottleSpeed) - return; - - var maxUploadPerSecond = m_options.MaxUploadPrSecond; - if (maxUploadPerSecond != m_lastUploadThrottleSpeed) - { - m_lastUploadThrottleSpeed = maxUploadPerSecond; - m_initialUploadThrottleSpeed = maxUploadPerSecond / m_maxConcurrentUploads; - m_progressUpdater.UpdateThrottleSpeeds(maxUploadPerSecond); - } - } - - private class Worker : IDisposable - { - public Task Task; - public IBackend Backend; - - public Worker(IBackend backend) - { - Backend = backend; - Task = Task.FromResult(true); - } - - public void Dispose() - { - this.Task?.Dispose(); - this.Backend?.Dispose(); - } - } - } -} \ No newline at end of file diff --git a/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs b/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs index 75297a5d3..372bc2cf5 100644 --- a/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs +++ b/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs @@ -260,6 +260,7 @@ namespace Duplicati.Library.Main.Operation.Backup return RunOnMain(() => m_database.UpdateFullBackupStateInFileset(filesetid, true, GetTransaction())); } + // TODO: Make IAsyncEnumerable or get rid of the SingleRunner setup public Task GetMissingIndexFilesAsync() { return RunOnMain(() => m_database.GetMissingIndexFiles(GetTransaction()).ToArray()); diff --git a/Duplicati/Library/Main/Operation/Backup/Channels.cs b/Duplicati/Library/Main/Operation/Backup/Channels.cs index a403e7816..6a6576f67 100644 --- a/Duplicati/Library/Main/Operation/Backup/Channels.cs +++ b/Duplicati/Library/Main/Operation/Backup/Channels.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using CoCoL; @@ -28,10 +28,6 @@ namespace Duplicati.Library.Main.Operation.Backup /// internal class Channels { - /// - /// Requests to the backend are send over this channel and picked up by the class - /// - public readonly IChannel BackendRequest = ChannelManager.CreateChannel(); /// /// When the backup completes, all in-progress archives are sent from the to the /// diff --git a/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs b/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs index 7e3900e89..7b43c4d36 100644 --- a/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs +++ b/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using CoCoL; @@ -24,7 +24,6 @@ using Duplicati.Library.Main.Operation.Common; using Duplicati.Library.Main.Volumes; using System; using System.Threading.Tasks; -using static Duplicati.Library.Main.Operation.Common.BackendHandler; namespace Duplicati.Library.Main.Operation.Backup { @@ -35,13 +34,12 @@ namespace Duplicati.Library.Main.Operation.Backup /// internal static class DataBlockProcessor { - public static Task Run(Channels channels, BackupDatabase database, Options options, ITaskReader taskreader) + public static Task Run(Channels channels, BackupDatabase database, IBackendManager backendManager, Options options, ITaskReader taskreader) { return AutomationExtensions.RunTask( new { Input = channels.OutputBlocks.AsRead(), - Output = channels.BackendRequest.AsWrite(), SpillPickup = channels.SpillPickup.AsWrite(), }, @@ -131,22 +129,22 @@ namespace Duplicati.Library.Main.Operation.Backup await database.CommitTransactionAsync("CommitAddBlockToOutputFlush"); - FileEntryItem blockEntry = blockvolume.CreateFileEntryForUpload(options); - - TemporaryIndexVolume indexVolumeCopy = null; + IndexVolumeWriter indexVolumeCopy = null; if (indexvolume != null) { - indexVolumeCopy = new TemporaryIndexVolume(options); - indexvolume.CopyTo(indexVolumeCopy, false); + // TODO: It is much easier to let the BackendManager deal with index files, + // but it adds a bit of strain to the database + indexVolumeCopy = await indexvolume.CreateVolume(blockvolume.RemoteFilename, options, database); + // Create link before upload is started, it will be removed later if upload fails + await database.AddIndexBlockLinkAsync(indexVolumeCopy.VolumeID, blockvolume.VolumeID).ConfigureAwait(false); } - var uploadRequest = new VolumeUploadRequest(blockvolume, blockEntry, indexVolumeCopy, options, database); - + var blockVolumeCopy = blockvolume; blockvolume = null; indexvolume = null; - // Write to output at the end here to prevent sending a full volume to the SpillCollector - await self.Output.WriteAsync(uploadRequest); + await backendManager.PutAsync(blockVolumeCopy, indexVolumeCopy, null, false, taskreader.ProgressToken); + } } diff --git a/Duplicati/Library/Main/Operation/Backup/RecreateMissingIndexFiles.cs b/Duplicati/Library/Main/Operation/Backup/RecreateMissingIndexFiles.cs index 89de62a74..46e80a80a 100644 --- a/Duplicati/Library/Main/Operation/Backup/RecreateMissingIndexFiles.cs +++ b/Duplicati/Library/Main/Operation/Backup/RecreateMissingIndexFiles.cs @@ -1,66 +1,56 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System.Threading.Tasks; -using CoCoL; using Duplicati.Library.Main.Operation.Common; -namespace Duplicati.Library.Main.Operation.Backup +namespace Duplicati.Library.Main.Operation.Backup; + +internal static class RecreateMissingIndexFiles { - internal static class RecreateMissingIndexFiles + /// + /// The tag used for log messages + /// + private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(RecreateMissingIndexFiles)); + + public static async Task Run(BackupDatabase database, IBackendManager backendManager, Options options, ITaskReader taskreader) { - /// - /// The tag used for log messages - /// - private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(RecreateMissingIndexFiles)); - - public static Task Run(Channels channels, BackupDatabase database, Options options, ITaskReader taskreader) + if (options.IndexfilePolicy != Options.IndexFileStrategy.None) { - return AutomationExtensions.RunTask(new + foreach (var blockfile in await database.GetMissingIndexFilesAsync().ConfigureAwait(false)) { - UploadChannel = channels.BackendRequest.AsWrite() - }, + if (!await taskreader.ProgressRendevouz().ConfigureAwait(false)) + return; - async self => - { - if (options.IndexfilePolicy != Options.IndexFileStrategy.None) - { - foreach (var blockfile in await database.GetMissingIndexFilesAsync()) - { - if (!await taskreader.ProgressRendevouz().ConfigureAwait(false)) - return; + Logging.Log.WriteInformationMessage(LOGTAG, "RecreateMissingIndexFile", "Re-creating missing index file for {0}", blockfile); + var w = await Common.IndexVolumeCreator.CreateIndexVolume(blockfile, options, database).ConfigureAwait(false); - Logging.Log.WriteInformationMessage(LOGTAG, "RecreateMissingIndexFile", "Re-creating missing index file for {0}", blockfile); - var w = await Common.IndexVolumeCreator.CreateIndexVolume(blockfile, options, database); - - if (!await taskreader.ProgressRendevouz().ConfigureAwait(false)) - return; - - await database.UpdateRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null); - await self.UploadChannel.WriteAsync(new IndexVolumeUploadRequest(w)); - } - } - }); + if (!await taskreader.ProgressRendevouz().ConfigureAwait(false)) + return; + await database.UpdateRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null).ConfigureAwait(false); + await backendManager.PutAsync(w, null, null, false, taskreader.ProgressToken).ConfigureAwait(false); + } } - } + } + diff --git a/Duplicati/Library/Main/Operation/Backup/SpillCollectorProcess.cs b/Duplicati/Library/Main/Operation/Backup/SpillCollectorProcess.cs index 3865ad2f0..c391f7b21 100644 --- a/Duplicati/Library/Main/Operation/Backup/SpillCollectorProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/SpillCollectorProcess.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using CoCoL; @@ -25,7 +25,6 @@ using Duplicati.Library.Main.Volumes; using System; using System.Collections.Generic; using System.Threading.Tasks; -using static Duplicati.Library.Main.Operation.Common.BackendHandler; namespace Duplicati.Library.Main.Operation.Backup { @@ -49,13 +48,12 @@ namespace Duplicati.Library.Main.Operation.Backup /// internal static class SpillCollectorProcess { - public static Task Run(Channels channels, Options options, BackupDatabase database, ITaskReader taskreader) + public static Task Run(Channels channels, Options options, BackupDatabase database, IBackendManager backendManager, ITaskReader taskreader) { return AutomationExtensions.RunTask( new { - Input = channels.SpillPickup.AsRead(), - Output = channels.BackendRequest.AsWrite(), + Input = channels.SpillPickup.AsRead() }, async self => @@ -135,7 +133,7 @@ namespace Duplicati.Library.Main.Operation.Backup if (target.BlockVolume.Filesize > options.VolumeSize - options.Blocksize) { target.BlockVolume.Close(); - await UploadVolumeAndIndex(target, self.Output, options, database).ConfigureAwait(false); + await UploadVolumeAndIndex(target, options, database, backendManager, taskreader).ConfigureAwait(false); target = null; } } @@ -156,25 +154,25 @@ namespace Duplicati.Library.Main.Operation.Backup await taskreader.ProgressRendevouz().ConfigureAwait(false); n.BlockVolume.Close(); - await UploadVolumeAndIndex(n, self.Output, options, database).ConfigureAwait(false); + await UploadVolumeAndIndex(n, options, database, backendManager, taskreader).ConfigureAwait(false); } }); } - private static async Task UploadVolumeAndIndex(SpillVolumeRequest target, IWriteChannel outputChannel, Options options, BackupDatabase database) + private static async Task UploadVolumeAndIndex(SpillVolumeRequest target, Options options, BackupDatabase database, IBackendManager backendManager, ITaskReader taskreader) { - var blockEntry = target.BlockVolume.CreateFileEntryForUpload(options); - - TemporaryIndexVolume indexVolumeCopy = null; + IndexVolumeWriter indexVolumeCopy = null; if (target.IndexVolume != null) { - indexVolumeCopy = new TemporaryIndexVolume(options); - target.IndexVolume.CopyTo(indexVolumeCopy, false); + // TODO: It is much easier to let the BackendManager deal with index files, + // but it adds a bit of strain to the database + indexVolumeCopy = await target.IndexVolume.CreateVolume(target.BlockVolume.RemoteFilename, options, database).ConfigureAwait(false); + // Create link before upload is started, it will be removed later if upload fails + await database.AddIndexBlockLinkAsync(indexVolumeCopy.VolumeID, target.BlockVolume.VolumeID).ConfigureAwait(false); } - var uploadRequest = new VolumeUploadRequest(target.BlockVolume, blockEntry, indexVolumeCopy, options, database); - await outputChannel.WriteAsync(uploadRequest).ConfigureAwait(false); + await backendManager.PutAsync(target.BlockVolume, indexVolumeCopy, null, false, taskreader.ProgressToken).ConfigureAwait(false); } } } diff --git a/Duplicati/Library/Main/Operation/Backup/UploadRealFilelist.cs b/Duplicati/Library/Main/Operation/Backup/UploadRealFilelist.cs index 3c088f83c..284884cf4 100644 --- a/Duplicati/Library/Main/Operation/Backup/UploadRealFilelist.cs +++ b/Duplicati/Library/Main/Operation/Backup/UploadRealFilelist.cs @@ -1,91 +1,83 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Threading.Tasks; -using CoCoL; using Duplicati.Library.Main.Volumes; -namespace Duplicati.Library.Main.Operation.Backup +namespace Duplicati.Library.Main.Operation.Backup; + +internal static class UploadRealFilelist { - internal static class UploadRealFilelist + /// + /// The tag used for log messages + /// + private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(UploadRealFilelist)); + + public static async Task Run(BackupResults result, BackupDatabase db, IBackendManager backendManager, Options options, FilesetVolumeWriter filesetvolume, long filesetid, Common.ITaskReader taskreader) { - /// - /// The tag used for log messages - /// - private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(UploadRealFilelist)); + // We ignore the stop signal, but not the pause and terminate + await taskreader.ProgressRendevouz().ConfigureAwait(false); - public static Task Run(Channels channels, BackupResults result, BackupDatabase db, Options options, FilesetVolumeWriter filesetvolume, long filesetid, Common.ITaskReader taskreader) + // Update the reported source and backend changes + using (new Logging.Timer(LOGTAG, "UpdateChangeStatistics", "UpdateChangeStatistics")) + await db.UpdateChangeStatisticsAsync(result); + + var changeCount = + result.AddedFiles + result.ModifiedFiles + result.DeletedFiles + + result.AddedFolders + result.ModifiedFolders + result.DeletedFolders + + result.AddedSymlinks + result.ModifiedSymlinks + result.DeletedSymlinks; + + //Changes in the filelist triggers a filelist upload + if (options.UploadUnchangedBackups || changeCount > 0) { - return AutomationExtensions.RunTask(new + using (new Logging.Timer(LOGTAG, "UploadNewFileset", "Uploading a new fileset")) { - Output = channels.BackendRequest.AsWrite() - }, + if (!string.IsNullOrEmpty(options.ControlFiles)) + foreach (var p in options.ControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries)) + filesetvolume.AddControlFile(p, options.GetCompressionHintFromFilename(p)); - async self => - { // We ignore the stop signal, but not the pause and terminate await taskreader.ProgressRendevouz().ConfigureAwait(false); - // Update the reported source and backend changes - using (new Logging.Timer(LOGTAG, "UpdateChangeStatistics", "UpdateChangeStatistics")) - await db.UpdateChangeStatisticsAsync(result); + await db.WriteFilesetAsync(filesetvolume, filesetid).ConfigureAwait(false); + filesetvolume.Close(); - var changeCount = - result.AddedFiles + result.ModifiedFiles + result.DeletedFiles + - result.AddedFolders + result.ModifiedFolders + result.DeletedFolders + - result.AddedSymlinks + result.ModifiedSymlinks + result.DeletedSymlinks; + // We ignore the stop signal, but not the pause and terminate + await taskreader.ProgressRendevouz().ConfigureAwait(false); - //Changes in the filelist triggers a filelist upload - if (options.UploadUnchangedBackups || changeCount > 0) - { - using (new Logging.Timer(LOGTAG, "UploadNewFileset", "Uploading a new fileset")) - { - if (!string.IsNullOrEmpty(options.ControlFiles)) - foreach (var p in options.ControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries)) - filesetvolume.AddControlFile(p, options.GetCompressionHintFromFilename(p)); + await db.UpdateRemoteVolumeAsync(filesetvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null).ConfigureAwait(false); + await db.CommitTransactionAsync("CommitUpdateRemoteVolume").ConfigureAwait(false); - // We ignore the stop signal, but not the pause and terminate - await taskreader.ProgressRendevouz().ConfigureAwait(false); - - await db.WriteFilesetAsync(filesetvolume, filesetid); - filesetvolume.Close(); - - // We ignore the stop signal, but not the pause and terminate - await taskreader.ProgressRendevouz().ConfigureAwait(false); - - await db.UpdateRemoteVolumeAsync(filesetvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null); - await db.CommitTransactionAsync("CommitUpdateRemoteVolume"); - await self.Output.WriteAsync(new FilesetUploadRequest(filesetvolume)); - } - } - else - { - Logging.Log.WriteVerboseMessage(LOGTAG, "RemovingLeftoverTempFile", "removing temp files, as no data needs to be uploaded"); - await db.RemoveRemoteVolumeAsync(filesetvolume.RemoteFilename); - } - - await db.CommitTransactionAsync("CommitUpdateRemoteVolume"); - }); + await backendManager.PutAsync(filesetvolume, null, null, false, taskreader.ProgressToken).ConfigureAwait(false); + } } + else + { + Logging.Log.WriteVerboseMessage(LOGTAG, "RemovingLeftoverTempFile", "removing temp files, as no data needs to be uploaded"); + await db.RemoveRemoteVolumeAsync(filesetvolume.RemoteFilename); + } + + await db.CommitTransactionAsync("CommitUpdateRemoteVolume"); } } + diff --git a/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs b/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs index 9f5d1e937..e788e0469 100644 --- a/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs +++ b/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs @@ -1,25 +1,24 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using CoCoL; using Duplicati.Library.Main.Operation.Common; using Duplicati.Library.Main.Volumes; using System; @@ -38,114 +37,105 @@ namespace Duplicati.Library.Main.Operation.Backup /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(UploadSyntheticFilelist)); - public static Task Run(Channels channels, BackupDatabase database, Options options, BackupResults result, ITaskReader taskreader, string lastTempFilelist, long lastTempFilesetId) + public static async Task Run(BackupDatabase database, Options options, BackupResults result, ITaskReader taskreader, IBackendManager backendManager, string lastTempFilelist, long lastTempFilesetId) { - return AutomationExtensions.RunTask(new + // Check if we should upload a synthetic filelist + if (options.DisableSyntheticFilelist || string.IsNullOrWhiteSpace(lastTempFilelist) || lastTempFilesetId <= 0) + return; + + // Check that we still need to process this after the cleanup has performed its duties + var syntbase = await database.GetRemoteVolumeFromFilesetIDAsync(lastTempFilesetId); + + // If we do not have a valid entry, warn and quit + if (syntbase.Name == null) { - UploadChannel = channels.BackendRequest.AsWrite() - }, + // TODO: If the repair succeeds, this could give a false warning? + Logging.Log.WriteWarningMessage(LOGTAG, "MissingTemporaryFilelist", null, "Expected there to be a temporary fileset for synthetic filelist ({0}, {1}), but none was found?", lastTempFilesetId, lastTempFilelist); + return; + } - async self => + // Files is missing or repaired + if (syntbase.State != RemoteVolumeState.Uploading && syntbase.State != RemoteVolumeState.Temporary) { - // Check if we should upload a synthetic filelist - if (options.DisableSyntheticFilelist || string.IsNullOrWhiteSpace(lastTempFilelist) || lastTempFilesetId <= 0) - return; + Logging.Log.WriteInformationMessage(LOGTAG, "SkippingSyntheticListUpload", "Skipping synthetic upload because temporary fileset appers to be complete: ({0}, {1}, {2})", lastTempFilesetId, lastTempFilelist, syntbase.State); + return; + } - // Check that we still need to process this after the cleanup has performed its duties - var syntbase = await database.GetRemoteVolumeFromFilesetIDAsync(lastTempFilesetId); + // Ready to build and upload the synthetic list + await database.CommitTransactionAsync("PreSyntheticFilelist"); + var incompleteFilesets = (await database.GetIncompleteFilesetsAsync()).OrderBy(x => x.Value).ToList(); - // If we do not have a valid entry, warn and quit - if (syntbase.Name == null) + if (!incompleteFilesets.Any()) + { + return; + } + + if (!await taskreader.ProgressRendevouz().ConfigureAwait(false)) + return; + + result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PreviousBackupFinalize); + Logging.Log.WriteInformationMessage(LOGTAG, "PreviousBackupFilelistUpload", "Uploading filelist from previous interrupted backup"); + + var incompleteSet = incompleteFilesets.Last(); + var badIds = from n in incompleteFilesets select n.Key; + + var prevs = (from n in await database.GetFilesetTimesAsync() + where + n.Key < incompleteSet.Key + && + !badIds.Contains(n.Key) + orderby n.Key + select n.Key).ToArray(); + + var prevId = prevs.Length == 0 ? -1 : prevs.Last(); + + FilesetVolumeWriter fsw = null; + try + { + var s = 1; + var fileTime = incompleteSet.Value + TimeSpan.FromSeconds(s); + + // Probe for an unused filename + while (s < 60) { - // TODO: If the repair succeeds, this could give a false warning? - Logging.Log.WriteWarningMessage(LOGTAG, "MissingTemporaryFilelist", null, "Expected there to be a temporary fileset for synthetic filelist ({0}, {1}), but none was found?", lastTempFilesetId, lastTempFilelist); - return; + var id = await database.GetRemoteVolumeIDAsync(VolumeBase.GenerateFilename(RemoteVolumeType.Files, options, null, fileTime)); + if (id < 0) + break; + + fileTime = incompleteSet.Value + TimeSpan.FromSeconds(++s); } - // Files is missing or repaired - if (syntbase.State != RemoteVolumeState.Uploading && syntbase.State != RemoteVolumeState.Temporary) - { - Logging.Log.WriteInformationMessage(LOGTAG, "SkippingSyntheticListUpload", "Skipping synthetic upload because temporary fileset appers to be complete: ({0}, {1}, {2})", lastTempFilesetId, lastTempFilelist, syntbase.State); - return; - } + fsw = new FilesetVolumeWriter(options, fileTime); + fsw.VolumeID = await database.RegisterRemoteVolumeAsync(fsw.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary); - // Ready to build and upload the synthetic list - await database.CommitTransactionAsync("PreSyntheticFilelist"); - var incompleteFilesets = (await database.GetIncompleteFilesetsAsync()).OrderBy(x => x.Value).ToList(); + if (!string.IsNullOrEmpty(options.ControlFiles)) + foreach (var p in options.ControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries)) + fsw.AddControlFile(p, options.GetCompressionHintFromFilename(p)); - if (!incompleteFilesets.Any()) - { - return; - } + // We declare this to be a partial backup since the synthetic filelist is only created + // when a backup is interrupted. + fsw.CreateFilesetFile(false); + var newFilesetID = await database.CreateFilesetAsync(fsw.VolumeID, fileTime); + await database.LinkFilesetToVolumeAsync(newFilesetID, fsw.VolumeID); + await database.AppendFilesFromPreviousSetAsync(null, newFilesetID, prevId, fileTime); + + await database.WriteFilesetAsync(fsw, newFilesetID); + fsw.Close(); if (!await taskreader.ProgressRendevouz().ConfigureAwait(false)) return; - result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PreviousBackupFinalize); - Logging.Log.WriteInformationMessage(LOGTAG, "PreviousBackupFilelistUpload", "Uploading filelist from previous interrupted backup"); + await database.UpdateRemoteVolumeAsync(fsw.RemoteFilename, RemoteVolumeState.Uploading, -1, null); + await database.CommitTransactionAsync("CommitUpdateFilelistVolume"); - var incompleteSet = incompleteFilesets.Last(); - var badIds = from n in incompleteFilesets select n.Key; - - var prevs = (from n in await database.GetFilesetTimesAsync() - where - n.Key < incompleteSet.Key - && - !badIds.Contains(n.Key) - orderby n.Key - select n.Key).ToArray(); - - var prevId = prevs.Length == 0 ? -1 : prevs.Last(); - - FilesetVolumeWriter fsw = null; - try - { - var s = 1; - var fileTime = incompleteSet.Value + TimeSpan.FromSeconds(s); - - // Probe for an unused filename - while (s < 60) - { - var id = await database.GetRemoteVolumeIDAsync(VolumeBase.GenerateFilename(RemoteVolumeType.Files, options, null, fileTime)); - if (id < 0) - break; - - fileTime = incompleteSet.Value + TimeSpan.FromSeconds(++s); - } - - fsw = new FilesetVolumeWriter(options, fileTime); - fsw.VolumeID = await database.RegisterRemoteVolumeAsync(fsw.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary); - - if (!string.IsNullOrEmpty(options.ControlFiles)) - foreach (var p in options.ControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries)) - fsw.AddControlFile(p, options.GetCompressionHintFromFilename(p)); - - // We declare this to be a partial backup since the synthetic filelist is only created - // when a backup is interrupted. - fsw.CreateFilesetFile(false); - var newFilesetID = await database.CreateFilesetAsync(fsw.VolumeID, fileTime); - await database.LinkFilesetToVolumeAsync(newFilesetID, fsw.VolumeID); - await database.AppendFilesFromPreviousSetAsync(null, newFilesetID, prevId, fileTime); - - await database.WriteFilesetAsync(fsw, newFilesetID); - fsw.Close(); - - if (!await taskreader.ProgressRendevouz().ConfigureAwait(false)) - return; - - await database.UpdateRemoteVolumeAsync(fsw.RemoteFilename, RemoteVolumeState.Uploading, -1, null); - await database.CommitTransactionAsync("CommitUpdateFilelistVolume"); - - await self.UploadChannel.WriteAsync(new FilesetUploadRequest(fsw)); - } - catch - { - await database.RollbackTransactionAsync(); - fsw?.Dispose(); - throw; - } + await backendManager.PutAsync(fsw, null, null, false, taskreader.ProgressToken); + } + catch + { + await database.RollbackTransactionAsync(); + fsw?.Dispose(); + throw; } - ); } } } diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 292d7cfb4..258119897 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -33,6 +33,7 @@ using Duplicati.Library.Utility; using Duplicati.Library.Common.IO; using Duplicati.Library.Logging; using Duplicati.Library.Main.Operation.Common; +using System.Data; namespace Duplicati.Library.Main.Operation { @@ -173,7 +174,6 @@ namespace Duplicati.Library.Main.Operation private sealed record PreBackupVerifyResult( LocalBackupDatabase Database, - BackendManager BackendManager, string LastTempFilelist, long LastTempFilesetId ); @@ -192,13 +192,12 @@ namespace Duplicati.Library.Main.Operation /// and returned to the caller in an open state in either case. /// /// Results from the pre-backup verification - private static async Task PreBackupVerify(string backendurl, Options options, BackupResults result) + private static async Task PreBackupVerify(Options options, BackupResults result, IBackendManager backendManager) { result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PreBackupVerify); // Setup variables LocalBackupDatabase database = null; - BackendManager backendManager = null; // If we have an interrupted backup, grab the fileset string lastTempFilelist = null; @@ -209,8 +208,6 @@ namespace Duplicati.Library.Main.Operation try { database = new LocalBackupDatabase(options.Dbpath, options); - backendManager = new BackendManager(backendurl, options, result.BackendWriter, database); - result.SetDatabase(database); result.Dryrun = options.Dryrun; @@ -248,41 +245,36 @@ namespace Duplicati.Library.Main.Operation { if (options.NoBackendverification) { - FilelistProcessor.VerifyLocalList(backendManager, database); + await FilelistProcessor.VerifyLocalList(backendManager, database, result.TaskControl.ProgressToken).ConfigureAwait(false); await UpdateStorageStatsFromDatabase(result, database, options, backendManager, result.TaskControl.ProgressToken).ConfigureAwait(false); } else - FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, new string[] { lastTempFilelist }, logErrors: false); + await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, new string[] { lastTempFilelist }, logErrors: false).ConfigureAwait(false); } catch (RemoteListVerificationException ex) { if (options.AutoCleanup) { - Logging.Log.WriteWarningMessage(LOGTAG, "BackendVerifyFailedAttemptingCleanup", ex, "Backend verification failed, attempting automatic cleanup"); + Log.WriteWarningMessage(LOGTAG, "BackendVerifyFailedAttemptingCleanup", ex, "Backend verification failed, attempting automatic cleanup"); result.RepairResults = new RepairResults(result); - - // Close the database to allow the repair to run, it may create a new database - backendManager.Dispose(); database.Dispose(); database = null; - backendManager = null; result.SetDatabase(null); - new RepairHandler(backendurl, options, (RepairResults)result.RepairResults).Run(); + new RepairHandler(options, (RepairResults)result.RepairResults).Run(backendManager, null); // Re-open the database and backend manager database = new LocalBackupDatabase(options.Dbpath, options); - backendManager = new BackendManager(backendurl, options, result.BackendWriter, database); result.SetDatabase(database); - Logging.Log.WriteInformationMessage(LOGTAG, "BackendCleanupFinished", "Backend cleanup finished, retrying verification"); - FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, new string[] { lastTempFilelist }); + Log.WriteInformationMessage(LOGTAG, "BackendCleanupFinished", "Backend cleanup finished, retrying verification"); + await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, new string[] { lastTempFilelist }).ConfigureAwait(false); } else throw; } - return new PreBackupVerifyResult(database, backendManager, lastTempFilelist, lastTempFilesetId); + return new PreBackupVerifyResult(database, lastTempFilelist, lastTempFilesetId); } catch { @@ -296,7 +288,7 @@ namespace Duplicati.Library.Main.Operation /// /// Performs the bulk of work by starting all relevant processes /// - private static async Task RunMainOperation(Backup.Channels channels, IEnumerable sources, Snapshots.ISnapshotService snapshot, UsnJournalService journalService, Backup.BackupDatabase database, Backup.BackupStatsCollector stats, Options options, IFilter sourcefilter, IFilter filter, BackupResults result, Common.ITaskReader taskreader, long filesetid, long lastfilesetid) + private static async Task RunMainOperation(Backup.Channels channels, IEnumerable sources, Snapshots.ISnapshotService snapshot, UsnJournalService journalService, Backup.BackupDatabase database, IBackendManager backendManager, Backup.BackupStatsCollector stats, Options options, IFilter sourcefilter, IFilter filter, BackupResults result, ITaskReader taskreader, long filesetid, long lastfilesetid) { using (new Logging.Timer(LOGTAG, "BackupMainOperation", "BackupMainOperation")) { @@ -306,7 +298,7 @@ namespace Duplicati.Library.Main.Operation Task all = Task.WhenAll( new[] { - Backup.DataBlockProcessor.Run(channels, database, options, taskreader), + Backup.DataBlockProcessor.Run(channels, database, backendManager, options, taskreader), Backup.FileBlockProcessor.Run(channels, snapshot, options, database, stats, taskreader), Backup.StreamBlockSplitter.Run(channels, options, database, taskreader), Backup.FileEnumerationProcess.Run(channels, sources, snapshot, journalService, @@ -316,21 +308,21 @@ namespace Duplicati.Library.Main.Operation () => result.PartialBackup = true, CancellationToken.None), Backup.FilePreFilterProcess.Run(channels, snapshot, options, stats, database), Backup.MetadataPreProcess.Run(channels, snapshot, options, database, lastfilesetid, taskreader), - Backup.SpillCollectorProcess.Run(channels, options, database, taskreader), + Backup.SpillCollectorProcess.Run(channels, options, database, backendManager, taskreader), Backup.ProgressHandler.Run(channels, result) } // Spawn additional block hashers - .Union( + .Concat( Enumerable.Range(0, options.ConcurrencyBlockHashers - 1).Select(x => Backup.StreamBlockSplitter.Run(channels, options, database, taskreader)) ) // Spawn additional compressors - .Union( + .Concat( Enumerable.Range(0, options.ConcurrencyCompressors - 1).Select(x => - Backup.DataBlockProcessor.Run(channels, database, options, taskreader)) + Backup.DataBlockProcessor.Run(channels, database, backendManager, options, taskreader)) ) // Spawn additional file processors - .Union( + .Concat( Enumerable.Range(0, options.ConcurrencyFileprocessors - 1).Select(x => Backup.FileBlockProcessor.Run(channels, snapshot, options, database, stats, taskreader) ) @@ -393,7 +385,7 @@ namespace Duplicati.Library.Main.Operation } } - private void CompactIfRequired(BackendManager backend, long lastVolumeSize) + private async Task CompactIfRequired(IBackendManager backendManager, long lastVolumeSize) { var currentIsSmall = lastVolumeSize != -1 && lastVolumeSize <= m_options.SmallFileSize; @@ -402,7 +394,7 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Delete); m_result.DeleteResults = new DeleteResults(m_result); using (var db = new LocalDeleteDatabase(m_database)) - new DeleteHandler(backend.BackendUrl, m_options, (DeleteResults)m_result.DeleteResults).DoRun(db, ref m_transaction, true, currentIsSmall, backend); + new DeleteHandler(m_options, (DeleteResults)m_result.DeleteResults).DoRun(db, ref m_transaction, true, currentIsSmall, backendManager); } else if (currentIsSmall && !m_options.NoAutoCompact) @@ -410,19 +402,19 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Compact); m_result.CompactResults = new CompactResults(m_result); using (var db = new LocalDeleteDatabase(m_database)) - new CompactHandler(backend.BackendUrl, m_options, (CompactResults)m_result.CompactResults).DoCompact(db, true, ref m_transaction, backend); + { + (var _, var tr) = await new CompactHandler(m_options, (CompactResults)m_result.CompactResults).DoCompact(db, true, m_transaction, backendManager); + m_transaction = tr; + } } } - private void PostBackupVerification(string currentFilelistVolume) + private async Task PostBackupVerification(string currentFilelistVolume, IBackendManager backendManager) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PostBackupVerify); - using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, m_database)) - { - using (new Logging.Timer(LOGTAG, "AfterBackupVerify", "AfterBackupVerify")) - FilelistProcessor.VerifyRemoteList(backend, m_options, m_database, m_result.BackendWriter, new string[] { currentFilelistVolume }); - backend.WaitForComplete(m_database, null); - } + using (new Logging.Timer(LOGTAG, "AfterBackupVerify", "AfterBackupVerify")) + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, m_database, m_result.BackendWriter, new string[] { currentFilelistVolume }).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(m_database, m_transaction, m_taskReader.ProgressToken); long remoteVolumeCount = m_database.GetRemoteVolumes().LongCount(x => x.State == RemoteVolumeState.Verified); long samplesToTest = Math.Max(m_options.BackupTestSampleCount, (long)Math.Round(remoteVolumeCount * (m_options.BackupTestPercentage / 100m), MidpointRounding.AwayFromZero)); @@ -432,16 +424,15 @@ namespace Duplicati.Library.Main.Operation m_result.TestResults = new TestResults(m_result); using (var testdb = new LocalTestDatabase(m_database)) - using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, testdb)) - new TestHandler(m_backendurl, m_options, (TestResults)m_result.TestResults) - .DoRun(samplesToTest, testdb, backend); + await new TestHandler(m_options, (TestResults)m_result.TestResults) + .DoRun(samplesToTest, testdb, backendManager).ConfigureAwait(false); } } /// /// Handler for computing backend statistics, without relying on a remote folder listing /// - private static async Task UpdateStorageStatsFromDatabase(BackupResults result, LocalBackupDatabase database, Options options, BackendManager backendManager, CancellationToken cancelToken) + private static async Task UpdateStorageStatsFromDatabase(BackupResults result, LocalBackupDatabase database, Options options, IBackendManager backendManager, CancellationToken cancelToken) { if (result.BackendWriter != null) { @@ -473,8 +464,10 @@ namespace Duplicati.Library.Main.Operation if (tasks == null || tasks.Length == 0) return source; - var ex = new List(); - ex.Add(source); + var ex = new List + { + source + }; foreach (var t in tasks) if (t != null) @@ -492,36 +485,30 @@ namespace Duplicati.Library.Main.Operation return new AggregateException(ex.First().Message, ex); } - private static async Task FlushBackend(BackupResults result, IWriteChannel uploadtarget, Task uploader) + private static async Task FlushBackend(LocalDatabase database, IDbTransaction transaction, BackupResults result, IBackendManager backendManager) { // Wait for upload completion result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_WaitForUpload); - if (!await uploadtarget.IsRetiredAsync) + try { - try - { - var flushReq = new Backup.FlushRequest(); - await uploadtarget.WriteAsync(flushReq).ConfigureAwait(false); - await uploader.ConfigureAwait(false); - // Grab the size of the last uploaded volume - return await flushReq.LastWriteSizeAsync; - } - catch (RetiredException) - { - // Retired check is not atomic, so this exception can still happen - } + await backendManager.WaitForEmptyAsync(database, transaction, result.TaskControl.ProgressToken).ConfigureAwait(false); + // Grab the size of the last uploaded volume + return backendManager.LastWriteSize; } - await uploader.ConfigureAwait(false); + catch (RetiredException) + { + } + return -1; } - public async Task RunAsync(string[] sources, Library.Utility.IFilter filter) + public async Task RunAsync(string[] sources, IBackendManager backendManager, Library.Utility.IFilter filter) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Begin); // Do a remote verification, unless disabled - var (database, backendManager, lastTempFilelist, lastTempFilesetId) = await PreBackupVerify(m_backendurl, m_options, m_result); + var (database, lastTempFilelist, lastTempFilesetId) = await PreBackupVerify(m_options, m_result, backendManager); Backup.Channels channels = new(); @@ -531,29 +518,22 @@ namespace Duplicati.Library.Main.Operation m_sourceFilter = new Library.Utility.FilterExpression(sources, true); Task parallelScanner = null; - Task uploaderTask = null; try { using (m_database = database) - using (backendManager) using (var db = new Backup.BackupDatabase(m_database, m_options)) // Setup runners and instances here using (var filesetvolume = new FilesetVolumeWriter(m_options, m_database.OperationTimestamp)) using (var stats = new Backup.BackupStatsCollector(m_result)) - using (var uploadtarget = ChannelManager.GetChannel(channels.BackendRequest.AsWrite())) { long filesetid; using var counterToken = CancellationTokenSource.CreateLinkedTokenSource(m_taskReader.ProgressToken); - var uploader = new Backup.BackendUploader(backendManager.BorrowBackend(), () => DynamicLoader.BackendLoader.GetBackend(m_backendurl, m_options.RawOptions), m_options, db, m_result.TaskControl, stats); using (var snapshot = GetSnapshot(sources, m_options)) { try { - // Start the uploader process - uploaderTask = uploader.Run(channels); - // If the previous backup was interrupted, send a synthetic list - await Backup.UploadSyntheticFilelist.Run(channels, db, m_options, m_result, m_result.TaskControl, lastTempFilelist, lastTempFilesetId); + await Backup.UploadSyntheticFilelist.Run(db, m_options, m_result, m_result.TaskControl, backendManager, lastTempFilelist, lastTempFilesetId); // Grab the previous backup ID, if any var prevfileset = m_database.FilesetTimes.FirstOrDefault(); @@ -563,7 +543,7 @@ namespace Duplicati.Library.Main.Operation var lastfilesetid = prevfileset.Value.Ticks == 0 ? -1 : prevfileset.Key; // Rebuild any index files that are missing - await Backup.RecreateMissingIndexFiles.Run(channels, db, m_options, m_result.TaskControl); + await Backup.RecreateMissingIndexFiles.Run(db, backendManager, m_options, m_result.TaskControl); // Prepare the operation by registering the filelist m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_ProcessingFiles); @@ -595,7 +575,7 @@ namespace Duplicati.Library.Main.Operation // Run the backup operation if (await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await RunMainOperation(channels, sources, snapshot, journalService, db, stats, m_options, m_sourceFilter, m_filter, m_result, m_result.TaskControl, filesetid, lastfilesetid).ConfigureAwait(false); + await RunMainOperation(channels, sources, snapshot, journalService, db, backendManager, stats, m_options, m_sourceFilter, m_filter, m_result, m_result.TaskControl, filesetid, lastfilesetid).ConfigureAwait(false); } } finally @@ -613,18 +593,15 @@ namespace Duplicati.Library.Main.Operation await db.VerifyConsistencyAsync(m_options.Blocksize, m_options.BlockhashSize, false); // Send the actual filelist - await Backup.UploadRealFilelist.Run(channels, m_result, db, m_options, filesetvolume, filesetid, m_result.TaskControl); + await Backup.UploadRealFilelist.Run(m_result, db, backendManager, m_options, filesetvolume, filesetid, m_result.TaskControl); // Wait for upload completion m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_WaitForUpload); - var lastVolumeSize = await FlushBackend(m_result, uploadtarget, uploaderTask).ConfigureAwait(false); + var lastVolumeSize = await FlushBackend(m_database, m_transaction, m_result, backendManager).ConfigureAwait(false); // Make sure we have the database up-to-date await db.CommitTransactionAsync("CommitAfterUpload", false); - // Return the backend instance to the manager, if possible - backendManager.ResetBackend(uploader.Backend); - // TODO: Remove this later m_transaction = m_database.BeginTransaction(); @@ -632,12 +609,12 @@ namespace Duplicati.Library.Main.Operation { // If this throws, we should roll back the transaction if (await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - CompactIfRequired(backendManager, lastVolumeSize); + await CompactIfRequired(backendManager, lastVolumeSize); if (m_options.UploadVerificationFile && await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_VerificationUpload); - FilelistProcessor.UploadVerificationFile(backendManager.BackendUrl, m_options, m_result.BackendWriter, m_database, m_transaction); + await FilelistProcessor.UploadVerificationFile(backendManager, m_options, m_database, m_transaction); } } catch @@ -674,7 +651,7 @@ namespace Duplicati.Library.Main.Operation if (m_options.NoBackendverification) await UpdateStorageStatsFromDatabase(m_result, m_database, m_options, backendManager, m_taskReader.ProgressToken).ConfigureAwait(false); else - PostBackupVerification(filesetvolume.RemoteFilename); + await PostBackupVerification(filesetvolume.RemoteFilename, backendManager).ConfigureAwait(false); } } @@ -693,7 +670,7 @@ namespace Duplicati.Library.Main.Operation } catch (Exception ex) { - var aex = BuildException(ex, uploaderTask, parallelScanner); + var aex = BuildException(ex, parallelScanner); Logging.Log.WriteErrorMessage(LOGTAG, "FatalError", ex, "Fatal error"); if (aex == ex) throw; diff --git a/Duplicati/Library/Main/Operation/Common/BackendHandler.cs b/Duplicati/Library/Main/Operation/Common/BackendHandler.cs deleted file mode 100644 index 78b98406a..000000000 --- a/Duplicati/Library/Main/Operation/Common/BackendHandler.cs +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - -using System; -using Duplicati.Library.Utility; - -namespace Duplicati.Library.Main.Operation.Common -{ - /// - /// This class encapsulates access to the backend and ensures at most one connection is active at a time - /// - internal class BackendHandler : SingleRunner - { - /// - /// The tag used for logging - /// - private static readonly string LOGTAG = Logging.Log.LogTagFromType(); - - /// - /// Data storage for an ongoing backend operation - /// - public class FileEntryItem - { - /// - /// The current operation this entry represents - /// - public readonly BackendActionType Operation; - /// - /// The name of the remote file - /// - public string RemoteFilename; - /// - /// The name of the local file - /// - public string LocalFilename { get { return LocalTempfile; } } - /// - /// A reference to a temporary file that is disposed upon - /// failure or completion of the item - /// - public TempFile LocalTempfile; - /// - /// True if the item has been encrypted - /// - public bool Encrypted; - /// - /// The expected hash value of the file - /// - public string Hash; - /// - /// The expected size of the file - /// - public long Size; - /// - /// A flag indicating if the file is a extra metadata file - /// that has no entry in the database - /// - public bool TrackedInDb = true; - /// - /// A flag indicating if the operation is a retry run - /// - public bool IsRetry; - - public FileEntryItem(BackendActionType operation, string remotefilename) - { - Operation = operation; - RemoteFilename = remotefilename; - Size = -1; - } - - public void SetLocalfilename(string name) - { - this.LocalTempfile = Library.Utility.TempFile.WrapExistingFile(name); - this.LocalTempfile.Protected = true; - } - - public void Encrypt(Options options) - { - if (!this.Encrypted && !options.NoEncryption) - { - var tempfile = new Library.Utility.TempFile(); - using (var enc = DynamicLoader.EncryptionLoader.GetModule(options.EncryptionModule, options.Passphrase, options.RawOptions)) - enc.Encrypt(this.LocalFilename, tempfile); - - this.DeleteLocalFile(); - - this.LocalTempfile = tempfile; - this.Hash = null; - this.Size = 0; - this.Encrypted = true; - } - } - - public static string CalculateFileHash(string filename, Options options) - { - using (System.IO.FileStream fs = System.IO.File.OpenRead(filename)) - using (var hasher = HashFactory.CreateHasher(options.FileHashAlgorithm)) - return Convert.ToBase64String(hasher.ComputeHash(fs)); - } - - - public bool UpdateHashAndSize(Options options) - { - if (Hash == null || Size < 0) - { - Hash = CalculateFileHash(this.LocalFilename, options); - Size = new System.IO.FileInfo(this.LocalFilename).Length; - return true; - } - - return false; - } - - public void DeleteLocalFile() - { - if (this.LocalTempfile != null) - { - try - { - this.LocalTempfile.Protected = false; - this.LocalTempfile.Dispose(); - } - catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "DeleteTemporaryFileError", ex, "Failed to dispose temporary file: {0}", this.LocalTempfile); } - finally { this.LocalTempfile = null; } - } - } - } - } -} - diff --git a/Duplicati/Library/Main/Operation/Common/IndexVolumeCreator.cs b/Duplicati/Library/Main/Operation/Common/IndexVolumeCreator.cs index 9f364f9d4..caa6349e1 100644 --- a/Duplicati/Library/Main/Operation/Common/IndexVolumeCreator.cs +++ b/Duplicati/Library/Main/Operation/Common/IndexVolumeCreator.cs @@ -27,6 +27,8 @@ using Duplicati.Library.Utility; namespace Duplicati.Library.Main.Operation.Common { + // TODO: Remove this class and simply rely on the database to create the index files + /// /// A collection class for keeping the temporary data required to build an index file /// @@ -67,22 +69,13 @@ namespace Duplicati.Library.Main.Operation.Common /// /// The index volume. /// The name of the block file. - /// Hash of the volume - /// Size of the volume - public async Task CreateVolume(string blockfilename, string blockHash, long blockSize, Options options, DatabaseCommon database) + /// The options used in this run. + /// The database to use. + public async Task CreateVolume(string blockfilename, Options options, DatabaseCommon database) { var w = new IndexVolumeWriter(options); w.VolumeID = await database.RegisterRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary); - w.StartVolume(blockfilename); - foreach (var n in blockHashes) - { - var args = n.Split(new char[] { ':' }, 2); - w.AddBlock(args[1], long.Parse(args[0])); - } - - w.FinishVolume(blockHash, blockSize); - var enumerator = blockListHashes.GetEnumerator(); while (enumerator.MoveNext()) { @@ -93,7 +86,12 @@ namespace Duplicati.Library.Main.Operation.Common w.WriteBlocklist(hash, data, 0, data.Length); } - w.Close(); + w.StartVolume(blockfilename); + foreach (var n in blockHashes) + { + var args = n.Split(new char[] { ':' }, 2); + w.AddBlock(args[1], long.Parse(args[0])); + } return w; } @@ -192,39 +190,6 @@ namespace Duplicati.Library.Main.Operation.Common return w; } } - - /*public static async Task ReCreateIndexVolume(string selfname, Options options, Repair.RepairDatabase database) - { - using(var h = HashFactory.CreateHasher(options.BlockHashAlgorithm)) - { - var w = new IndexVolumeWriter(options); - w.SetRemoteFilename(selfname); - - foreach(var blockvolume in await database.GetBlockVolumesFromIndexNameAsync(selfname)) - { - w.StartVolume(blockvolume.Name); - var volumeid = await database.GetRemoteVolumeIDAsync(blockvolume.Name); - - foreach(var b in await database.GetBlocksAsync(volumeid)) - w.AddBlock(b.Hash, b.Size); - - w.FinishVolume(blockvolume.Hash, blockvolume.Size); - - if (options.IndexfilePolicy == Options.IndexFileStrategy.Full) - foreach(var b in await database.GetBlocklistsAsync(volumeid, options.Blocksize, options.BlockhashSize)) - { - var bh = Convert.ToBase64String(h.ComputeHash(b.Item2, 0, b.Item3)); - if (bh != b.Item1) - throw new Exception(string.Format("Internal consistency check failed, generated index block has wrong hash, {0} vs {1}", bh, b.Item1)); - w.WriteBlocklist(b.Item1, b.Item2, 0, b.Item3); - } - } - - w.Close(); - - return w; - } - }*/ } } diff --git a/Duplicati/Library/Main/Operation/CompactHandler.cs b/Duplicati/Library/Main/Operation/CompactHandler.cs index ef2cf14db..5f245b916 100644 --- a/Duplicati/Library/Main/Operation/CompactHandler.cs +++ b/Duplicati/Library/Main/Operation/CompactHandler.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Main.Database; @@ -25,6 +25,8 @@ using Duplicati.Library.Utility; using System; using System.Collections.Generic; using System.Linq; +using System.Threading; +using System.Threading.Tasks; namespace Duplicati.Library.Main.Operation { @@ -34,18 +36,16 @@ namespace Duplicati.Library.Main.Operation /// The tag used for logging /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(); - protected readonly string m_backendurl; protected readonly Options m_options; protected readonly CompactResults m_result; - public CompactHandler(string backend, Options options, CompactResults result) + public CompactHandler(Options options, CompactResults result) { - m_backendurl = backend; m_options = options; m_result = result; } - public virtual void Run() + public virtual async Task Run(IBackendManager backendManager) { if (!System.IO.File.Exists(m_options.Dbpath)) throw new Exception(string.Format("Database file does not exist: {0}", m_options.Dbpath)); @@ -59,10 +59,12 @@ namespace Duplicati.Library.Main.Operation Utility.UpdateOptionsFromDb(db, m_options); Utility.VerifyOptionsAndUpdateDatabase(db, m_options); - var changed = DoCompact(db, false, ref tr, null); + // TODO: Handle transaction scopes better + var (changed, tr2) = DoCompact(db, false, tr, backendManager).Await(); + tr = tr2; if (changed && m_options.UploadVerificationFile) - FilelistProcessor.UploadVerificationFile(m_backendurl, m_options, m_result.BackendWriter, db, null); + await FilelistProcessor.UploadVerificationFile(backendManager, m_options, db, null); if (!m_options.Dryrun) { @@ -92,220 +94,216 @@ namespace Duplicati.Library.Main.Operation } } - internal bool DoCompact(LocalDeleteDatabase db, bool hasVerifiedBackend, ref System.Data.IDbTransaction transaction, BackendManager sharedBackend) + internal async Task<(bool, System.Data.IDbTransaction)> DoCompact(LocalDeleteDatabase db, bool hasVerifiedBackend, System.Data.IDbTransaction transaction, IBackendManager backendManager) { + var cancellationToken = CancellationToken.None; var report = db.GetCompactReport(m_options.VolumeSize, m_options.Threshold, m_options.SmallFileSize, m_options.SmallFileMaxCount, transaction); report.ReportCompactData(); if (report.ShouldReclaim || report.ShouldCompact) { // Workaround where we allow a running backendmanager to be used - using (var bk = sharedBackend == null ? new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db) : null) + if (!hasVerifiedBackend) + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, true, transaction).ConfigureAwait(false); + + var newvol = new BlockVolumeWriter(m_options); + newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, transaction); + + IndexVolumeWriter newvolindex = null; + if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None) { - var backend = bk ?? sharedBackend; - if (!hasVerifiedBackend) + newvolindex = new IndexVolumeWriter(m_options); + newvolindex.VolumeID = db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, transaction); + db.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, transaction); + } + + var blocksInVolume = 0L; + var buffer = new byte[m_options.Blocksize]; + var remoteList = db.GetRemoteVolumes().Where(n => n.State == RemoteVolumeState.Uploaded || n.State == RemoteVolumeState.Verified).ToArray(); + + //These are for bookkeeping + var uploadedVolumes = new List>(); + var deletedVolumes = new List>(); + var downloadedVolumes = new List>(); + + //We start by deleting unused volumes to save space before uploading new stuff + List fullyDeleteable = []; + if (report.DeleteableVolumes.Any()) + { + var deleteableVolumesAsHashSet = new HashSet(report.DeleteableVolumes); + fullyDeleteable = + remoteList + .Where(n => deleteableVolumesAsHashSet.Contains(n.Name)) + .Cast() + .ToList(); + } + deletedVolumes.AddRange(DoDelete(db, backendManager, fullyDeleteable, ref transaction, cancellationToken)); + + // This list is used to pick up unused volumes, + // so they can be deleted once the upload of the + // required fragments is complete + var deleteableVolumes = new List(); + + if (report.ShouldCompact) + { + newvolindex?.StartVolume(newvol.RemoteFilename); + List volumesToDownload = []; + if (report.CompactableVolumes.Any()) { - FilelistProcessor.VerifyRemoteList(backend, m_options, db, m_result.BackendWriter, true, transaction); - } - - BlockVolumeWriter newvol = new BlockVolumeWriter(m_options); - newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, transaction); - - IndexVolumeWriter newvolindex = null; - if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None) - { - newvolindex = new IndexVolumeWriter(m_options); - newvolindex.VolumeID = db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, transaction); - db.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, transaction); - } - - long blocksInVolume = 0; - byte[] buffer = new byte[m_options.Blocksize]; - var remoteList = db.GetRemoteVolumes().Where(n => n.State == RemoteVolumeState.Uploaded || n.State == RemoteVolumeState.Verified).ToArray(); - - //These are for bookkeeping - var uploadedVolumes = new List>(); - var deletedVolumes = new List>(); - var downloadedVolumes = new List>(); - - //We start by deleting unused volumes to save space before uploading new stuff - List fullyDeleteable = []; - if (report.DeleteableVolumes.Any()) - { - var deleteableVolumesAsHashSet = new HashSet(report.DeleteableVolumes); - fullyDeleteable = + var compactableVolumesAsHashSet = new HashSet(report.CompactableVolumes); + volumesToDownload = remoteList - .Where(n => deleteableVolumesAsHashSet.Contains(n.Name)) + .Where(n => compactableVolumesAsHashSet.Contains(n.Name)) .Cast() .ToList(); } - deletedVolumes.AddRange(DoDelete(db, backend, fullyDeleteable, ref transaction)); - // This list is used to pick up unused volumes, - // so they can be deleted once the upload of the - // required fragments is complete - var deleteableVolumes = new List(); - - if (report.ShouldCompact) + using (var q = db.CreateBlockQueryHelper(transaction)) { - newvolindex?.StartVolume(newvol.RemoteFilename); - List volumesToDownload = []; - if (report.CompactableVolumes.Any()) + await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(volumesToDownload, m_result.TaskControl.ProgressToken)) { - var compactableVolumesAsHashSet = new HashSet(report.CompactableVolumes); - volumesToDownload = - remoteList - .Where(n => compactableVolumesAsHashSet.Contains(n.Name)) - .Cast() - .ToList(); - } - - using (var q = db.CreateBlockQueryHelper(transaction)) - { - foreach (var entry in new AsyncDownloader(volumesToDownload, backend)) + using (tmpfile) { - using (var tmpfile = entry.TempFile) + var entry = new RemoteVolume(name, hash, size); + if (!m_result.TaskControl.ProgressRendevouz().Await()) { - if (!m_result.TaskControl.ProgressRendevouz().Await()) - { - backend.WaitForComplete(db, transaction); - return false; - } + backendManager.WaitForEmptyAsync(db, transaction, cancellationToken).Await(); + return (false, transaction); + } - downloadedVolumes.Add(new KeyValuePair(entry.Name, entry.Size)); - var inst = VolumeBase.ParseFilename(entry.Name); - using (var f = new BlockVolumeReader(inst.CompressionModule, tmpfile, m_options)) + downloadedVolumes.Add(new KeyValuePair(entry.Name, entry.Size)); + var inst = VolumeBase.ParseFilename(entry.Name); + using (var f = new BlockVolumeReader(inst.CompressionModule, tmpfile, m_options)) + { + foreach (var e in f.Blocks) { - foreach (var e in f.Blocks) + if (q.UseBlock(e.Key, e.Value, transaction)) { - if (q.UseBlock(e.Key, e.Value, transaction)) + //TODO: How do we get the compression hint? Reverse query for filename in db? + var s = f.ReadBlock(e.Key, buffer); + if (s != e.Value) + throw new Exception(string.Format("Size mismatch problem for block {0}, {1} vs {2}", e.Key, s, e.Value)); + + newvol.AddBlock(e.Key, buffer, 0, s, Duplicati.Library.Interface.CompressionHint.Compressible); + if (newvolindex != null) + newvolindex.AddBlock(e.Key, e.Value); + + db.RegisterDuplicatedBlock(e.Key, e.Value, newvol.VolumeID, transaction); + blocksInVolume++; + + if (newvol.Filesize > m_options.VolumeSize) { - //TODO: How do we get the compression hint? Reverse query for filename in db? - var s = f.ReadBlock(e.Key, buffer); - if (s != e.Value) - throw new Exception(string.Format("Size mismatch problem for block {0}, {1} vs {2}", e.Key, s, e.Value)); + FinishVolumeAndUpload(db, backendManager, newvol, newvolindex, uploadedVolumes); - newvol.AddBlock(e.Key, buffer, 0, s, Duplicati.Library.Interface.CompressionHint.Compressible); - if (newvolindex != null) - newvolindex.AddBlock(e.Key, e.Value); + newvol = new BlockVolumeWriter(m_options); + newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, transaction); - db.RegisterDuplicatedBlock(e.Key, e.Value, newvol.VolumeID, transaction); - blocksInVolume++; - - if (newvol.Filesize > m_options.VolumeSize) + if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None) { - FinishVolumeAndUpload(db, backend, newvol, newvolindex, uploadedVolumes); + newvolindex = new IndexVolumeWriter(m_options); + newvolindex.VolumeID = db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, transaction); + db.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, transaction); + newvolindex.StartVolume(newvol.RemoteFilename); + } - newvol = new BlockVolumeWriter(m_options); - newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, transaction); + blocksInVolume = 0; - if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None) - { - newvolindex = new IndexVolumeWriter(m_options); - newvolindex.VolumeID = db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, transaction); - db.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, transaction); - newvolindex.StartVolume(newvol.RemoteFilename); - } + // Wait for the backend to catch up + backendManager.WaitForEmptyAsync(db, transaction, cancellationToken).Await(); - blocksInVolume = 0; - - // Wait for the backend to catch up - backend.WaitForEmpty(db, transaction); - - // Commit as we have uploaded a volume - if (!m_options.Dryrun) - { - transaction.Commit(); - transaction = db.BeginTransaction(); - } + // Commit as we have uploaded a volume + if (!m_options.Dryrun) + { + transaction.Commit(); + transaction = db.BeginTransaction(); } } } } - - deleteableVolumes.Add(entry); } - } - if (blocksInVolume > 0) - { - FinishVolumeAndUpload(db, backend, newvol, newvolindex, uploadedVolumes); + deleteableVolumes.Add(entry); } - else + } + + if (blocksInVolume > 0) + { + FinishVolumeAndUpload(db, backendManager, newvol, newvolindex, uploadedVolumes); + } + else + { + db.RemoveRemoteVolume(newvol.RemoteFilename, transaction); + if (newvolindex != null) { - db.RemoveRemoteVolume(newvol.RemoteFilename, transaction); - if (newvolindex != null) - { - db.RemoveRemoteVolume(newvolindex.RemoteFilename, transaction); - newvolindex.FinishVolume(null, 0); - } + db.RemoveRemoteVolume(newvolindex.RemoteFilename, transaction); + newvolindex.FinishVolume(null, 0); } } } - else - { - newvolindex?.Dispose(); - newvol.Dispose(); - } - - deletedVolumes.AddRange(DoDelete(db, backend, deleteableVolumes, ref transaction)); - - var downloadSize = downloadedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a, x) => a + x.Value); - var deletedSize = deletedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a, x) => a + x.Value); - var uploadSize = uploadedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a, x) => a + x.Value); - - m_result.DeletedFileCount = deletedVolumes.Count; - m_result.DownloadedFileCount = downloadedVolumes.Count; - m_result.UploadedFileCount = uploadedVolumes.Count; - m_result.DeletedFileSize = deletedSize; - m_result.DownloadedFileSize = downloadSize; - m_result.UploadedFileSize = uploadSize; - m_result.Dryrun = m_options.Dryrun; - - if (m_result.Dryrun) - { - if (downloadedVolumes.Count == 0) - Logging.Log.WriteDryrunMessage(LOGTAG, "CompactResults", "Would delete {0} files, which would reduce storage by {1}", m_result.DeletedFileCount, Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize)); - else - Logging.Log.WriteDryrunMessage(LOGTAG, "CompactResults", "Would download {0} file(s) with a total size of {1}, delete {2} file(s) with a total size of {3}, and compact to {4} file(s) with a size of {5}, which would reduce storage by {6} file(s) and {7}", - m_result.DownloadedFileCount, - Library.Utility.Utility.FormatSizeString(m_result.DownloadedFileSize), - m_result.DeletedFileCount, - Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize), m_result.UploadedFileCount, - Library.Utility.Utility.FormatSizeString(m_result.UploadedFileSize), - m_result.DeletedFileCount - m_result.UploadedFileCount, - Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize - m_result.UploadedFileSize)); - } - else - { - if (m_result.DownloadedFileCount == 0) - Logging.Log.WriteInformationMessage(LOGTAG, "CompactResults", "Deleted {0} files, which reduced storage by {1}", m_result.DeletedFileCount, Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize)); - else - Logging.Log.WriteInformationMessage(LOGTAG, "CompactResults", "Downloaded {0} file(s) with a total size of {1}, deleted {2} file(s) with a total size of {3}, and compacted to {4} file(s) with a size of {5}, which reduced storage by {6} file(s) and {7}", - m_result.DownloadedFileCount, - Library.Utility.Utility.FormatSizeString(downloadSize), - m_result.DeletedFileCount, - Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize), - m_result.UploadedFileCount, - Library.Utility.Utility.FormatSizeString(m_result.UploadedFileSize), - m_result.DeletedFileCount - m_result.UploadedFileCount, - Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize - m_result.UploadedFileSize)); - } - - backend.WaitForComplete(db, transaction); + } + else + { + newvolindex?.Dispose(); + newvol.Dispose(); } + deletedVolumes.AddRange(DoDelete(db, backendManager, deleteableVolumes, ref transaction, cancellationToken)); + + var downloadSize = downloadedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a, x) => a + x.Value); + var deletedSize = deletedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a, x) => a + x.Value); + var uploadSize = uploadedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a, x) => a + x.Value); + + m_result.DeletedFileCount = deletedVolumes.Count; + m_result.DownloadedFileCount = downloadedVolumes.Count; + m_result.UploadedFileCount = uploadedVolumes.Count; + m_result.DeletedFileSize = deletedSize; + m_result.DownloadedFileSize = downloadSize; + m_result.UploadedFileSize = uploadSize; + m_result.Dryrun = m_options.Dryrun; + + if (m_result.Dryrun) + { + if (downloadedVolumes.Count == 0) + Logging.Log.WriteDryrunMessage(LOGTAG, "CompactResults", "Would delete {0} files, which would reduce storage by {1}", m_result.DeletedFileCount, Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize)); + else + Logging.Log.WriteDryrunMessage(LOGTAG, "CompactResults", "Would download {0} file(s) with a total size of {1}, delete {2} file(s) with a total size of {3}, and compact to {4} file(s) with a size of {5}, which would reduce storage by {6} file(s) and {7}", + m_result.DownloadedFileCount, + Library.Utility.Utility.FormatSizeString(m_result.DownloadedFileSize), + m_result.DeletedFileCount, + Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize), m_result.UploadedFileCount, + Library.Utility.Utility.FormatSizeString(m_result.UploadedFileSize), + m_result.DeletedFileCount - m_result.UploadedFileCount, + Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize - m_result.UploadedFileSize)); + } + else + { + if (m_result.DownloadedFileCount == 0) + Logging.Log.WriteInformationMessage(LOGTAG, "CompactResults", "Deleted {0} files, which reduced storage by {1}", m_result.DeletedFileCount, Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize)); + else + Logging.Log.WriteInformationMessage(LOGTAG, "CompactResults", "Downloaded {0} file(s) with a total size of {1}, deleted {2} file(s) with a total size of {3}, and compacted to {4} file(s) with a size of {5}, which reduced storage by {6} file(s) and {7}", + m_result.DownloadedFileCount, + Library.Utility.Utility.FormatSizeString(downloadSize), + m_result.DeletedFileCount, + Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize), + m_result.UploadedFileCount, + Library.Utility.Utility.FormatSizeString(m_result.UploadedFileSize), + m_result.DeletedFileCount - m_result.UploadedFileCount, + Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize - m_result.UploadedFileSize)); + } + + backendManager.WaitForEmptyAsync(db, transaction, cancellationToken).Await(); + m_result.EndTime = DateTime.UtcNow; - return (m_result.DeletedFileCount + m_result.UploadedFileCount) > 0; + return ((m_result.DeletedFileCount + m_result.UploadedFileCount) > 0, transaction); } else { m_result.EndTime = DateTime.UtcNow; - return false; + return (false, transaction); } } - private IEnumerable> DoDelete(LocalDeleteDatabase db, BackendManager backend, IEnumerable deleteableVolumes, ref System.Data.IDbTransaction transaction) + private IEnumerable> DoDelete(LocalDeleteDatabase db, IBackendManager backend, IEnumerable deleteableVolumes, ref System.Data.IDbTransaction transaction, CancellationToken cancellationToken) { // Find volumes that can be deleted var remoteFilesToRemove = db.ReOrderDeleteableVolumes(deleteableVolumes, transaction).ToList(); @@ -324,7 +322,7 @@ namespace Duplicati.Library.Main.Operation } // Before we commit the current state, make sure the backend has caught up - backend.WaitForEmpty(db, transaction); + backend.WaitForEmptyAsync(db, transaction, cancellationToken).Await(); if (!m_options.Dryrun) { @@ -332,10 +330,10 @@ namespace Duplicati.Library.Main.Operation transaction = db.BeginTransaction(); } - return PerformDelete(backend, remoteFilesToRemove); + return PerformDelete(backend, remoteFilesToRemove, cancellationToken); } - private void FinishVolumeAndUpload(LocalDeleteDatabase db, BackendManager backend, BlockVolumeWriter newvol, IndexVolumeWriter newvolindex, List> uploadedVolumes) + private void FinishVolumeAndUpload(LocalDeleteDatabase db, IBackendManager backendManager, BlockVolumeWriter newvol, IndexVolumeWriter newvolindex, List> uploadedVolumes) { Action indexVolumeFinished = () => { @@ -352,17 +350,17 @@ namespace Duplicati.Library.Main.Operation if (newvolindex != null) uploadedVolumes.Add(new KeyValuePair(newvolindex.RemoteFilename, newvolindex.Filesize)); if (!m_options.Dryrun) - backend.Put(newvol, newvolindex, indexVolumeFinished); + backendManager.PutAsync(newvol, newvolindex, indexVolumeFinished, false, CancellationToken.None).Await(); else Logging.Log.WriteDryrunMessage(LOGTAG, "WouldUploadGeneratedBlockset", "Would upload generated blockset of size {0}", Library.Utility.Utility.FormatSizeString(newvol.Filesize)); } - private IEnumerable> PerformDelete(BackendManager backend, IEnumerable list) + private IEnumerable> PerformDelete(IBackendManager backendManager, IEnumerable list, CancellationToken cancelToken) { foreach (var f in list) { if (!m_options.Dryrun) - backend.Delete(f.Name, f.Size); + backendManager.DeleteAsync(f.Name, f.Size, false, cancelToken).Await(); else Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteRemoteFile", "Would delete remote file: {0}, size: {1}", f.Name, Library.Utility.Utility.FormatSizeString(f.Size)); diff --git a/Duplicati/Library/Main/Operation/DeleteHandler.cs b/Duplicati/Library/Main/Operation/DeleteHandler.cs index 37cb36e8b..fdd233154 100644 --- a/Duplicati/Library/Main/Operation/DeleteHandler.cs +++ b/Duplicati/Library/Main/Operation/DeleteHandler.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -25,6 +25,7 @@ using System.Collections.Generic; using Duplicati.Library.Interface; using Duplicati.Library.Main.Database; using Duplicati.Library.Utility; +using System.Threading; namespace Duplicati.Library.Main.Operation { @@ -36,17 +37,15 @@ namespace Duplicati.Library.Main.Operation internal static readonly string LOGTAG = Logging.Log.LogTagFromType(); private readonly DeleteResults m_result; - protected readonly string m_backendurl; protected readonly Options m_options; - public DeleteHandler(string backend, Options options, DeleteResults result) + public DeleteHandler(Options options, DeleteResults result) { - m_backendurl = backend; m_options = options; m_result = result; } - public void Run() + public void Run(IBackendManager backendManager) { if (!System.IO.File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseFileMissing"); @@ -60,7 +59,7 @@ namespace Duplicati.Library.Main.Operation Utility.UpdateOptionsFromDb(db, m_options); Utility.VerifyOptionsAndUpdateDatabase(db, m_options); - DoRun(db, ref tr, false, false, null); + DoRun(db, ref tr, false, false, backendManager); if (!m_options.Dryrun) { @@ -83,93 +82,86 @@ namespace Duplicati.Library.Main.Operation } } - public void DoRun(Database.LocalDeleteDatabase db, ref System.Data.IDbTransaction transaction, bool hasVerifiedBackend, bool forceCompact, BackendManager sharedManager) + public void DoRun(Database.LocalDeleteDatabase db, ref System.Data.IDbTransaction transaction, bool hasVerifiedBackend, bool forceCompact, IBackendManager backendManager) { - // Workaround where we allow a running backendmanager to be used - using (var bk = sharedManager == null ? new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db) : null) + CancellationToken cancellationToken = CancellationToken.None; + if (!hasVerifiedBackend) + FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, true, transaction).Await(); + + IListResultFileset[] filesets = db.FilesetsWithBackupVersion.ToArray(); + List versionsToDelete = + [ + .. new SpecificVersionsRemover(this.m_options).GetFilesetsToDelete(filesets), + .. new KeepTimeRemover(this.m_options).GetFilesetsToDelete(filesets), + .. new RetentionPolicyRemover(this.m_options).GetFilesetsToDelete(filesets), + ]; + + // When determining the number of full versions to keep, we need to ignore the versions already marked for removal. + versionsToDelete.AddRange(new KeepVersionsRemover(this.m_options).GetFilesetsToDelete(filesets.Except(versionsToDelete))); + + if (!m_options.AllowFullRemoval && filesets.Length == versionsToDelete.Count) { - var backend = bk ?? sharedManager; - - if (!hasVerifiedBackend) - { - FilelistProcessor.VerifyRemoteList(backend, m_options, db, m_result.BackendWriter, true, transaction); - } - - IListResultFileset[] filesets = db.FilesetsWithBackupVersion.ToArray(); - List versionsToDelete = new List(); - versionsToDelete.AddRange(new SpecificVersionsRemover(this.m_options).GetFilesetsToDelete(filesets)); - versionsToDelete.AddRange(new KeepTimeRemover(this.m_options).GetFilesetsToDelete(filesets)); - versionsToDelete.AddRange(new RetentionPolicyRemover(this.m_options).GetFilesetsToDelete(filesets)); - - // When determining the number of full versions to keep, we need to ignore the versions already marked for removal. - versionsToDelete.AddRange(new KeepVersionsRemover(this.m_options).GetFilesetsToDelete(filesets.Except(versionsToDelete))); - - if (!m_options.AllowFullRemoval && filesets.Length == versionsToDelete.Count) - { - Logging.Log.WriteInformationMessage(LOGTAG, "PreventingLastFilesetRemoval", "Preventing removal of last fileset, use --{0} to allow removal ...", "allow-full-removal"); - versionsToDelete = versionsToDelete.OrderBy(x => x.Version).Skip(1).ToList(); - } - - if (versionsToDelete.Count > 0) - Logging.Log.WriteInformationMessage(LOGTAG, "DeleteRemoteFileset", "Deleting {0} remote fileset(s) ...", versionsToDelete.Count); - - var lst = db.DropFilesetsFromTable(versionsToDelete.Select(x => x.Time).ToArray(), transaction).ToArray(); - foreach (var f in lst) - db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, transaction); - - if (!m_options.Dryrun) - { - transaction.Commit(); - transaction = db.BeginTransaction(); - } - - foreach (var f in lst) - { - if (!m_result.TaskControl.ProgressRendevouz().Await()) - { - backend.WaitForComplete(db, transaction); - return; - } - - if (!m_options.Dryrun) - backend.Delete(f.Key, f.Value); - else - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteRemoteFileset", "Would delete remote fileset: {0}", f.Key); - } - - if (sharedManager == null) - backend.WaitForComplete(db, transaction); - else - backend.WaitForEmpty(db, transaction); - - var count = lst.Length; - if (!m_options.Dryrun) - { - if (count == 0) - Logging.Log.WriteInformationMessage(LOGTAG, "DeleteResults", "No remote filesets were deleted"); - else - Logging.Log.WriteInformationMessage(LOGTAG, "DeleteResults", "Deleted {0} remote fileset(s)", count); - } - else - { - - if (count == 0) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteResults", "No remote filesets would be deleted"); - else - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteResults", "{0} remote fileset(s) would be deleted", count); - - if (count > 0 && m_options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteHelp", "Remove --dry-run to actually delete files"); - } - - if (!m_options.NoAutoCompact && (forceCompact || versionsToDelete.Count > 0)) - { - m_result.CompactResults = new CompactResults(m_result); - new CompactHandler(m_backendurl, m_options, (CompactResults)m_result.CompactResults).DoCompact(db, true, ref transaction, sharedManager); - } - - m_result.SetResults(versionsToDelete.Select(v => new Tuple(v.Version, v.Time)), m_options.Dryrun); + Logging.Log.WriteInformationMessage(LOGTAG, "PreventingLastFilesetRemoval", "Preventing removal of last fileset, use --{0} to allow removal ...", "allow-full-removal"); + versionsToDelete = versionsToDelete.OrderBy(x => x.Version).Skip(1).ToList(); } + + if (versionsToDelete.Count > 0) + Logging.Log.WriteInformationMessage(LOGTAG, "DeleteRemoteFileset", "Deleting {0} remote fileset(s) ...", versionsToDelete.Count); + + var lst = db.DropFilesetsFromTable(versionsToDelete.Select(x => x.Time).ToArray(), transaction).ToArray(); + foreach (var f in lst) + db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, transaction); + + if (!m_options.Dryrun) + { + transaction.Commit(); + transaction = db.BeginTransaction(); + } + + foreach (var f in lst) + { + if (!m_result.TaskControl.ProgressRendevouz().Await()) + { + backendManager.WaitForEmptyAsync(db, transaction, cancellationToken).Await(); + return; + } + + if (!m_options.Dryrun) + backendManager.DeleteAsync(f.Key, f.Value, false, cancellationToken).Await(); + else + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteRemoteFileset", "Would delete remote fileset: {0}", f.Key); + } + + backendManager.WaitForEmptyAsync(db, transaction, cancellationToken).Await(); + + var count = lst.Length; + if (!m_options.Dryrun) + { + if (count == 0) + Logging.Log.WriteInformationMessage(LOGTAG, "DeleteResults", "No remote filesets were deleted"); + else + Logging.Log.WriteInformationMessage(LOGTAG, "DeleteResults", "Deleted {0} remote fileset(s)", count); + } + else + { + + if (count == 0) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteResults", "No remote filesets would be deleted"); + else + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteResults", "{0} remote fileset(s) would be deleted", count); + + if (count > 0 && m_options.Dryrun) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteHelp", "Remove --dry-run to actually delete files"); + } + + if (!m_options.NoAutoCompact && (forceCompact || versionsToDelete.Count > 0)) + { + m_result.CompactResults = new CompactResults(m_result); + var (_, tr) = new CompactHandler(m_options, (CompactResults)m_result.CompactResults).DoCompact(db, true, transaction, backendManager).Await(); + transaction = tr; + } + + m_result.SetResults(versionsToDelete.Select(v => new Tuple(v.Version, v.Time)), m_options.Dryrun); } } diff --git a/Duplicati/Library/Main/Operation/FilelistProcessor.cs b/Duplicati/Library/Main/Operation/FilelistProcessor.cs index f6d8da99f..b3c6405c4 100644 --- a/Duplicati/Library/Main/Operation/FilelistProcessor.cs +++ b/Duplicati/Library/Main/Operation/FilelistProcessor.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -27,6 +27,7 @@ using System.Linq; using Duplicati.Library.Interface; using System.Threading; using Duplicati.Library.Utility; +using System.Threading.Tasks; namespace Duplicati.Library.Main.Operation { @@ -42,7 +43,7 @@ namespace Duplicati.Library.Main.Operation /// Throws an error if there are issues with the remote storage /// /// The database to compare with - public static void VerifyLocalList(BackendManager backend, LocalDatabase database) + public static async Task VerifyLocalList(IBackendManager backendManager, LocalDatabase database, CancellationToken cancellationToken) { var locallist = database.GetRemoteVolumes(); foreach (var i in locallist) @@ -60,7 +61,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteInformationMessage(LOGTAG, "RemovingStaleFile", "Removing remote file listed as {0}: {1}", i.State, i.Name); try { - backend.Delete(i.Name, i.Size, true); + await backendManager.DeleteAsync(i.Name, i.Size, true, cancellationToken).ConfigureAwait(false); } catch (Exception ex) { @@ -74,17 +75,17 @@ namespace Duplicati.Library.Main.Operation break; } - backend.FlushDbMessages(); + await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false); } } - public static void VerifyRemoteList(BackendManager backend, Options options, LocalDatabase database, IBackendWriter backendWriter, bool latestVolumesOnly, IDbTransaction transaction) + public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter backendWriter, bool latestVolumesOnly, IDbTransaction transaction) { if (!options.NoBackendverification) { LocalBackupDatabase backupDatabase = new LocalBackupDatabase(database, options); IEnumerable protectedFiles = backupDatabase.GetTemporaryFilelistVolumeNames(latestVolumesOnly, transaction); - FilelistProcessor.VerifyRemoteList(backend, options, database, backendWriter, protectedFiles); + await VerifyRemoteList(backend, options, database, backendWriter, protectedFiles).ConfigureAwait(false); } } @@ -98,9 +99,9 @@ namespace Duplicati.Library.Main.Operation /// The log instance to use /// Filenames that should be exempted from deletion /// Disable the logging of errors to prevent spamming the log; exceptions will be thrown regardless - public static void VerifyRemoteList(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log, IEnumerable protectedFiles = null, bool logErrors = true) + public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter log, IEnumerable protectedFiles = null, bool logErrors = true) { - var tp = RemoteListAnalysis(backend, options, database, log, protectedFiles); + var tp = await RemoteListAnalysis(backend, options, database, log, protectedFiles).ConfigureAwait(false); long extraCount = 0; long missingCount = 0; @@ -177,17 +178,15 @@ namespace Duplicati.Library.Main.Operation /// /// The backend url /// The options to use - /// The result writer /// The attached database /// An optional transaction object - public static void UploadVerificationFile(string backendurl, Options options, IBackendWriter result, LocalDatabase db, System.Data.IDbTransaction transaction) + public static async Task UploadVerificationFile(IBackendManager backendManager, Options options, LocalDatabase db, IDbTransaction transaction) { - using (var backend = new BackendManager(backendurl, options, result, db)) using (var tempfile = new Library.Utility.TempFile()) { var remotename = options.Prefix + "-verification.json"; - using (var stream = new System.IO.StreamWriter(tempfile, false, System.Text.Encoding.UTF8)) - FilelistProcessor.CreateVerificationFile(db, stream); + await using (var stream = new System.IO.StreamWriter(tempfile, false, System.Text.Encoding.UTF8)) + CreateVerificationFile(db, stream); if (options.Dryrun) { @@ -195,8 +194,8 @@ namespace Duplicati.Library.Main.Operation } else { - backend.PutUnencrypted(remotename, tempfile); - backend.WaitForComplete(db, transaction); + await backendManager.PutVerificationFileAsync(remotename, tempfile, CancellationToken.None).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, transaction, CancellationToken.None).ConfigureAwait(false); } } } @@ -209,9 +208,9 @@ namespace Duplicati.Library.Main.Operation /// The options used /// The database to compare with /// Filenames that should be exempted from deletion - public static RemoteAnalysisResult RemoteListAnalysis(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log, IEnumerable protectedFiles) + public static async Task RemoteListAnalysis(IBackendManager backendManager, Options options, LocalDatabase database, IBackendWriter log, IEnumerable protectedFiles) { - var rawlist = backend.List(); + var rawlist = await backendManager.ListAsync(CancellationToken.None).ConfigureAwait(false); var lookup = new Dictionary(); protectedFiles = protectedFiles ?? Enumerable.Empty(); @@ -243,7 +242,7 @@ namespace Duplicati.Library.Main.Operation log.BackupListCount = database.FilesetTimes.Count(); log.LastBackupDate = filesets.Count == 0 ? new DateTime(0) : filesets[0].Time.ToLocalTime(); - CheckQuota(backend, options, log, knownFileSize); + await CheckQuota(backendManager, options, log, knownFileSize).ConfigureAwait(false); foreach (var s in remotelist) lookup[s.File.Name] = s; @@ -282,7 +281,7 @@ namespace Duplicati.Library.Main.Operation if (remoteFound) { Logging.Log.WriteInformationMessage(LOGTAG, "RemoveUnwantedRemoteFile", "removing remote file listed as {0}: {1}", i.State, i.Name); - backend.Delete(i.Name, i.Size, true); + await backendManager.DeleteAsync(i.Name, i.Size, true, CancellationToken.None).ConfigureAwait(false); } else { @@ -333,7 +332,7 @@ namespace Duplicati.Library.Main.Operation else { Logging.Log.WriteInformationMessage(LOGTAG, "Remove incomplete file", "removing incomplete remote file listed as {0}: {1}", i.State, i.Name); - backend.Delete(i.Name, i.Size, true); + await backendManager.DeleteAsync(i.Name, i.Size, true, CancellationToken.None).ConfigureAwait(false); } } break; @@ -361,7 +360,7 @@ namespace Duplicati.Library.Main.Operation break; } - backend.FlushDbMessages(); + await backendManager.WaitForEmptyAsync(database, null, CancellationToken.None).ConfigureAwait(false); } // cleanup deleted volumes in DB en block @@ -380,12 +379,12 @@ namespace Duplicati.Library.Main.Operation }; } - private static void CheckQuota(BackendManager backend, Options options, IBackendWriter log, long knownFileSize) + private static async Task CheckQuota(IBackendManager backendManager, Options options, IBackendWriter log, long knownFileSize) { if (options.QuotaDisable) return; - var quota = backend.GetQuotaInfoAsync(CancellationToken.None).Await(); + var quota = await backendManager.GetQuotaInfoAsync(CancellationToken.None).ConfigureAwait(false); if (quota != null) { log.TotalQuotaSpace = quota.TotalQuotaSpace; diff --git a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs index 5781df668..4212d1bb4 100644 --- a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -24,6 +24,7 @@ using System.Collections; using System.Collections.Generic; using System.Linq; using Duplicati.Library.Interface; +using Duplicati.Library.Utility; namespace Duplicati.Library.Main.Operation { @@ -33,28 +34,26 @@ namespace Duplicati.Library.Main.Operation /// The tag used for logging /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(ListBrokenFilesHandler)); - protected readonly string m_backendurl; protected readonly Options m_options; protected readonly ListBrokenFilesResults m_result; - public ListBrokenFilesHandler(string backend, Options options, ListBrokenFilesResults result) + public ListBrokenFilesHandler(Options options, ListBrokenFilesResults result) { - m_backendurl = backend; m_options = options; m_result = result; } - public void Run(Library.Utility.IFilter filter, Func callbackhandler = null) + public void Run(IBackendManager backendManager, IFilter filter, Func callbackhandler = null) { if (!System.IO.File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); using (var db = new Database.LocalListBrokenFilesDatabase(m_options.Dbpath)) using (var tr = db.BeginTransaction()) - DoRun(db, tr, filter, callbackhandler); + DoRun(backendManager, db, tr, filter, callbackhandler); } - public static Tuple[] GetBrokenFilesetsFromRemote(string backendurl, BasicResults result, Database.LocalListBrokenFilesDatabase db, System.Data.IDbTransaction transaction, Options options, out List missing) + public static Tuple[] GetBrokenFilesetsFromRemote(IBackendManager backendManager, BasicResults result, Database.LocalListBrokenFilesDatabase db, System.Data.IDbTransaction transaction, Options options, out List missing) { missing = null; var brokensets = db.GetBrokenFilesets(options.Time, options.Version, transaction).ToArray(); @@ -66,35 +65,32 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesetsInDatabase", "No broken filesets found in database, checking for missing remote files"); - using (var backend = new BackendManager(backendurl, options, result.BackendWriter, db)) + var remotestate = FilelistProcessor.RemoteListAnalysis(backendManager, options, db, result.BackendWriter, null).Await(); + if (!remotestate.ParsedVolumes.Any()) + throw new UserInformationException("No remote volumes were found, refusing purge", "CannotPurgeWithNoRemoteVolumes"); + + missing = remotestate.MissingVolumes.ToList(); + if (missing.Count == 0) { - var remotestate = FilelistProcessor.RemoteListAnalysis(backend, options, db, result.BackendWriter, null); - if (!remotestate.ParsedVolumes.Any()) - throw new UserInformationException("No remote volumes were found, refusing purge", "CannotPurgeWithNoRemoteVolumes"); - - missing = remotestate.MissingVolumes.ToList(); - if (missing.Count == 0) - { - Logging.Log.WriteInformationMessage(LOGTAG, "NoMissingFilesFound", "Skipping operation because no files were found to be missing, and no filesets were recorded as broken."); - return null; - } - - // Mark all volumes as disposable - foreach (var f in missing) - db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, transaction); - - Logging.Log.WriteInformationMessage(LOGTAG, "MarkedRemoteFilesForDeletion", "Marked {0} remote files for deletion", missing.Count); - - // Drop all content from tables - db.RemoveMissingBlocks(missing.Select(x => x.Name), transaction); + Logging.Log.WriteInformationMessage(LOGTAG, "NoMissingFilesFound", "Skipping operation because no files were found to be missing, and no filesets were recorded as broken."); + return null; } + + // Mark all volumes as disposable + foreach (var f in missing) + db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, transaction); + + Logging.Log.WriteInformationMessage(LOGTAG, "MarkedRemoteFilesForDeletion", "Marked {0} remote files for deletion", missing.Count); + + // Drop all content from tables + db.RemoveMissingBlocks(missing.Select(x => x.Name), transaction); brokensets = db.GetBrokenFilesets(options.Time, options.Version, transaction).ToArray(); } return brokensets; } - private void DoRun(Database.LocalListBrokenFilesDatabase db, System.Data.IDbTransaction transaction, Library.Utility.IFilter filter, Func callbackhandler) + private void DoRun(IBackendManager backendManager, Database.LocalListBrokenFilesDatabase db, System.Data.IDbTransaction transaction, Library.Utility.IFilter filter, Func callbackhandler) { if (filter != null && !filter.Empty) throw new UserInformationException("Filters are not supported for this operation", "FiltersAreNotSupportedForListBrokenFiles"); @@ -103,7 +99,7 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException("The command does not work on partially recreated databases", "ListBrokenFilesDoesNotWorkOnPartialDatabase"); List missing; - var brokensets = GetBrokenFilesetsFromRemote(m_backendurl, m_result, db, transaction, m_options, out missing); + var brokensets = GetBrokenFilesetsFromRemote(backendManager, m_result, db, transaction, m_options, out missing); if (brokensets == null) return; @@ -123,8 +119,8 @@ namespace Duplicati.Library.Main.Operation var fstimes = db.FilesetTimes.ToList(); - var brokenfilesets = - brokensets.Select(x => new + var brokenfilesets = + brokensets.Select(x => new { Version = fstimes.FindIndex(y => y.Key == x.Item2), Timestamp = x.Item1, @@ -138,8 +134,8 @@ namespace Duplicati.Library.Main.Operation m_result.BrokenFiles = brokenfilesets.Select( x => new Tuple>>( - x.Version, - x.Timestamp, + x.Version, + x.Timestamp, callbackhandler == null && !m_options.ListSetsOnly ? db.GetBrokenFilenames(x.FilesetID, transaction).ToArray().AsEnumerable() : new MockList>((int)x.BrokenCount) @@ -169,7 +165,7 @@ namespace Duplicati.Library.Main.Operation public int Count { get; private set; } - public bool IsReadOnly { get { return true; }} + public bool IsReadOnly => true; public void Add(T item) { diff --git a/Duplicati/Library/Main/Operation/ListChangesHandler.cs b/Duplicati/Library/Main/Operation/ListChangesHandler.cs index 10634cfe9..037532e38 100644 --- a/Duplicati/Library/Main/Operation/ListChangesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListChangesHandler.cs @@ -1,27 +1,28 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; using System.Linq; +using System.Threading; using Duplicati.Library.Interface; using Duplicati.Library.Utility; @@ -34,13 +35,11 @@ namespace Duplicati.Library.Main.Operation /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(ListChangesHandler)); - private readonly string m_backendurl; private readonly Options m_options; private readonly ListChangesResults m_result; - public ListChangesHandler(string backend, Options options, ListChangesResults result) + public ListChangesHandler(Options options, ListChangesResults result) { - m_backendurl = backend; m_options = options; m_result = result; } @@ -65,8 +64,9 @@ namespace Duplicati.Library.Main.Operation return res; } - public void Run(string baseVersion, string compareVersion, IEnumerable filterstrings = null, Library.Utility.IFilter compositefilter = null, Action>> callback = null) + public void Run(string baseVersion, string compareVersion, IBackendManager backendManager, IEnumerable filterstrings, IFilter compositefilter, Action>> callback) { + var cancellationToken = CancellationToken.None; var filter = Library.Utility.JoinedFilterExpression.Join(new Library.Utility.FilterExpression(filterstrings), compositefilter); var useLocalDb = !m_options.NoLocalDb && System.IO.File.Exists(m_options.Dbpath); @@ -81,7 +81,6 @@ namespace Duplicati.Library.Main.Operation using (var tmpdb = useLocalDb ? null : new Library.Utility.TempFile()) using (var db = new Database.LocalListChangesDatabase(useLocalDb ? m_options.Dbpath : (string)tmpdb)) - using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db)) using (var storageKeeper = db.CreateStorageHelper()) { m_result.SetDatabase(db); @@ -107,7 +106,7 @@ namespace Duplicati.Library.Main.Operation { Logging.Log.WriteInformationMessage(LOGTAG, "NoLocalDatabase", "No local database, accessing remote store"); - var parsedlist = (from n in backend.List() + var parsedlist = (from n in backendManager.ListAsync(cancellationToken).Await() let p = Volumes.VolumeBase.ParseFilename(n) where p != null && p.FileType == RemoteVolumeType.Files orderby p.Time descending @@ -124,37 +123,37 @@ namespace Duplicati.Library.Main.Operation numberedList.Remove(bt); SelectTime(compareVersion, numberedList, out compareVersionIndex, out compareVersionTime, out compareFile); - Func conv = (x) => + Func conv = (x) => { switch (x) { case FilelistEntryType.File: - return Library.Interface.ListChangesElementType.File; + return ListChangesElementType.File; case FilelistEntryType.Folder: - return Library.Interface.ListChangesElementType.Folder; + return ListChangesElementType.Folder; case FilelistEntryType.Symlink: - return Library.Interface.ListChangesElementType.Symlink; + return ListChangesElementType.Symlink; default: - return (Library.Interface.ListChangesElementType)(-1); + return (ListChangesElementType)(-1); } }; if (!m_result.TaskControl.ProgressRendevouz().Await()) return; - using (var tmpfile = backend.Get(baseFile.File.Name, baseFile.File.Size, null)) + using (var tmpfile = backendManager.GetAsync(baseFile.File.Name, null, baseFile.File.Size, cancellationToken).Await()) using (var rd = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(baseFile.File.Name), tmpfile, m_options)) foreach (var f in rd.Files) - if (Library.Utility.FilterExpression.Matches(filter, f.Path)) + if (FilterExpression.Matches(filter, f.Path)) storageKeeper.AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), false); if (!m_result.TaskControl.ProgressRendevouz().Await()) return; - using (var tmpfile = backend.Get(compareFile.File.Name, compareFile.File.Size, null)) + using (var tmpfile = backendManager.GetAsync(compareFile.File.Name, null, compareFile.File.Size, cancellationToken).Await()) using (var rd = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(compareFile.File.Name), tmpfile, m_options)) foreach (var f in rd.Files) - if (Library.Utility.FilterExpression.Matches(filter, f.Path)) + if (FilterExpression.Matches(filter, f.Path)) storageKeeper.AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), true); } diff --git a/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs b/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs index 49ab9e1a9..27aa80653 100644 --- a/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs @@ -1,58 +1,56 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; -using System.Linq; using System.Collections.Generic; using Duplicati.Library.Main.Database; using Duplicati.Library.Utility; +using System.Threading; namespace Duplicati.Library.Main.Operation { internal class ListControlFilesHandler { private readonly Options m_options; - private readonly string m_backendurl; private readonly ListResults m_result; - public ListControlFilesHandler(string backendurl, Options options, ListResults result) + public ListControlFilesHandler(Options options, ListResults result) { m_options = options; - m_backendurl = backendurl; m_result = result; } - public void Run(IEnumerable filterstrings = null, Library.Utility.IFilter compositefilter = null) + public void Run(IBackendManager backendManager, IEnumerable filterstrings, Library.Utility.IFilter compositefilter) { - using (var tmpdb = new Library.Utility.TempFile()) - using (var db = new Database.LocalDatabase(System.IO.File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "ListControlFiles", true)) - using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db)) + var cancellationToken = CancellationToken.None; + using (var tmpdb = new TempFile()) + using (var db = new LocalDatabase(System.IO.File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "ListControlFiles", true)) { m_result.SetDatabase(db); - var filter = Library.Utility.JoinedFilterExpression.Join(new Library.Utility.FilterExpression(filterstrings), compositefilter); + var filter = JoinedFilterExpression.Join(new Library.Utility.FilterExpression(filterstrings), compositefilter); try { - var filteredList = ListFilesHandler.ParseAndFilterFilesets(backend.List(), m_options); + var filteredList = ListFilesHandler.ParseAndFilterFilesets(backendManager.ListAsync(cancellationToken).Await(), m_options); if (filteredList.Count == 0) throw new Exception("No filesets found on remote target"); @@ -68,7 +66,7 @@ namespace Duplicati.Library.Main.Operation var entry = db.GetRemoteVolume(file.Name); var files = new List(); - using (var tmpfile = backend.Get(file.Name, entry.Size < 0 ? file.Size : entry.Size, entry.Hash)) + using (var tmpfile = backendManager.GetAsync(file.Name, entry.Hash, entry.Size < 0 ? file.Size : entry.Size, cancellationToken).Await()) using (var tmp = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(file.Name), tmpfile, m_options)) foreach (var cf in tmp.ControlFiles) if (Library.Utility.FilterExpression.Matches(filter, cf.Key)) @@ -90,7 +88,7 @@ namespace Duplicati.Library.Main.Operation } finally { - backend.WaitForComplete(db, null); + backendManager.WaitForEmptyAsync(db, null, cancellationToken).Await(); } } } diff --git a/Duplicati/Library/Main/Operation/ListFilesHandler.cs b/Duplicati/Library/Main/Operation/ListFilesHandler.cs index b9506867a..dfdc414ed 100644 --- a/Duplicati/Library/Main/Operation/ListFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFilesHandler.cs @@ -1,27 +1,27 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System; using System.Collections.Generic; using System.Linq; -using System.Text; +using System.Threading; +using System.Threading.Tasks; using Duplicati.Library.Interface; using Duplicati.Library.Main.Database; using Duplicati.Library.Main.Volumes; @@ -36,22 +36,21 @@ namespace Duplicati.Library.Main.Operation /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(); - private readonly string m_backendurl; private readonly Options m_options; private readonly ListResults m_result; - public ListFilesHandler(string backend, Options options, ListResults result) + public ListFilesHandler(Options options, ListResults result) { - m_backendurl = backend; m_options = options; m_result = result; } - public void Run(IEnumerable filterstrings = null, Library.Utility.IFilter compositefilter = null) + public async Task Run(IBackendManager backendManager, IEnumerable filterstrings, IFilter compositefilter) { - var parsedfilter = new Library.Utility.FilterExpression(filterstrings); - var filter = Library.Utility.JoinedFilterExpression.Join(parsedfilter, compositefilter); - var simpleList = !((filter is FilterExpression expression && expression.Type == Library.Utility.FilterType.Simple) || m_options.AllVersions); + var cancellationToken = m_result.TaskControl.ProgressToken; + var parsedfilter = new FilterExpression(filterstrings); + var filter = JoinedFilterExpression.Join(parsedfilter, compositefilter); + var simpleList = !((filter is FilterExpression expression && expression.Type == FilterType.Simple) || m_options.AllVersions); //Use a speedy local query if (!m_options.NoLocalDb && System.IO.File.Exists(m_options.Dbpath)) @@ -120,17 +119,16 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException("Listing prefixes is not supported without a local database, consider using the \"repair\" option to rebuild the database.", "PrefixListingRequiresLocalDatabase"); // Otherwise, grab info from remote location - using (var tmpdb = new Library.Utility.TempFile()) - using (var db = new Database.LocalDatabase(tmpdb, "List", true)) - using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db)) + using (var tmpdb = new TempFile()) + using (var db = new LocalDatabase(tmpdb, "List", true)) { m_result.SetDatabase(db); - var filteredList = ParseAndFilterFilesets(backend.List(), m_options); + var filteredList = ParseAndFilterFilesets(await backendManager.ListAsync(cancellationToken).ConfigureAwait(false), m_options); if (filteredList.Count == 0) throw new UserInformationException("No filesets found on remote target", "EmptyRemoteFolder"); - var numberSeq = CreateResultSequence(filteredList, backend, m_options); + var numberSeq = await CreateResultSequence(filteredList, backendManager, m_options, cancellationToken); if (filter.Empty) { m_result.SetResult(numberSeq, null); @@ -142,11 +140,11 @@ namespace Duplicati.Library.Main.Operation filteredList.RemoveAt(0); Dictionary> res; - if (!m_result.TaskControl.ProgressRendevouz().Await()) + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) return; - using (var tmpfile = backend.Get(firstEntry.File.Name, firstEntry.File.Size, null)) - using (var rd = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(firstEntry.File.Name), tmpfile, m_options)) + using (var tmpfile = await backendManager.GetAsync(firstEntry.File.Name, null, firstEntry.File.Size, cancellationToken).ConfigureAwait(false)) + using (var rd = new FilesetVolumeReader(RestoreHandler.GetCompressionModule(firstEntry.File.Name), tmpfile, m_options)) if (simpleList) { m_result.SetResult( @@ -177,11 +175,14 @@ namespace Duplicati.Library.Main.Operation } long flindex = 1; - foreach (var flentry in filteredList) - using (var tmpfile = backend.Get(flentry.Value.File.Name, flentry.Value.File == null ? -1 : flentry.Value.File.Size, null)) - using (var rd = new Volumes.FilesetVolumeReader(flentry.Value.CompressionModule, tmpfile, m_options)) + var filteredListMap = filteredList.ToDictionary(x => x.Value.File.Name, x => x.Value); + await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(filteredList.Select(x => new RemoteVolumeMapper(x.Value)), cancellationToken)) + { + var flentry = filteredListMap[name]; + using (tmpfile) + using (var rd = new FilesetVolumeReader(flentry.CompressionModule, tmpfile, m_options)) { - if (!m_result.TaskControl.ProgressRendevouz().Await()) + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) return; foreach (var p in from n in rd.Files where Library.Utility.FilterExpression.Matches(filter, n.Path) select n) @@ -203,6 +204,7 @@ namespace Duplicati.Library.Main.Operation flindex++; } + } m_result.SetResult( numberSeq, @@ -224,22 +226,30 @@ namespace Duplicati.Library.Main.Operation return filelistFilter(parsedlist).ToList(); } - private static IEnumerable CreateResultSequence(IEnumerable> filteredList, BackendManager backendManager, Options options) + private class RemoteVolumeMapper(IParsedVolume Volume) : IRemoteVolume { - List list = new List(); - foreach (KeyValuePair entry in filteredList) + public string Name => Volume.File.Name; + public string Hash => null; + public long Size => Volume.File.Size; + } + + private static async Task> CreateResultSequence(IEnumerable> filteredList, IBackendManager backendManager, Options options, CancellationToken cancelToken) + { + var list = new List(); + var map = filteredList.ToDictionary(x => x.Value.File.Name, x => x); + await foreach (var (file, hash, size, name) in backendManager.GetFilesOverlappedAsync(filteredList.Select(x => new RemoteVolumeMapper(x.Value)), cancelToken)) { - AsyncDownloader downloader = new AsyncDownloader(new IRemoteVolume[] { new RemoteVolume(entry.Value.File) }, backendManager); - foreach (IAsyncDownloadedFile file in downloader) + // We must obtain the partial/full status from the fileset file in the dlist files. + // Without this, the restore dialog will show all versions as full, or all versions + // as partial. While the dlist files are already downloaded elsewhere, doing so again + // here is the most direct way to obtain the partial/full status without a major + // refactoring. Since restoring directly from the backend files should be a relatively + // rare event, we can work on improving the performance later. + using (file) { - // We must obtain the partial/full status from the fileset file in the dlist files. - // Without this, the restore dialog will show all versions as full, or all versions - // as partial. While the dlist files are already downloaded elsewhere, doing so again - // here is the most direct way to obtain the partial/full status without a major - // refactoring. Since restoring directly from the backend files should be a relatively - // rare event, we can work on improving the performance later. - VolumeBase.FilesetData filesetData = VolumeReaderBase.GetFilesetData(entry.Value.CompressionModule, file.TempFile, options); - list.Add(new ListResultFileset(entry.Key, filesetData.IsFullBackup ? BackupType.FULL_BACKUP : BackupType.PARTIAL_BACKUP, entry.Value.Time.ToLocalTime(), -1, -1)); + var ent = map[name]; + var filesetData = VolumeReaderBase.GetFilesetData(ent.Value.CompressionModule, file, options); + list.Add(new ListResultFileset(ent.Key, filesetData.IsFullBackup ? BackupType.FULL_BACKUP : BackupType.PARTIAL_BACKUP, ent.Value.Time.ToLocalTime(), -1, -1)); } } diff --git a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs index 3bbf5ec5a..f4487c6d8 100644 --- a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs @@ -32,18 +32,16 @@ namespace Duplicati.Library.Main.Operation /// The tag used for logging /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(PurgeBrokenFilesHandler)); - protected readonly string m_backendurl; protected readonly Options m_options; protected readonly PurgeBrokenFilesResults m_result; - public PurgeBrokenFilesHandler(string backend, Options options, PurgeBrokenFilesResults result) + public PurgeBrokenFilesHandler(Options options, PurgeBrokenFilesResults result) { - m_backendurl = backend; m_options = options; m_result = result; } - public void Run(Library.Utility.IFilter filter) + public void Run(IBackendManager backendManager, Library.Utility.IFilter filter) { if (!System.IO.File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); @@ -52,17 +50,17 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException("Filters are not supported for this operation", "FiltersNotAllowedOnPurgeBrokenFiles"); List missing = null; - + using (var db = new Database.LocalListBrokenFilesDatabase(m_options.Dbpath)) using (var tr = db.BeginTransaction()) { if (db.PartiallyRecreated) throw new UserInformationException("The command does not work on partially recreated databases", "CannotPurgeOnPartialDatabase"); - var sets = ListBrokenFilesHandler.GetBrokenFilesetsFromRemote(m_backendurl, m_result, db, tr, m_options, out missing); + var sets = ListBrokenFilesHandler.GetBrokenFilesetsFromRemote(backendManager, m_result, db, tr, m_options, out missing); if (sets == null) return; - + if (sets.Length == 0) { if (missing == null) @@ -73,7 +71,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenSetsButMissingRemoteFiles", string.Format("Found no broken filesets, but {0} missing remote files. Purging from database.", missing.Count)); } else - { + { Logging.Log.WriteInformationMessage(LOGTAG, "FoundBrokenFilesets", "Found {0} broken filesets with {1} affected files, purging files", sets.Length, sets.Sum(x => x.Item3)); var pgoffset = 0.0f; @@ -116,8 +114,8 @@ namespace Duplicati.Library.Main.Operation opts.RawOptions.Remove("time"); opts.RawOptions["no-auto-compact"] = "true"; - new DeleteHandler(m_backendurl, opts, (DeleteResults)m_result.DeleteResults) - .DoRun(rmdb, ref deltr, true, false, null); + new DeleteHandler(opts, (DeleteResults)m_result.DeleteResults) + .DoRun(rmdb, ref deltr, true, false, backendManager); if (!m_options.Dryrun) { @@ -163,7 +161,7 @@ namespace Duplicati.Library.Main.Operation opts.RawOptions.Remove("time"); opts.RawOptions["no-auto-compact"] = "true"; - new PurgeFilesHandler(m_backendurl, opts, (PurgeFilesResults)m_result.PurgeResults).Run(pgdb, pgoffset, pgspan, (cmd, filesetid, tablename) => + new PurgeFilesHandler(opts, (PurgeFilesResults)m_result.PurgeResults).Run(backendManager, pgdb, pgoffset, pgspan, (cmd, filesetid, tablename) => { if (filesetid != bs.FilesetID) throw new Exception(string.Format("Unexpected filesetid: {0}, expected {1}", filesetid, bs.FilesetID)); @@ -180,7 +178,7 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(0.95f); if (!m_options.Dryrun && db.RepairInProgress) - { + { Logging.Log.WriteInformationMessage(LOGTAG, "ValidatingDatabase", "Database was previously marked as in-progress, checking if it is valid after purging files"); db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, null); Logging.Log.WriteInformationMessage(LOGTAG, "UpdatingDatabase", "Purge completed, and consistency checks completed, marking database as complete"); diff --git a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs index bb0e92a4e..a405d2a2b 100644 --- a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs @@ -1,29 +1,29 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; -using System.ComponentModel; using System.Linq; +using System.Threading; using Duplicati.Library.Interface; -using Duplicati.Library.Main.Database; +using Duplicati.Library.Utility; namespace Duplicati.Library.Main.Operation { @@ -33,18 +33,16 @@ namespace Duplicati.Library.Main.Operation /// The tag used for logging /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(); - protected readonly string m_backendurl; protected readonly Options m_options; protected readonly PurgeFilesResults m_result; - public PurgeFilesHandler(string backend, Options options, PurgeFilesResults result) + public PurgeFilesHandler(Options options, PurgeFilesResults result) { - m_backendurl = backend; m_options = options; m_result = result; } - public void Run(Library.Utility.IFilter filter) + public void Run(IBackendManager backendManager, Library.Utility.IFilter filter) { if (filter == null || filter.Empty) throw new UserInformationException("Cannot purge with an empty filter, as that would cause all files to be removed.\nTo remove an entire backup set, use the \"delete\" command.", "EmptyFilterPurgeNotAllowed"); @@ -53,214 +51,213 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); using (var db = new Database.LocalPurgeDatabase(m_options.Dbpath)) - DoRun(db, filter, null, 0, 1); + DoRun(backendManager, db, filter, null, 0, 1); } - public void Run(Database.LocalPurgeDatabase db, float pgoffset, float pgspan, Action filtercommand) + public void Run(IBackendManager backendManager, Database.LocalPurgeDatabase db, float pgoffset, float pgspan, Action filtercommand) { - DoRun(db, null, filtercommand, pgoffset, pgspan); + DoRun(backendManager, db, null, filtercommand, pgoffset, pgspan); } - private void DoRun(Database.LocalPurgeDatabase db, Library.Utility.IFilter filter, Action filtercommand, float pgoffset, float pgspan) + private void DoRun(IBackendManager backendManager, Database.LocalPurgeDatabase db, IFilter filter, Action filtercommand, float pgoffset, float pgspan) { + var cancellationToken = CancellationToken.None; m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Begin); Logging.Log.WriteInformationMessage(LOGTAG, "StartingPurge", "Starting purge operation"); var doCompactStep = !m_options.NoAutoCompact && filtercommand == null; - using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db)) + if (db.PartiallyRecreated) + throw new UserInformationException("The purge command does not work on partially recreated databases", "PurgeNotAllowedOnPartialDatabase"); + + if (db.RepairInProgress && filtercommand == null) + throw new UserInformationException(string.Format("The purge command does not work on an incomplete database, try the {0} operation.", "purge-broken-files"), "PurgeNotAllowedOnIncompleteDatabase"); + + var versions = db.GetFilesetIDs(m_options.Time, m_options.Version).OrderByDescending(x => x).ToArray(); + if (versions.Length <= 0) + throw new UserInformationException("No filesets matched the supplied time or versions", "NoFilesetFoundForTimeOrVersion"); + + var orphans = db.CountOrphanFiles(null); + if (orphans != 0) + throw new UserInformationException(string.Format("Unable to start the purge process as there are {0} orphan file(s)", orphans), "CannotPurgeWithOrphans"); + + Utility.UpdateOptionsFromDb(db, m_options); + Utility.VerifyOptionsAndUpdateDatabase(db, m_options); + + if (filtercommand == null) { - if (db.PartiallyRecreated) - throw new UserInformationException("The purge command does not work on partially recreated databases", "PurgeNotAllowedOnPartialDatabase"); + db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, false, null); - if (db.RepairInProgress && filtercommand == null) - throw new UserInformationException(string.Format("The purge command does not work on an incomplete database, try the {0} operation.", "purge-broken-files"), "PurgeNotAllowedOnIncompleteDatabase"); + if (m_options.NoBackendverification) + FilelistProcessor.VerifyLocalList(backendManager, db, cancellationToken).Await(); + else + FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, null).Await(); + } - var versions = db.GetFilesetIDs(m_options.Time, m_options.Version).OrderByDescending(x => x).ToArray(); - if (versions.Length <= 0) - throw new UserInformationException("No filesets matched the supplied time or versions", "NoFilesetFoundForTimeOrVersion"); + var filesets = db.FilesetTimes.OrderByDescending(x => x.Value).ToArray(); - var orphans = db.CountOrphanFiles(null); - if (orphans != 0) - throw new UserInformationException(string.Format("Unable to start the purge process as there are {0} orphan file(s)", orphans), "CannotPurgeWithOrphans"); + var versionprogress = ((doCompactStep ? 0.75f : 1.0f) / versions.Length) * pgspan; + var currentprogress = pgoffset; + var progress = 0; - Utility.UpdateOptionsFromDb(db, m_options); - Utility.VerifyOptionsAndUpdateDatabase(db, m_options); + m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Process); + m_result.OperationProgressUpdater.UpdateProgress(currentprogress); - if (filtercommand == null) + // Reverse makes sure we re-write the old versions first + foreach (var versionid in versions.Reverse()) + { + progress++; + Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingFilelistVolumes", "Processing filelist volume {0} of {1}", progress, versions.Length); + + using (var tr = db.BeginTransaction()) { - db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, false, null); - - if (m_options.NoBackendverification) - FilelistProcessor.VerifyLocalList(backend, db); - else - FilelistProcessor.VerifyRemoteList(backend, m_options, db, m_result.BackendWriter, null); - } - - var filesets = db.FilesetTimes.OrderByDescending(x => x.Value).ToArray(); - - var versionprogress = ((doCompactStep ? 0.75f : 1.0f) / versions.Length) * pgspan; - var currentprogress = pgoffset; - var progress = 0; - - m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Process); - m_result.OperationProgressUpdater.UpdateProgress(currentprogress); - - // Reverse makes sure we re-write the old versions first - foreach (var versionid in versions.Reverse()) - { - progress++; - Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingFilelistVolumes", "Processing filelist volume {0} of {1}", progress, versions.Length); - - using (var tr = db.BeginTransaction()) - { - var ix = -1; - for (var i = 0; i < filesets.Length; i++) - if (filesets[i].Key == versionid) - { - ix = i; - break; - } - - if (ix < 0) - throw new InvalidProgramException(string.Format("Fileset was reported with id {0}, but could not be found?", versionid)); - - var secs = 0; - while (secs < 60) + var ix = -1; + for (var i = 0; i < filesets.Length; i++) + if (filesets[i].Key == versionid) { - secs++; - var tfn = Volumes.VolumeBase.GenerateFilename(RemoteVolumeType.Files, m_options, null, filesets[ix].Value.AddSeconds(secs)); - if (db.GetRemoteVolumeID(tfn, tr) < 0) - break; + ix = i; + break; } - var tsOriginal = filesets[ix].Value; - var ts = tsOriginal.AddSeconds(secs); + if (ix < 0) + throw new InvalidProgramException(string.Format("Fileset was reported with id {0}, but could not be found?", versionid)); - var prevfilename = db.GetRemoteVolumeNameForFileset(filesets[ix].Key, tr); + var secs = 0; + while (secs < 60) + { + secs++; + var tfn = Volumes.VolumeBase.GenerateFilename(RemoteVolumeType.Files, m_options, null, filesets[ix].Value.AddSeconds(secs)); + if (db.GetRemoteVolumeID(tfn, tr) < 0) + break; + } - if (secs >= 60) - throw new Exception(string.Format("Unable to create a new fileset for {0} because the resulting timestamp {1} is more than 60 seconds away", prevfilename, ts)); + var tsOriginal = filesets[ix].Value; + var ts = tsOriginal.AddSeconds(secs); - if (ix != 0 && filesets[ix - 1].Value <= ts) - throw new Exception(string.Format("Unable to create a new fileset for {0} because the resulting timestamp {1} is larger than the next timestamp {2}", prevfilename, ts, filesets[ix - 1].Value)); + var prevfilename = db.GetRemoteVolumeNameForFileset(filesets[ix].Key, tr); - using (var tempset = db.CreateTemporaryFileset(versionid, tr)) + if (secs >= 60) + throw new Exception(string.Format("Unable to create a new fileset for {0} because the resulting timestamp {1} is more than 60 seconds away", prevfilename, ts)); + + if (ix != 0 && filesets[ix - 1].Value <= ts) + throw new Exception(string.Format("Unable to create a new fileset for {0} because the resulting timestamp {1} is larger than the next timestamp {2}", prevfilename, ts, filesets[ix - 1].Value)); + + using (var tempset = db.CreateTemporaryFileset(versionid, tr)) + { + if (filtercommand == null) + tempset.ApplyFilter(filter); + else + tempset.ApplyFilter(filtercommand); + + if (tempset.RemovedFileCount == 0) { - if (filtercommand == null) - tempset.ApplyFilter(filter); - else - tempset.ApplyFilter(filtercommand); + Logging.Log.WriteInformationMessage(LOGTAG, "NotWritingNewFileset", "Not writing a new fileset for {0} as it was not changed", prevfilename); + currentprogress += versionprogress; + tr.Rollback(); + continue; + } + else + { + using (var tf = new Library.Utility.TempFile()) + using (var vol = new Volumes.FilesetVolumeWriter(m_options, ts)) + { + var isOriginalFilesetFullBackup = db.IsFilesetFullBackup(tsOriginal); + var newids = tempset.ConvertToPermanentFileset(vol.RemoteFilename, ts, isOriginalFilesetFullBackup); + vol.VolumeID = newids.Item1; + vol.CreateFilesetFile(isOriginalFilesetFullBackup); - if (tempset.RemovedFileCount == 0) - { - Logging.Log.WriteInformationMessage(LOGTAG, "NotWritingNewFileset", "Not writing a new fileset for {0} as it was not changed", prevfilename); - currentprogress += versionprogress; - tr.Rollback(); - continue; - } - else - { - using (var tf = new Library.Utility.TempFile()) - using (var vol = new Volumes.FilesetVolumeWriter(m_options, ts)) + Logging.Log.WriteInformationMessage(LOGTAG, "ReplacingFileset", "Replacing fileset {0} with {1} which has with {2} fewer file(s) ({3} reduction)", prevfilename, vol.RemoteFilename, tempset.RemovedFileCount, Library.Utility.Utility.FormatSizeString(tempset.RemovedFileSize)); + + db.WriteFileset(vol, newids.Item2, tr); + + m_result.RemovedFileSize += tempset.RemovedFileSize; + m_result.RemovedFileCount += tempset.RemovedFileCount; + m_result.RewrittenFileLists++; + + currentprogress += (versionprogress / 2); + m_result.OperationProgressUpdater.UpdateProgress(currentprogress); + + if (m_options.Dryrun || m_options.FullResult) { - var isOriginalFilesetFullBackup = db.IsFilesetFullBackup(tsOriginal); - var newids = tempset.ConvertToPermanentFileset(vol.RemoteFilename, ts, isOriginalFilesetFullBackup); - vol.VolumeID = newids.Item1; - vol.CreateFilesetFile(isOriginalFilesetFullBackup); - - Logging.Log.WriteInformationMessage(LOGTAG, "ReplacingFileset", "Replacing fileset {0} with {1} which has with {2} fewer file(s) ({3} reduction)", prevfilename, vol.RemoteFilename, tempset.RemovedFileCount, Library.Utility.Utility.FormatSizeString(tempset.RemovedFileSize)); - - db.WriteFileset(vol, newids.Item2, tr); - - m_result.RemovedFileSize += tempset.RemovedFileSize; - m_result.RemovedFileCount += tempset.RemovedFileCount; - m_result.RewrittenFileLists++; - - currentprogress += (versionprogress / 2); - m_result.OperationProgressUpdater.UpdateProgress(currentprogress); - - if (m_options.Dryrun || m_options.FullResult) + foreach (var fe in tempset.ListAllDeletedFiles()) { - foreach (var fe in tempset.ListAllDeletedFiles()) - { - var msg = string.Format(" Purging file {0} ({1})", fe.Key, Library.Utility.Utility.FormatSizeString(fe.Value)); + var msg = string.Format(" Purging file {0} ({1})", fe.Key, Library.Utility.Utility.FormatSizeString(fe.Value)); - Logging.Log.WriteProfilingMessage(LOGTAG, "PurgeFile", msg); - Logging.Log.WriteVerboseMessage(LOGTAG, "PurgeFile", msg); - - if (m_options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPurgeFile", msg); - } + Logging.Log.WriteProfilingMessage(LOGTAG, "PurgeFile", msg); + Logging.Log.WriteVerboseMessage(LOGTAG, "PurgeFile", msg); if (m_options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldWriteRemoteFiles", "Would write files to remote storage"); - - Logging.Log.WriteVerboseMessage(LOGTAG, "WritingRemoteFiles", "Writing files to remote storage"); + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPurgeFile", msg); } if (m_options.Dryrun) - { - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldUploadAndDelete", "Would upload file {0} ({1}) and delete file {2}, removing {3} files", vol.RemoteFilename, Library.Utility.Utility.FormatSizeString(vol.Filesize), prevfilename, tempset.RemovedFileCount); - tr.Rollback(); - } - else - { - var lst = db.DropFilesetsFromTable(new[] { filesets[ix].Value }, tr).ToArray(); - foreach (var f in lst) - db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, tr); + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldWriteRemoteFiles", "Would write files to remote storage"); - tr.Commit(); - backend.Put(vol, synchronous: true); - backend.Delete(prevfilename, -1, true); - backend.FlushDbMessages(); - } + Logging.Log.WriteVerboseMessage(LOGTAG, "WritingRemoteFiles", "Writing files to remote storage"); + } + + if (m_options.Dryrun) + { + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldUploadAndDelete", "Would upload file {0} ({1}) and delete file {2}, removing {3} files", vol.RemoteFilename, Library.Utility.Utility.FormatSizeString(vol.Filesize), prevfilename, tempset.RemovedFileCount); + tr.Rollback(); + } + else + { + var lst = db.DropFilesetsFromTable(new[] { filesets[ix].Value }, tr).ToArray(); + foreach (var f in lst) + db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, tr); + + tr.Commit(); + backendManager.PutAsync(vol, null, null, true, cancellationToken).Await(); + backendManager.DeleteAsync(prevfilename, -1, true, cancellationToken).Await(); + backendManager.WaitForEmptyAsync(db, null, cancellationToken).Await(); } } } } - - currentprogress += (versionprogress / 2); - m_result.OperationProgressUpdater.UpdateProgress(currentprogress); } + currentprogress += (versionprogress / 2); + m_result.OperationProgressUpdater.UpdateProgress(currentprogress); + } - if (doCompactStep) + + if (doCompactStep) + { + if (m_result.RewrittenFileLists == 0) { - if (m_result.RewrittenFileLists == 0) + Logging.Log.WriteInformationMessage(LOGTAG, "SkippingCompacting", "Skipping compacting as no new volumes were written"); + } + else + { + m_result.OperationProgressUpdater.UpdateProgress(pgoffset + (0.75f * pgspan)); + m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Compact); + m_result.CompactResults = new CompactResults(m_result); + using (var cdb = new Database.LocalDeleteDatabase(db)) { - Logging.Log.WriteInformationMessage(LOGTAG, "SkippingCompacting", "Skipping compacting as no new volumes were written"); - } - else - { - m_result.OperationProgressUpdater.UpdateProgress(pgoffset + (0.75f * pgspan)); - m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Compact); - m_result.CompactResults = new CompactResults(m_result); - using (var cdb = new Database.LocalDeleteDatabase(db)) + var tr = cdb.BeginTransaction(); + try { - var tr = cdb.BeginTransaction(); - try - { - new CompactHandler(backend.BackendUrl, m_options, (CompactResults)m_result.CompactResults).DoCompact(cdb, true, ref tr, backend); - } - catch - { - try { tr.Rollback(); } - catch { } - } - finally - { - try { tr.Commit(); } - catch { } - } + var (_, tr2) = new CompactHandler(m_options, (CompactResults)m_result.CompactResults).DoCompact(cdb, true, tr, backendManager).Await(); + tr2 = tr; + } + catch + { + try { tr.Rollback(); } + catch { } + } + finally + { + try { tr.Commit(); } + catch { } } } - - m_result.OperationProgressUpdater.UpdateProgress(pgoffset + pgspan); - m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Complete); } - backend.WaitForComplete(db, null); + m_result.OperationProgressUpdater.UpdateProgress(pgoffset + pgspan); + m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Complete); + + backendManager.WaitForEmptyAsync(db, null, cancellationToken).Await(); } } } diff --git a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs index 16a6e8c8f..ebfc4f2e8 100644 --- a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs +++ b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs @@ -21,6 +21,8 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading; +using System.Threading.Tasks; using Duplicati.Library.Interface; using Duplicati.Library.Main.Database; using Duplicati.Library.Main.Volumes; @@ -35,17 +37,15 @@ namespace Duplicati.Library.Main.Operation /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(); - private readonly string m_backendurl; private readonly Options m_options; private readonly RecreateDatabaseResults m_result; public delegate IEnumerable> NumberedFilterFilelistDelegate(IEnumerable filelist); public delegate void BlockVolumePostProcessor(string volumename, BlockVolumeReader reader); - public RecreateDatabaseHandler(string backendurl, Options options, RecreateDatabaseResults result) + public RecreateDatabaseHandler(Options options, RecreateDatabaseResults result) { m_options = options; - m_backendurl = backendurl; m_result = result; } @@ -53,10 +53,11 @@ namespace Duplicati.Library.Main.Operation /// Run the recreate procedure /// /// Path to the database that will be created + /// The backend manager to use for downloading files /// A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist /// Filters the files in a filelist to prevent downloading unwanted data /// A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists - public void Run(string path, Library.Utility.IFilter filter = null, NumberedFilterFilelistDelegate filelistfilter = null, BlockVolumePostProcessor blockprocessor = null) + public void Run(string path, IBackendManager backendManager, IFilter filter, NumberedFilterFilelistDelegate filelistfilter, BlockVolumePostProcessor blockprocessor) { if (System.IO.File.Exists(path)) throw new UserInformationException(string.Format("Cannot recreate database because file already exists: {0}", path), "RecreateTargetDatabaseExists"); @@ -64,7 +65,7 @@ namespace Duplicati.Library.Main.Operation using (var db = new LocalDatabase(path, "Recreate", true)) { m_result.SetDatabase(db); - DoRun(db, false, filter, filelistfilter, blockprocessor); + DoRun(backendManager, db, false, filter, filelistfilter, blockprocessor).Await(); db.WriteResults(); } } @@ -75,7 +76,7 @@ namespace Duplicati.Library.Main.Operation /// A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist /// Filters the files in a filelist to prevent downloading unwanted data /// A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists - public void RunUpdate(Library.Utility.IFilter filter = null, NumberedFilterFilelistDelegate filelistfilter = null, BlockVolumePostProcessor blockprocessor = null) + public void RunUpdate(IBackendManager backendManager, Library.Utility.IFilter filter, NumberedFilterFilelistDelegate filelistfilter, BlockVolumePostProcessor blockprocessor) { if (!m_options.RepairOnlyPaths) throw new UserInformationException(string.Format("Can only update with paths, try setting --{0}", "repair-only-paths"), "RepairUpdateRequiresPathsOnly"); @@ -97,7 +98,7 @@ namespace Duplicati.Library.Main.Operation if (preexistingOptionsInDatabase) Utility.VerifyOptionsAndUpdateDatabase(db, m_options, null); - DoRun(db, true, filter, filelistfilter, blockprocessor); + DoRun(backendManager, db, true, filter, filelistfilter, blockprocessor).Await(); db.WriteResults(); } } @@ -110,13 +111,13 @@ namespace Duplicati.Library.Main.Operation /// A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist /// Filters the files in a filelist to prevent downloading unwanted data /// A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists - internal void DoRun(LocalDatabase dbparent, bool updating, Library.Utility.IFilter filter = null, NumberedFilterFilelistDelegate filelistfilter = null, BlockVolumePostProcessor blockprocessor = null) + internal async Task DoRun(IBackendManager backendManager, LocalDatabase dbparent, bool updating, IFilter filter = null, NumberedFilterFilelistDelegate filelistfilter = null, BlockVolumePostProcessor blockprocessor = null) { + var cancellationToken = CancellationToken.None; m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Recreate_Running); //We build a local database in steps. using (var restoredb = new LocalRecreateDatabase(dbparent, m_options)) - using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, restoredb)) { restoredb.RepairInProgress = true; var expRecreateDb = false; // experimental recreate db code flag @@ -127,7 +128,7 @@ namespace Duplicati.Library.Main.Operation expRecreateDb = true; } - var rawlist = backend.List(); + var rawlist = await backendManager.ListAsync(cancellationToken); //First step is to examine the remote storage to see what // kind of data we can find @@ -142,7 +143,7 @@ namespace Duplicati.Library.Main.Operation if (remotefiles.Length == 0) { - if (rawlist.Count == 0) + if (rawlist.Count() == 0) throw new UserInformationException("No files were found at the remote location, perhaps the target url is incorrect?", "EmptyRemoteLocation"); else { @@ -155,11 +156,11 @@ namespace Duplicati.Library.Main.Operation var types = tmp.Distinct().ToArray(); if (tmp.Length == 0) - throw new UserInformationException(string.Format("Found {0} files at the remote storage, but none that could be parsed", rawlist.Count), "EmptyRemoteLocation"); + throw new UserInformationException(string.Format("Found {0} files at the remote storage, but none that could be parsed", rawlist.Count()), "EmptyRemoteLocation"); else if (types.Length == 1) throw new UserInformationException(string.Format("Found {0} parse-able files with the prefix {1}, did you forget to set the backup prefix?", tmp.Length, types[0]), "EmptyRemoteLocationWithPrefix"); else - throw new UserInformationException(string.Format("Found {0} parse-able files (of {1} files) with different prefixes: {2}, did you forget to set the backup prefix?", tmp.Length, rawlist.Count, string.Join(", ", types)), "EmptyRemoteLocationWithPrefix"); + throw new UserInformationException(string.Format("Found {0} parse-able files (of {1} files) with different prefixes: {2}, did you forget to set the backup prefix?", tmp.Length, rawlist.Count(), string.Join(", ", types)), "EmptyRemoteLocationWithPrefix"); } } @@ -206,12 +207,14 @@ namespace Duplicati.Library.Main.Operation var blocksize = m_options.Blocksize; var hashes_pr_block = blocksize / m_options.BlockhashSize; - foreach (var entry in new AsyncDownloader(filelistWork, backend)) + await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(filelistWork, m_result.TaskControl.ProgressToken)) + { + var entry = new RemoteVolume(name, hash, size); try { - if (!m_result.TaskControl.ProgressRendevouz().Await()) + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - backend.WaitForComplete(restoredb, null); + await backendManager.WaitForEmptyAsync(restoredb, tr, cancellationToken).ConfigureAwait(false); m_result.EndTime = DateTime.UtcNow; return; } @@ -227,12 +230,12 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingFilelistVolumes", "Processing filelist volume {0} of {1}", progress, filelistWork.Count); } - using (var tmpfile = entry.TempFile) + using (tmpfile) { isFirstFilelist = false; - if (entry.Hash != null && entry.Size > 0) - restoredb.UpdateRemoteVolume(entry.Name, RemoteVolumeState.Verified, entry.Size, entry.Hash, tr); + if (!string.IsNullOrWhiteSpace(hash) && size > 0) + restoredb.UpdateRemoteVolume(entry.Name, RemoteVolumeState.Verified, size, hash, tr); var parsed = VolumeBase.ParseFilename(entry.Name); @@ -288,7 +291,7 @@ namespace Duplicati.Library.Main.Operation restoredb.AddSmallBlocksetLink(fe.Hash, fe.Blockhash, fe.Blocksize, tr); else if (m_options.BlockHashAlgorithm == m_options.FileHashAlgorithm) restoredb.AddSmallBlocksetLink(fe.Hash, fe.Hash, fe.Size, tr); - else + else if (fe.Size > 0) Logging.Log.WriteWarningMessage(LOGTAG, "MissingBlockHash", null, "No block hash found for file: {0}", fe.Path); } @@ -338,6 +341,7 @@ namespace Duplicati.Library.Main.Operation throw; } + } //Make sure we write the config if it has been read from a manifest if (hasUpdatedOptions) @@ -368,12 +372,13 @@ namespace Duplicati.Library.Main.Operation var progress = 0; - foreach (var sf in new AsyncDownloader(indexfiles, backend)) + await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(indexfiles, m_result.TaskControl.ProgressToken)) + { try { - if (!m_result.TaskControl.ProgressRendevouz().Await()) + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - backend.WaitForComplete(restoredb, null); + await backendManager.WaitForEmptyAsync(restoredb, tr, cancellationToken).ConfigureAwait(false); m_result.EndTime = DateTime.UtcNow; return; } @@ -382,12 +387,12 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress((((float)progress / indexfiles.Count) * 0.5f) + 0.2f); Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingIndexlistVolumes", "Processing indexlist volume {0} of {1}", progress, indexfiles.Count); - using (var tmpfile = sf.TempFile) + using (tmpfile) { - if (sf.Hash != null && sf.Size > 0) - restoredb.UpdateRemoteVolume(sf.Name, RemoteVolumeState.Verified, sf.Size, sf.Hash, tr); + if (!string.IsNullOrWhiteSpace(hash) && size > 0) + restoredb.UpdateRemoteVolume(name, RemoteVolumeState.Verified, size, hash, tr); - using (var svr = new IndexVolumeReader(RestoreHandler.GetCompressionModule(sf.Name), tmpfile, m_options, hashsize)) + using (var svr = new IndexVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options, hashsize)) { foreach (var a in svr.Volumes) { @@ -405,7 +410,7 @@ namespace Duplicati.Library.Main.Operation var p = VolumeBase.ParseFilename(filename); if (p == null) throw new Exception(string.Format("Unable to parse filename: {0}", filename)); - Logging.Log.WriteWarningMessage(LOGTAG, "MissingFileDetected", null, "Remote file referenced as {0} by {1}, but not found in list, registering a missing remote file", filename, sf.Name); + Logging.Log.WriteWarningMessage(LOGTAG, "MissingFileDetected", null, "Remote file referenced as {0} by {1}, but not found in list, registering a missing remote file", filename, name); missing = true; volumeID = restoredb.RegisterRemoteVolume(filename, p.FileType, RemoteVolumeState.Temporary, tr); } @@ -416,7 +421,7 @@ namespace Duplicati.Library.Main.Operation restoredb.UpdateBlock(b.Key, b.Value, volumeID, tr, ref anyChange); restoredb.UpdateRemoteVolume(filename, missing ? RemoteVolumeState.Temporary : RemoteVolumeState.Verified, a.Length, a.Hash, tr); - restoredb.AddIndexBlockLink(restoredb.GetRemoteVolumeID(sf.Name), volumeID, tr); + restoredb.AddIndexBlockLink(restoredb.GetRemoteVolumeID(name), volumeID, tr); } //If there are blocklists in the index file, add them to the temp blocklist hashes table @@ -434,13 +439,13 @@ namespace Duplicati.Library.Main.Operation } catch (System.IO.InvalidDataException e) { - Logging.Log.WriteVerboseMessage(LOGTAG, "InvalidDataBlocklist", e, "Exception while processing blocklists in {0}", sf.Name); + Logging.Log.WriteVerboseMessage(LOGTAG, "InvalidDataBlocklist", e, "Exception while processing blocklists in {0}", name); ++wrongHashes; } } if (wrongHashes != 0) { - Logging.Log.WriteWarningMessage(LOGTAG, "WrongBlocklistHashes", null, "{0} had invalid blocklists which could not be used. Consider deleting this index file and run repair to recreate it.", sf.Name); + Logging.Log.WriteWarningMessage(LOGTAG, "WrongBlocklistHashes", null, "{0} had invalid blocklists which could not be used. Consider deleting this index file and run repair to recreate it.", name); } } } @@ -448,7 +453,7 @@ namespace Duplicati.Library.Main.Operation catch (Exception ex) { //Not fatal - Logging.Log.WriteErrorMessage(LOGTAG, "IndexFileProcessingFailed", ex, "Failed to process index file: {0}", sf.Name); + Logging.Log.WriteErrorMessage(LOGTAG, "IndexFileProcessingFailed", ex, "Failed to process index file: {0}", name); if (ex is System.Threading.ThreadAbortException) { m_result.EndTime = DateTime.UtcNow; @@ -458,6 +463,7 @@ namespace Duplicati.Library.Main.Operation if (m_options.UnittestMode) throw; } + } using (new Logging.Timer(LOGTAG, "CommitRecreateDb", "CommitRecreatedDb")) tr.Commit(); @@ -504,17 +510,17 @@ namespace Duplicati.Library.Main.Operation } var progress = 0; - foreach (var sf in new AsyncDownloader(lst, backend)) + await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(lst, m_result.TaskControl.ProgressToken)) { try { - using (var tmpfile = sf.TempFile) - using (var rd = new BlockVolumeReader(RestoreHandler.GetCompressionModule(sf.Name), tmpfile, m_options)) + using (tmpfile) + using (var rd = new BlockVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options)) using (var tr = restoredb.BeginTransaction()) { if (!m_result.TaskControl.ProgressRendevouz().Await()) { - backend.WaitForComplete(restoredb, null); + backendManager.WaitForEmptyAsync(restoredb, tr, cancellationToken).Await(); m_result.EndTime = DateTime.UtcNow; return; } @@ -523,9 +529,9 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress((((float)progress / lst.Count) * 0.1f) + 0.7f + (i * 0.1f)); Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingBlocklistVolumes", "Pass {0} of 3, processing blocklist volume {1} of {2}", (i + 1), progress, lst.Count); - var volumeid = restoredb.GetRemoteVolumeID(sf.Name); + var volumeid = restoredb.GetRemoteVolumeID(name); - restoredb.UpdateRemoteVolume(sf.Name, RemoteVolumeState.Uploaded, sf.Size, sf.Hash, tr); + restoredb.UpdateRemoteVolume(name, RemoteVolumeState.Uploaded, size, hash, tr); bool anyChange = false; // Update the block table so we know about the block/volume map @@ -552,7 +558,7 @@ namespace Duplicati.Library.Main.Operation { if (i == 2) { - Logging.Log.WriteWarningMessage(LOGTAG, "UpdatingTables", null, "Unexpected changes caused by block {0}", sf.Name); + Logging.Log.WriteWarningMessage(LOGTAG, "UpdatingTables", null, "Unexpected changes caused by block {0}", name); } if (expRecreateDb) restoredb.AddBlockAndBlockSetEntryFromTemp(hashsize, m_options.Blocksize, tr, false); @@ -565,12 +571,12 @@ namespace Duplicati.Library.Main.Operation //At this point we can patch files with data from the block volume if (blockprocessor != null) - blockprocessor(sf.Name, rd); + blockprocessor(name, rd); } } catch (Exception ex) { - Logging.Log.WriteWarningMessage(LOGTAG, "FailedRebuildingWithFile", ex, "Failed to use information from {0} to rebuild database: {1}", sf.Name, ex.Message); + Logging.Log.WriteWarningMessage(LOGTAG, "FailedRebuildingWithFile", ex, "Failed to use information from {0} to rebuild database: {1}", name, ex.Message); if (m_options.UnittestMode) throw; } @@ -578,7 +584,7 @@ namespace Duplicati.Library.Main.Operation } } - backend.WaitForComplete(restoredb, null); + backendManager.WaitForEmptyAsync(restoredb, null, cancellationToken).Await(); if (!m_options.RepairOnlyPaths) { diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 111dbe6a6..be4083989 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -1,28 +1,28 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; using System.Linq; -using System.Security.Cryptography; -using System.Text; +using System.Threading; +using System.Threading.Tasks; using Duplicati.Library.Interface; using Duplicati.Library.Main.Database; using Duplicati.Library.Main.Volumes; @@ -36,13 +36,11 @@ namespace Duplicati.Library.Main.Operation /// The tag used for logging /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(); - private readonly string m_backendurl; private readonly Options m_options; private readonly RepairResults m_result; - public RepairHandler(string backend, Options options, RepairResults result) + public RepairHandler(Options options, RepairResults result) { - m_backendurl = backend; m_options = options; m_result = result; @@ -50,11 +48,11 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException(Strings.Common.PassphraseChangeUnsupported, "PassphraseChangeUnsupported"); } - public void Run(Library.Utility.IFilter filter = null) + public void Run(IBackendManager backendManager, IFilter filter) { if (!System.IO.File.Exists(m_options.Dbpath)) { - RunRepairLocal(filter); + RunRepairLocal(backendManager, filter); RunRepairCommon(); m_result.EndTime = DateTime.UtcNow; return; @@ -88,20 +86,20 @@ namespace Duplicati.Library.Main.Operation System.IO.File.Move(m_options.Dbpath, baseName); } - RunRepairLocal(filter); + RunRepairLocal(backendManager, filter); RunRepairCommon(); } else { RunRepairCommon(); - RunRepairRemote(); + RunRepairRemote(backendManager, m_result.TaskControl.ProgressToken).Await(); } m_result.EndTime = DateTime.UtcNow; } - public void RunRepairLocal(Library.Utility.IFilter filter = null) + public void RunRepairLocal(IBackendManager backendManager, Library.Utility.IFilter filter) { m_result.RecreateDatabaseResults = new RecreateDatabaseResults(m_result); using (new Logging.Timer(LOGTAG, "RecreateDbForRepair", "Recreate database for repair")) @@ -112,12 +110,12 @@ namespace Duplicati.Library.Main.Operation var filelistfilter = RestoreHandler.FilterNumberedFilelist(m_options.Time, m_options.Version); - new RecreateDatabaseHandler(m_backendurl, m_options, (RecreateDatabaseResults)m_result.RecreateDatabaseResults) - .Run(m_options.Dryrun ? (string)f : m_options.Dbpath, filter, filelistfilter); + new RecreateDatabaseHandler(m_options, (RecreateDatabaseResults)m_result.RecreateDatabaseResults) + .Run(m_options.Dryrun ? (string)f : m_options.Dbpath, backendManager, filter, filelistfilter, null); } } - public void RunRepairRemote() + public async Task RunRepairRemote(IBackendManager backendManager, CancellationToken cancellationToken) { if (!System.IO.File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "RepairDatabaseFileDoesNotExist"); @@ -125,7 +123,6 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(0); using (var db = new LocalRepairDatabase(m_options.Dbpath)) - using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db)) { m_result.SetDatabase(db); Utility.UpdateOptionsFromDb(db, m_options); @@ -137,7 +134,7 @@ namespace Duplicati.Library.Main.Operation if (db.RepairInProgress) throw new UserInformationException("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsInRepairState"); - var tp = FilelistProcessor.RemoteListAnalysis(backend, m_options, db, m_result.BackendWriter, null); + var tp = await FilelistProcessor.RemoteListAnalysis(backendManager, m_options, db, m_result.BackendWriter, null).ConfigureAwait(false); var buffer = new byte[m_options.Blocksize]; var hashsize = HashFactory.HashSizeBytes(m_options.BlockHashAlgorithm); @@ -168,20 +165,18 @@ namespace Duplicati.Library.Main.Operation foreach (var n in tp.VerificationRequiredVolumes) try { - if (!m_result.TaskControl.ProgressRendevouz().Await()) + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - backend.WaitForComplete(db, null); + await backendManager.WaitForEmptyAsync(testdb, null, cancellationToken).ConfigureAwait(false); return; } progress++; m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess); - long size; - string hash; KeyValuePair>> res; - - using (var tf = backend.GetWithInfo(n.Name, out size, out hash)) + (var tf, var hash, var size) = await backendManager.GetWithInfoAsync(n.Name, n.Hash, n.Size, cancellationToken).ConfigureAwait(false); + using (tf) res = TestHandler.TestVolumeInternals(testdb, n, tf, m_options, 1); if (res.Value.Any()) @@ -207,9 +202,9 @@ namespace Duplicati.Library.Main.Operation foreach (var n in tp.ExtraVolumes) try { - if (!m_result.TaskControl.ProgressRendevouz().Await()) + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - backend.WaitForComplete(db, null); + await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false); return; } @@ -222,9 +217,8 @@ namespace Duplicati.Library.Main.Operation { try { - string hash; - long size; - using (var tf = backend.GetWithInfo(n.File.Name, out size, out hash)) + (var tf, var hash, var size) = await backendManager.GetWithInfoAsync(n.File.Name, null, n.File.Size, cancellationToken).ConfigureAwait(false); + using (tf) using (var ifr = new IndexVolumeReader(n.CompressionModule, tf, m_options, m_options.BlockhashSize)) { foreach (var rv in ifr.Volumes) @@ -277,7 +271,7 @@ namespace Duplicati.Library.Main.Operation if (!m_options.Dryrun) { db.RegisterRemoteVolume(n.File.Name, n.FileType, n.File.Size, RemoteVolumeState.Deleting); - backend.Delete(n.File.Name, n.File.Size); + await backendManager.DeleteAsync(n.File.Name, n.File.Size, false, cancellationToken).ConfigureAwait(false); } else Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteFile", "would delete file {0}", n.File.Name); @@ -302,9 +296,9 @@ namespace Duplicati.Library.Main.Operation try { - if (!m_result.TaskControl.ProgressRendevouz().Await()) + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - backend.WaitForComplete(db, null); + await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false); return; } @@ -331,7 +325,7 @@ namespace Duplicati.Library.Main.Operation else { db.UpdateRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null); - backend.Put(volumeWriter); + await backendManager.PutAsync(volumeWriter, null, null, false, cancellationToken).ConfigureAwait(false); } } else if (n.Type == RemoteVolumeType.Index) @@ -372,7 +366,7 @@ namespace Duplicati.Library.Main.Operation else { db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null); - backend.Put(w); + await backendManager.PutAsync(w, null, null, false, cancellationToken).ConfigureAwait(false); } } else if (n.Type == RemoteVolumeType.Blocks) @@ -423,20 +417,20 @@ namespace Duplicati.Library.Main.Operation } //Then we grab all remote volumes that have the missing blocks - foreach (var vol in new AsyncDownloader(mbl.GetMissingBlockSources().ToList(), backend)) + await foreach (var (tmpfile, _, _, name) in backendManager.GetFilesOverlappedAsync(mbl.GetMissingBlockSources().ToList(), cancellationToken)) { try { - using (var tmpfile = vol.TempFile) - using (var f = new BlockVolumeReader(RestoreHandler.GetCompressionModule(vol.Name), tmpfile, m_options)) + using (tmpfile) + using (var f = new BlockVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options)) foreach (var b in f.Blocks) if (mbl.SetBlockRestored(b.Key, b.Value)) if (f.ReadBlock(b.Key, buffer) == b.Value) - w.AddBlock(b.Key, buffer, 0, (int)b.Value, Duplicati.Library.Interface.CompressionHint.Default); + w.AddBlock(b.Key, buffer, 0, (int)b.Value, CompressionHint.Default); } catch (Exception ex) { - Logging.Log.WriteErrorMessage(LOGTAG, "RemoteFileAccessError", ex, "Failed to access remote file: {0}", vol.Name); + Logging.Log.WriteErrorMessage(LOGTAG, "RemoteFileAccessError", ex, "Failed to access remote file: {0}", name); } } @@ -468,7 +462,7 @@ namespace Duplicati.Library.Main.Operation else { db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null); - backend.Put(w); + await backendManager.PutAsync(w, null, null, false, cancellationToken).ConfigureAwait(false); } } } @@ -494,7 +488,7 @@ namespace Duplicati.Library.Main.Operation } m_result.OperationProgressUpdater.UpdateProgress(1); - backend.WaitForComplete(db, null); + await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false); db.WriteResults(); } } diff --git a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs index b8e770bc8..de1e8178f 100644 --- a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs @@ -92,11 +92,15 @@ namespace Duplicati.Library.Main.Operation.Restore /// /// Dictionary for keeping track of how many times each block is requested. Used to determine when a block is no longer needed. /// - private readonly Dictionary m_blockcount = new (); + private readonly Dictionary m_blockcount = new(); + /// + /// Lock for the block count dictionary. + /// + private readonly object m_blockcount_lock = new(); /// /// Dictionary for keeping track of how many times each volume is requested. Used to determine when a volume is no longer needed. /// - private readonly Dictionary m_volumecount = new (); + private readonly Dictionary m_volumecount = new(); /// /// The options for the restore. /// @@ -118,9 +122,9 @@ namespace Duplicati.Library.Main.Operation.Restore m_block_cache = new MemoryCache(cache_options); } this.readers = readers; - sw_cacheevict = options.InternalProfiling ? new () : null; - sw_checkcounts = options.InternalProfiling ? new () : null; - sw_get_wait = options.InternalProfiling ? new () : null; + sw_cacheevict = options.InternalProfiling ? new() : null; + sw_checkcounts = options.InternalProfiling ? new() : null; + sw_get_wait = options.InternalProfiling ? new() : null; foreach (var (block_id, volume_id) in db.GetBlocksAndVolumeIDs(options.SkipMetadata)) { @@ -143,12 +147,13 @@ namespace Duplicati.Library.Main.Operation.Restore { long error_block_id = -1; long error_volume_id = -1; + var emit_evict = false; - lock (m_blockcount) + lock (m_blockcount_lock) { sw_checkcounts?.Start(); - var block_count = m_blockcount.TryGetValue(blockRequest.BlockID, out var c) ? c-1 : 0; + var block_count = m_blockcount.TryGetValue(blockRequest.BlockID, out var c) ? c - 1 : 0; if (block_count > 0) { @@ -165,7 +170,7 @@ namespace Duplicati.Library.Main.Operation.Restore error_block_id = blockRequest.BlockID; } - var vol_count = m_volumecount.TryGetValue(blockRequest.VolumeID, out var vc) ? vc-1 : 0; + var vol_count = m_volumecount.TryGetValue(blockRequest.VolumeID, out var vc) ? vc - 1 : 0; if (vol_count > 0) { m_volumecount[blockRequest.VolumeID] = vol_count; @@ -175,7 +180,7 @@ namespace Duplicati.Library.Main.Operation.Restore // Notify the `VolumeManager` that it should evict the volume. m_volumecount.Remove(blockRequest.VolumeID); blockRequest.CacheDecrEvict = true; - m_volume_request.Write(blockRequest); + emit_evict = true; } else // vol_count < 0 { @@ -184,6 +189,9 @@ namespace Duplicati.Library.Main.Operation.Restore sw_checkcounts?.Stop(); } + if (emit_evict) + m_volume_request.Write(blockRequest); + if (error_block_id != -1) { Logging.Log.WriteWarningMessage(LOGTAG, "BlockCountError", null, $"Block {blockRequest.BlockID} has a count below 0"); @@ -222,8 +230,10 @@ namespace Duplicati.Library.Main.Operation.Restore } sw_get_wait?.Stop(); - return new_tcs.Task.ContinueWith(t => { - CheckCounts(block_request); + return new_tcs.Task.ContinueWith(t => + { + if (t.IsCompletedSuccessfully) + CheckCounts(block_request); return t.Result; }); } @@ -264,7 +274,8 @@ namespace Duplicati.Library.Main.Operation.Restore /// public void Retire() { - if (Interlocked.Decrement(ref readers) <= 0) { + if (Interlocked.Decrement(ref readers) <= 0) + { m_volume_request.Retire(); } } @@ -338,15 +349,16 @@ namespace Duplicati.Library.Main.Operation.Restore using SleepableDictionary cache = new(db, self.Output, options, fp_requests.Length); // The volume consumer will read blocks from the input channel (data blocks from the volumes) and store them in the cache. - var volume_consumer = Task.Run(async () => { - Stopwatch sw_read = options.InternalProfiling ? new () : null; - Stopwatch sw_set = options.InternalProfiling ? new () : null; + var volume_consumer = Task.Run(async () => + { + Stopwatch sw_read = options.InternalProfiling ? new() : null; + Stopwatch sw_set = options.InternalProfiling ? new() : null; try { while (true) { sw_read?.Start(); - var (block_request, data) = await self.Input.ReadAsync(); + var (block_request, data) = await self.Input.ReadAsync().ConfigureAwait(false); sw_read?.Stop(); sw_set?.Start(); @@ -376,17 +388,18 @@ namespace Duplicati.Library.Main.Operation.Restore }); // The block handlers will read block requests from the `FileProcessor`, access the cache for the blocks, and write the resulting blocks to the `FileProcessor`. - var block_handlers = fp_requests.Zip(fp_responses, (req, res) => Task.Run(async () => { - Stopwatch sw_req = options.InternalProfiling ? new () : null; - Stopwatch sw_resp = options.InternalProfiling ? new () : null; - Stopwatch sw_cache = options.InternalProfiling ? new () : null; - Stopwatch sw_get = options.InternalProfiling ? new () : null; + var block_handlers = fp_requests.Zip(fp_responses, (req, res) => Task.Run(async () => + { + Stopwatch sw_req = options.InternalProfiling ? new() : null; + Stopwatch sw_resp = options.InternalProfiling ? new() : null; + Stopwatch sw_cache = options.InternalProfiling ? new() : null; + Stopwatch sw_get = options.InternalProfiling ? new() : null; try { while (true) { sw_req?.Start(); - var block_request = await req.ReadAsync(); + var block_request = await req.ReadAsync().ConfigureAwait(false); sw_req?.Stop(); if (block_request.CacheDecrEvict) { @@ -398,11 +411,11 @@ namespace Duplicati.Library.Main.Operation.Restore else { sw_get?.Start(); - var data = await cache.Get(block_request); + var data = await cache.Get(block_request).ConfigureAwait(false); sw_get?.Stop(); sw_resp?.Start(); - await res.WriteAsync(data); + await res.WriteAsync(data).ConfigureAwait(false); sw_resp?.Stop(); } } @@ -420,7 +433,7 @@ namespace Duplicati.Library.Main.Operation.Restore } })).ToArray(); - await Task.WhenAll([volume_consumer, ..block_handlers]); + await Task.WhenAll([volume_consumer, .. block_handlers]).ConfigureAwait(false); }); } } diff --git a/Duplicati/Library/Main/Operation/Restore/Channels.cs b/Duplicati/Library/Main/Operation/Restore/Channels.cs index 3f5f4cd90..de1f18aab 100644 --- a/Duplicati/Library/Main/Operation/Restore/Channels.cs +++ b/Duplicati/Library/Main/Operation/Restore/Channels.cs @@ -19,10 +19,10 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +using System.Threading.Tasks; using CoCoL; using Duplicati.Library.Main.Volumes; using Duplicati.Library.Utility; -using static Duplicati.Library.Main.BackendManager; namespace Duplicati.Library.Main.Operation.Restore { @@ -46,7 +46,7 @@ namespace Duplicati.Library.Main.Operation.Restore /// /// Channel between and . /// - public readonly IChannel<(long, IDownloadWaitHandle)> DownloadRequest = ChannelManager.CreateChannel<(long, IDownloadWaitHandle)>(buffersize: BufferSize); + public readonly IChannel<(long, Task)> DownloadRequest = ChannelManager.CreateChannel<(long, Task)>(buffersize: BufferSize); /// /// Channel between and diff --git a/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs b/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs index b3a5b17d0..7aafb3e38 100644 --- a/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs +++ b/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs @@ -55,9 +55,9 @@ namespace Duplicati.Library.Main.Operation.Restore }, async self => { - Stopwatch sw_read = options.InternalProfiling ? new () : null; - Stopwatch sw_write = options.InternalProfiling ? new () : null; - Stopwatch sw_wait = options.InternalProfiling ? new () : null; + Stopwatch sw_read = options.InternalProfiling ? new() : null; + Stopwatch sw_write = options.InternalProfiling ? new() : null; + Stopwatch sw_wait = options.InternalProfiling ? new() : null; try { @@ -65,23 +65,22 @@ namespace Duplicati.Library.Main.Operation.Restore { // Get the block request from the `BlockManager` process. sw_read?.Start(); - var (volume_id, handle) = await self.Input.ReadAsync(); + var (volume_id, waittask) = await self.Input.ReadAsync(); sw_read?.Stop(); - // Trigger the download. + // Trigger the download. sw_wait?.Start(); TempFile f = null; try { - f = handle.Wait(); + f = await waittask.ConfigureAwait(false); } catch (Exception) { var (volume_name, _, _) = db.GetVolumeInfo(volume_id).First(); lock (results) - { results.BrokenRemoteFiles.Add(volume_name); - } + throw; } sw_wait?.Stop(); diff --git a/Duplicati/Library/Main/Operation/Restore/VolumeManager.cs b/Duplicati/Library/Main/Operation/Restore/VolumeManager.cs index f178fdd79..4622ce03d 100644 --- a/Duplicati/Library/Main/Operation/Restore/VolumeManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/VolumeManager.cs @@ -50,7 +50,7 @@ namespace Duplicati.Library.Main.Operation.Restore /// The backend manager, used to fetch the volumes from the backend. /// The restore options. /// The restore results. - public static Task Run(Channels channels, LocalRestoreDatabase db, BackendManager backend, Options options, RestoreResults results) + public static Task Run(Channels channels, LocalRestoreDatabase db, IBackendManager backend, Options options, RestoreResults results) { return AutomationExtensions.RunTask( new @@ -65,12 +65,12 @@ namespace Duplicati.Library.Main.Operation.Restore Dictionary files = []; Dictionary> in_flight = []; - Stopwatch sw_cache_set = options.InternalProfiling ? new () : null; - Stopwatch sw_cache_evict = options.InternalProfiling ? new () : null; - Stopwatch sw_query = options.InternalProfiling ? new () : null; - Stopwatch sw_backend = options.InternalProfiling ? new () : null; - Stopwatch sw_request = options.InternalProfiling ? new () : null; - Stopwatch sw_wakeup = options.InternalProfiling ? new () : null; + Stopwatch sw_cache_set = options.InternalProfiling ? new() : null; + Stopwatch sw_cache_evict = options.InternalProfiling ? new() : null; + Stopwatch sw_query = options.InternalProfiling ? new() : null; + Stopwatch sw_backend = options.InternalProfiling ? new() : null; + Stopwatch sw_request = options.InternalProfiling ? new() : null; + Stopwatch sw_wakeup = options.InternalProfiling ? new() : null; try { @@ -110,12 +110,12 @@ namespace Duplicati.Library.Main.Operation.Restore var (volume_name, volume_size, volume_hash) = db.GetVolumeInfo(request.VolumeID).First(); sw_query?.Stop(); sw_backend?.Start(); - var handle = backend.GetAsync(volume_name, volume_size, volume_hash); + var handle = backend.GetAsync(volume_name, volume_hash, volume_size, results.TaskControl.TransferToken); sw_backend?.Stop(); await self.DownloadRequest.WriteAsync((request.VolumeID, handle)); in_flight[request.VolumeID] = [request]; } - }; + } sw_request?.Stop(); } break; diff --git a/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs b/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs index 74c3c50d0..e3739f4dc 100644 --- a/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs @@ -1,27 +1,26 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; -using System.Linq; -using System.Text; +using System.Threading; using Duplicati.Library.Utility; namespace Duplicati.Library.Main.Operation @@ -29,18 +28,17 @@ namespace Duplicati.Library.Main.Operation internal class RestoreControlFilesHandler { private readonly Options m_options; - private readonly string m_backendurl; private readonly RestoreControlFilesResults m_result; - public RestoreControlFilesHandler(string backendurl, Options options, RestoreControlFilesResults result) + public RestoreControlFilesHandler(Options options, RestoreControlFilesResults result) { m_options = options; - m_backendurl = backendurl; m_result = result; } - public void Run(IEnumerable filterstrings = null, Library.Utility.IFilter compositefilter = null) + public void Run(IEnumerable filterstrings, IBackendManager backendManager, Library.Utility.IFilter compositefilter) { + var cancellationToken = CancellationToken.None; if (string.IsNullOrEmpty(m_options.Restorepath)) throw new Exception("Cannot restore control files without --restore-path"); if (!System.IO.Directory.Exists(m_options.Restorepath)) @@ -48,7 +46,6 @@ namespace Duplicati.Library.Main.Operation using (var tmpdb = new Library.Utility.TempFile()) using (var db = new Database.LocalDatabase(System.IO.File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "RestoreControlFiles", true)) - using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db)) { m_result.SetDatabase(db); @@ -56,7 +53,7 @@ namespace Duplicati.Library.Main.Operation try { - var filteredList = ListFilesHandler.ParseAndFilterFilesets(backend.List(), m_options); + var filteredList = ListFilesHandler.ParseAndFilterFilesets(backendManager.ListAsync(cancellationToken).Await(), m_options); if (filteredList.Count == 0) throw new Exception("No filesets found on remote target"); @@ -67,7 +64,7 @@ namespace Duplicati.Library.Main.Operation { if (!m_result.TaskControl.ProgressRendevouz().Await()) { - backend.WaitForComplete(db, null); + backendManager.WaitForEmptyAsync(db, null, cancellationToken).Await(); return; } @@ -75,7 +72,7 @@ namespace Duplicati.Library.Main.Operation var entry = db.GetRemoteVolume(file.Name); var res = new List(); - using (var tmpfile = backend.Get(file.Name, entry.Size < 0 ? file.Size : entry.Size, entry.Hash)) + using (var tmpfile = backendManager.GetAsync(file.Name, entry.Hash, entry.Size < 0 ? file.Size : entry.Size, cancellationToken).Await()) using (var tmp = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(file.Name), tmpfile, m_options)) foreach (var cf in tmp.ControlFiles) if (Library.Utility.FilterExpression.Matches(filter, cf.Key)) @@ -103,7 +100,7 @@ namespace Duplicati.Library.Main.Operation } finally { - backend.WaitForComplete(db, null); + backendManager.WaitForEmptyAsync(db, null, cancellationToken).Await(); } db.WriteResults(); diff --git a/Duplicati/Library/Main/Operation/RestoreHandler.cs b/Duplicati/Library/Main/Operation/RestoreHandler.cs index 3a7aa6b06..73620a62f 100644 --- a/Duplicati/Library/Main/Operation/RestoreHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreHandler.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -40,16 +40,14 @@ namespace Duplicati.Library.Main.Operation /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(); - private readonly string m_backendurl; private readonly Options m_options; private byte[] m_blockbuffer; private readonly RestoreResults m_result; private static readonly string DIRSEP = Util.DirectorySeparatorString; - public RestoreHandler(string backendurl, Options options, RestoreResults result) + public RestoreHandler(Options options, RestoreResults result) { m_options = options; - m_backendurl = backendurl; m_result = result; } @@ -104,7 +102,7 @@ namespace Duplicati.Library.Main.Operation }; } - public void Run(string[] paths, Library.Utility.IFilter filter = null) + public void Run(string[] paths, IBackendManager backendManager, Library.Utility.IFilter filter) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_Begin); @@ -128,8 +126,8 @@ namespace Duplicati.Library.Main.Operation db = new LocalRestoreDatabase(tmpdb); m_result.RecreateDatabaseResults = new RecreateDatabaseResults(m_result); using (new Logging.Timer(LOGTAG, "RecreateTempDbForRestore", "Recreate temporary database for restore")) - new RecreateDatabaseHandler(m_backendurl, m_options, (RecreateDatabaseResults)m_result.RecreateDatabaseResults) - .DoRun(db, false, filter, filelistfilter, null); + new RecreateDatabaseHandler(m_options, (RecreateDatabaseResults)m_result.RecreateDatabaseResults) + .DoRun(backendManager, db, false, filter, filelistfilter, null).Await(); if (!m_options.SkipMetadata) ApplyStoredMetadata(m_options, new RestoreHandlerMetadataStorage()); @@ -141,9 +139,9 @@ namespace Duplicati.Library.Main.Operation } if (m_options.RestoreLegacy) - DoRun(db, filter); + DoRun(backendManager, db, filter, m_result.TaskControl.ProgressToken).Await(); else - DoRunNew(db, filter); + DoRunNew(backendManager, db, filter, m_result.TaskControl.ProgressToken).Await(); db.WriteResults(); } @@ -309,27 +307,24 @@ namespace Duplicati.Library.Main.Operation /// /// The database containing information about the restore. /// The filter of which files to restore. - private void DoRunNew(LocalRestoreDatabase database, Library.Utility.IFilter filter) + private async Task DoRunNew(IBackendManager backendManager, LocalRestoreDatabase database, Library.Utility.IFilter filter, CancellationToken cancellationToken) { // Perform initial setup Utility.UpdateOptionsFromDb(database, m_options); Utility.VerifyOptionsAndUpdateDatabase(database, m_options); - // Open the backend and metadata storage - using var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, database); - // Verify the backend if necessary if (!m_options.NoBackendverification) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PreRestoreVerify); - FilelistProcessor.VerifyRemoteList(backend, m_options, database, m_result.BackendWriter, false, null); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, database, m_result.BackendWriter, false, null).ConfigureAwait(false); } // Prepare the block and file list and create the directory structure m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateFileList); PrepareBlockAndFileList(database, m_options, filter, m_result); m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateTargetFolders); - CreateDirectoryStructure(database, m_options, m_result); + await CreateDirectoryStructure(database, m_options, m_result).ConfigureAwait(false); database.SetResult(m_result); // Create the channels between BlockManager and FileProcessor @@ -341,7 +336,7 @@ namespace Duplicati.Library.Main.Operation var filelister = Restore.FileLister.Run(channels, database, m_options, m_result); var fileprocessors = Enumerable.Range(0, m_options.RestoreFileProcessors).Select(i => Restore.FileProcessor.Run(channels, database, fileprocessor_requests[i], fileprocessor_responses[i], m_options, m_result)).ToArray(); var blockmanager = Restore.BlockManager.Run(channels, database, m_options, fileprocessor_requests, fileprocessor_responses); - var volumecache = Restore.VolumeManager.Run(channels, database, backend, m_options, m_result); + var volumecache = Restore.VolumeManager.Run(channels, database, backendManager, m_options, m_result); var volumedownloaders = Enumerable.Range(0, m_options.RestoreVolumeDownloaders).Select(i => Restore.VolumeDownloader.Run(channels, database, m_options, m_result)).ToArray(); var volumedecryptors = Enumerable.Range(0, m_options.RestoreVolumeDecryptors).Select(i => Restore.VolumeDecryptor.Run(channels, m_options)).ToArray(); var volumedecompressors = Enumerable.Range(0, m_options.RestoreVolumeDecompressors).Select(i => Restore.VolumeDecompressor.Run(channels, m_options)).ToArray(); @@ -359,20 +354,22 @@ namespace Duplicati.Library.Main.Operation ]; // Start the progress updater - var kill_updater = new CancellationTokenSource(); - var updater = Task.Run(async () => + using (var kill_updater = new CancellationTokenSource()) { - while (true) + var updater = Task.Run(async () => { - m_result.OperationProgressUpdater.UpdatefilesProcessed(m_result.RestoredFiles, m_result.SizeOfRestoredFiles); - await Task.Delay(1000); - } - }, kill_updater.Token); + while (!kill_updater.Token.IsCancellationRequested) + { + m_result.OperationProgressUpdater.UpdatefilesProcessed(m_result.RestoredFiles, m_result.SizeOfRestoredFiles); + await Task.Delay(1000, kill_updater.Token).ConfigureAwait(false); + } + }, kill_updater.Token); - Task.WhenAll(all).Wait(); - kill_updater.Cancel(); + await Task.WhenAll(all).ConfigureAwait(false); + kill_updater.Cancel(); + } - if (!m_result.TaskControl.ProgressRendevouz().Await()) + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) return; m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PostRestoreVerify); @@ -399,7 +396,7 @@ namespace Duplicati.Library.Main.Operation // Drop the temp tables database.DropRestoreTable(); - backend.WaitForComplete(database, null); + await backendManager.WaitForEmptyAsync(database, null, CancellationToken.None).ConfigureAwait(false); // Report that the restore is complete m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_Complete); @@ -412,12 +409,11 @@ namespace Duplicati.Library.Main.Operation /// /// The database containing information about the restore. /// The filter of which files to restore. - private void DoRun(LocalRestoreDatabase database, Library.Utility.IFilter filter) + private async Task DoRun(IBackendManager backendManager, LocalRestoreDatabase database, Library.Utility.IFilter filter, CancellationToken cancellationToken) { //In this case, we check that the remote storage fits with the database. //We can then query the database and find the blocks that we need to do the restore //using (var database = new LocalRestoreDatabase(dbparent)) - using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, database)) using (var metadatastorage = new RestoreHandlerMetadataStorage()) { database.SetResult(m_result); @@ -428,7 +424,7 @@ namespace Duplicati.Library.Main.Operation if (!m_options.NoBackendverification) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PreRestoreVerify); - FilelistProcessor.VerifyRemoteList(backend, m_options, database, m_result.BackendWriter, false, null); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, database, m_result.BackendWriter, false, null).ConfigureAwait(false); } //Figure out what files are to be patched, and what blocks are needed @@ -439,14 +435,14 @@ namespace Duplicati.Library.Main.Operation //Make the entire output setup m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateTargetFolders); using (new Logging.Timer(LOGTAG, "CreateDirectory", "CreateDirectory")) - CreateDirectoryStructure(database, m_options, m_result); + await CreateDirectoryStructure(database, m_options, m_result); //If we are patching an existing target folder, do not touch stuff that is already updated m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_ScanForExistingFiles); using (var blockhasher = HashFactory.CreateHasher(m_options.BlockHashAlgorithm)) using (var filehasher = HashFactory.CreateHasher(m_options.FileHashAlgorithm)) using (new Logging.Timer(LOGTAG, "ScanForExistingTargetBlocks", "ScanForExistingTargetBlocks")) - ScanForExistingTargetBlocks(database, m_blockbuffer, blockhasher, filehasher, m_options, m_result); + await ScanForExistingTargetBlocks(database, m_blockbuffer, blockhasher, filehasher, m_options, m_result).ConfigureAwait(false); //Look for existing blocks in the original source files only if (m_options.UseLocalBlocks && !string.IsNullOrEmpty(m_options.Restorepath)) @@ -455,13 +451,13 @@ namespace Duplicati.Library.Main.Operation using (new Logging.Timer(LOGTAG, "ScanForExistingSourceBlocksFast", "ScanForExistingSourceBlocksFast")) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_ScanForLocalBlocks); - ScanForExistingSourceBlocksFast(database, m_options, m_blockbuffer, blockhasher, m_result); + await ScanForExistingSourceBlocksFast(database, m_options, m_blockbuffer, blockhasher, m_result).ConfigureAwait(false); } } - if (!m_result.TaskControl.ProgressRendevouz().Await()) + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - backend.WaitForComplete(database, null); + await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false); return; } @@ -471,12 +467,12 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PatchWithLocalBlocks); using (var blockhasher = HashFactory.CreateHasher(m_options.BlockHashAlgorithm)) using (new Logging.Timer(LOGTAG, "PatchWithLocalBlocks", "PatchWithLocalBlocks")) - ScanForExistingSourceBlocks(database, m_options, m_blockbuffer, blockhasher, m_result, metadatastorage); + await ScanForExistingSourceBlocks(database, m_options, m_blockbuffer, blockhasher, m_result, metadatastorage).ConfigureAwait(false); } - if (!m_result.TaskControl.ProgressRendevouz().Await()) + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - backend.WaitForComplete(database, null); + await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false); return; } @@ -493,26 +489,28 @@ namespace Duplicati.Library.Main.Operation var brokenFiles = new List(); - foreach (var blockvolume in new AsyncDownloader(volumes, backend)) + await foreach (var (tmpfile, _, _, name) in backendManager.GetFilesOverlappedAsync(volumes, cancellationToken)) + { try { - if (!m_result.TaskControl.ProgressRendevouz().Await()) + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - backend.WaitForComplete(database, null); + await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false); return; } - using (var tmpfile = blockvolume.TempFile) - using (var blocks = new BlockVolumeReader(GetCompressionModule(blockvolume.Name), tmpfile, m_options)) + using (tmpfile) + using (var blocks = new BlockVolumeReader(GetCompressionModule(name), tmpfile, m_options)) PatchWithBlocklist(database, blocks, m_options, m_result, m_blockbuffer, metadatastorage); } catch (Exception ex) { - brokenFiles.Add(blockvolume.Name); - Logging.Log.WriteErrorMessage(LOGTAG, "PatchingFailed", ex, "Failed to patch with remote file: \"{0}\", message: {1}", blockvolume.Name, ex.Message); + brokenFiles.Add(name); + Logging.Log.WriteErrorMessage(LOGTAG, "PatchingFailed", ex, "Failed to patch with remote file: \"{0}\", message: {1}", name, ex.Message); if (ex is System.Threading.ThreadAbortException) throw; } + } var fileErrors = 0L; @@ -545,7 +543,7 @@ namespace Duplicati.Library.Main.Operation if (!m_options.SkipMetadata) ApplyStoredMetadata(m_options, metadatastorage); - if (!m_result.TaskControl.ProgressRendevouz().Await()) + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) return; m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PostRestoreVerify); @@ -560,9 +558,9 @@ namespace Duplicati.Library.Main.Operation { try { - if (!m_result.TaskControl.ProgressRendevouz().Await()) + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - backend.WaitForComplete(database, null); + await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false); return; } @@ -600,7 +598,7 @@ namespace Duplicati.Library.Main.Operation // Drop the temp tables database.DropRestoreTable(); - backend.WaitForComplete(database, null); + await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false); } m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_Complete); @@ -676,7 +674,7 @@ namespace Duplicati.Library.Main.Operation } } - private static void ScanForExistingSourceBlocksFast(LocalRestoreDatabase database, Options options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, RestoreResults result) + private static async Task ScanForExistingSourceBlocksFast(LocalRestoreDatabase database, Options options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, RestoreResults result) { // Fill BLOCKS with data from known local source files using (var blockmarker = database.CreateBlockMarker()) @@ -708,7 +706,7 @@ namespace Duplicati.Library.Main.Operation { foreach (var block in entry.Blocks) { - if (!result.TaskControl.ProgressRendevouz().Await()) + if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) return; //TODO: Handle metadata @@ -748,7 +746,7 @@ namespace Duplicati.Library.Main.Operation if ((++updateCount) % 20 == 0) { blockmarker.UpdateProcessed(result.OperationProgressUpdater); - if (!result.TaskControl.ProgressRendevouz().Await()) + if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) return; } @@ -781,7 +779,7 @@ namespace Duplicati.Library.Main.Operation } } - private static void ScanForExistingSourceBlocks(LocalRestoreDatabase database, Options options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, RestoreResults result, RestoreHandlerMetadataStorage metadatastorage) + private static async Task ScanForExistingSourceBlocks(LocalRestoreDatabase database, Options options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, RestoreResults result, RestoreHandlerMetadataStorage metadatastorage) { // Fill BLOCKS with data from known local source files using (var blockmarker = database.CreateBlockMarker()) @@ -794,7 +792,7 @@ namespace Duplicati.Library.Main.Operation var patched = false; try { - if (!result.TaskControl.ProgressRendevouz().Await()) + if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) return; var folderpath = SystemIO.IO_OS.PathGetDirectoryName(targetpath); @@ -811,7 +809,7 @@ namespace Duplicati.Library.Main.Operation { try { - if (!result.TaskControl.ProgressRendevouz().Await()) + if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) return; if (SystemIO.IO_OS.FileExists(source.Path)) @@ -923,7 +921,7 @@ namespace Duplicati.Library.Main.Operation } - private static void CreateDirectoryStructure(LocalRestoreDatabase database, Options options, RestoreResults result) + private static async Task CreateDirectoryStructure(LocalRestoreDatabase database, Options options, RestoreResults result) { // This part is not protected by try/catch as we need the target folder to exist if (!string.IsNullOrEmpty(options.Restorepath)) @@ -941,7 +939,7 @@ namespace Duplicati.Library.Main.Operation { try { - if (!result.TaskControl.ProgressRendevouz().Await()) + if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) return; if (!SystemIO.IO_OS.DirectoryExists(folder)) @@ -965,7 +963,7 @@ namespace Duplicati.Library.Main.Operation } } - private static void ScanForExistingTargetBlocks(LocalRestoreDatabase database, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm blockhasher, System.Security.Cryptography.HashAlgorithm filehasher, Options options, RestoreResults result) + private static async Task ScanForExistingTargetBlocks(LocalRestoreDatabase database, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm blockhasher, System.Security.Cryptography.HashAlgorithm filehasher, Options options, RestoreResults result) { // Scan existing files for existing BLOCKS using (var blockmarker = database.CreateBlockMarker()) @@ -982,7 +980,7 @@ namespace Duplicati.Library.Main.Operation { try { - if (!result.TaskControl.ProgressRendevouz().Await()) + if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) return; var currentfilelength = SystemIO.IO_OS.FileLength(targetpath); @@ -1045,7 +1043,7 @@ namespace Duplicati.Library.Main.Operation var key = Convert.ToBase64String(blockhasher.ComputeHash(blockbuffer, 0, size)); - if (calcFileHashTask != null) calcFileHashTask.Await(); // wait because blockbuffer will be overwritten. + if (calcFileHashTask != null) await calcFileHashTask.ConfigureAwait(false); // wait because blockbuffer will be overwritten. if (key == targetblock.Hash) { @@ -1099,7 +1097,7 @@ namespace Duplicati.Library.Main.Operation if ((++updateCount) % 20 == 0) { blockmarker.UpdateProcessed(result.OperationProgressUpdater); - if (!result.TaskControl.ProgressRendevouz().Await()) + if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) return; } } diff --git a/Duplicati/Library/Main/Operation/TestHandler.cs b/Duplicati/Library/Main/Operation/TestHandler.cs index 3aa34aa59..21f485cf3 100644 --- a/Duplicati/Library/Main/Operation/TestHandler.cs +++ b/Duplicati/Library/Main/Operation/TestHandler.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -24,8 +24,9 @@ using Duplicati.Library.Main.Database; using System.Linq; using System.Collections.Generic; using Duplicati.Library.Interface; -using System.Security.Cryptography; using Duplicati.Library.Utility; +using System.Threading; +using System.Threading.Tasks; namespace Duplicati.Library.Main.Operation { @@ -37,36 +38,33 @@ namespace Duplicati.Library.Main.Operation private static readonly string LOGTAG = Logging.Log.LogTagFromType(); private readonly Options m_options; - private readonly string m_backendurl; private readonly TestResults m_results; - public TestHandler(string backendurl, Options options, TestResults results) + public TestHandler(Options options, TestResults results) { m_options = options; - m_backendurl = backendurl; m_results = results; } - public void Run(long samples) + public void Run(long samples, IBackendManager backendManager) { if (!System.IO.File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); using (var db = new LocalTestDatabase(m_options.Dbpath)) - using (var backend = new BackendManager(m_backendurl, m_options, m_results.BackendWriter, db)) { db.SetResult(m_results); Utility.UpdateOptionsFromDb(db, m_options); Utility.VerifyOptionsAndUpdateDatabase(db, m_options); db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, !m_options.DisableFilelistConsistencyChecks, null); - FilelistProcessor.VerifyRemoteList(backend, m_options, db, m_results.BackendWriter, true, null); + FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_results.BackendWriter, true, null).Await(); - DoRun(samples, db, backend); + DoRun(samples, db, backendManager).Await(); db.WriteResults(); } } - public void DoRun(long samples, LocalTestDatabase db, BackendManager backend) + public async Task DoRun(long samples, LocalTestDatabase db, IBackendManager backend) { var files = db.SelectTestTargets(samples, m_options).ToList(); @@ -76,13 +74,14 @@ namespace Duplicati.Library.Main.Operation if (m_options.FullRemoteVerification != Options.RemoteTestStrategy.False) { - foreach (var vol in new AsyncDownloader(files, backend)) + await foreach (var (tf, hash, size, name) in backend.GetFilesOverlappedAsync(files, CancellationToken.None)) { + var vol = new RemoteVolume(name, hash, size); try { if (!m_results.TaskControl.ProgressRendevouz().Await()) { - backend.WaitForComplete(db, null); + backend.WaitForEmptyAsync(db, null, CancellationToken.None).Await(); m_results.EndTime = DateTime.UtcNow; return; } @@ -91,7 +90,7 @@ namespace Duplicati.Library.Main.Operation m_results.OperationProgressUpdater.UpdateProgress((float)progress / files.Count); KeyValuePair>> res; - using (var tf = vol.TempFile) + using (tf) res = TestVolumeInternals(db, vol, tf, m_options, m_options.FullBlockVerification ? 1.0 : 0.2); m_results.AddResult(res.Key, res.Value); @@ -152,10 +151,10 @@ namespace Duplicati.Library.Main.Operation { Logging.Log.WriteInformationMessage(LOGTAG, "MissingRemoteHash", "No hash or size recorded for {0}, performing full verification", f.Name); KeyValuePair>> res; - string hash; - long size; - using (var tf = backend.GetWithInfo(f.Name, out size, out hash)) + (var tf, var hash, var size) = backend.GetWithInfoAsync(f.Name, f.Hash, f.Size, CancellationToken.None).Await(); + + using (tf) res = TestVolumeInternals(db, f, tf, m_options, 1); m_results.AddResult(res.Key, res.Value); @@ -176,16 +175,19 @@ namespace Duplicati.Library.Main.Operation } } else - backend.GetForTesting(f.Name, f.Size, f.Hash); + { + using (var tf = backend.GetAsync(f.Name, f.Hash, f.Size, CancellationToken.None).Await()) + { } + } db.UpdateVerificationCount(f.Name); - m_results.AddResult(f.Name, new KeyValuePair[0]); + m_results.AddResult(f.Name, []); } catch (Exception ex) { - m_results.AddResult(f.Name, new KeyValuePair[] { new KeyValuePair(Duplicati.Library.Interface.TestEntryStatus.Error, ex.Message) }); + m_results.AddResult(f.Name, [new KeyValuePair(TestEntryStatus.Error, ex.Message)]); Logging.Log.WriteErrorMessage(LOGTAG, "FailedToProcessFile", ex, "Failed to process file {0}", f.Name); - if (ex is System.Threading.ThreadAbortException) + if (ex is ThreadAbortException || ex is TaskCanceledException) { m_results.EndTime = DateTime.UtcNow; throw; diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index 81678673d..6131d1dd1 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -382,12 +382,13 @@ namespace Duplicati.Library.Main new CommandLineArgument("symlink-policy", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.SymlinkpolicyShort, Strings.Options.SymlinkpolicyLong("store", "ignore", "follow"), Enum.GetName(typeof(SymlinkStrategy), SymlinkStrategy.Store), null, Enum.GetNames(typeof(SymlinkStrategy))), new CommandLineArgument("hardlink-policy", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.HardlinkpolicyShort, Strings.Options.HardlinkpolicyLong("first", "all", "none"), Enum.GetName(typeof(HardlinkStrategy), HardlinkStrategy.All), null, Enum.GetNames(typeof(HardlinkStrategy))), new CommandLineArgument("exclude-files-attributes", CommandLineArgument.ArgumentType.String, Strings.Options.ExcludefilesattributesShort, Strings.Options.ExcludefilesattributesLong(Enum.GetNames(typeof(System.IO.FileAttributes)))), - new CommandLineArgument("backup-name", CommandLineArgument.ArgumentType.String, Strings.Options.BackupnameShort, Strings.Options.BackupnameLong, DefaultBackupName), new CommandLineArgument("compression-extension-file", CommandLineArgument.ArgumentType.Path, Strings.Options.CompressionextensionfileShort, Strings.Options.CompressionextensionfileLong(DEFAULT_COMPRESSED_EXTENSION_FILE), DEFAULT_COMPRESSED_EXTENSION_FILE), - new CommandLineArgument("backup-id", CommandLineArgument.ArgumentType.String, Strings.Options.BackupidShort, Strings.Options.BackupidLong, ""), new CommandLineArgument("machine-id", CommandLineArgument.ArgumentType.String, Strings.Options.MachineidShort, Strings.Options.MachineidLong, Library.AutoUpdater.UpdaterManager.InstallID), new CommandLineArgument("machine-name", CommandLineArgument.ArgumentType.String, Strings.Options.MachinenameShort, Strings.Options.MachinenameLong, Library.AutoUpdater.UpdaterManager.MachineName), + new CommandLineArgument("backup-id", CommandLineArgument.ArgumentType.String, Strings.Options.BackupidShort, Strings.Options.BackupidLong, ""), + new CommandLineArgument("backup-name", CommandLineArgument.ArgumentType.String, Strings.Options.BackupnameShort, Strings.Options.BackupnameLong, DefaultBackupName), + new CommandLineArgument("next-scheduled-run", CommandLineArgument.ArgumentType.String, Strings.Options.NextscheduledrunShort, Strings.Options.NextscheduledrunLong), new CommandLineArgument("verbose", CommandLineArgument.ArgumentType.Boolean, Strings.Options.VerboseShort, Strings.Options.VerboseLong, "false", null, null, Strings.Options.VerboseDeprecated), new CommandLineArgument("full-result", CommandLineArgument.ArgumentType.Boolean, Strings.Options.FullresultShort, Strings.Options.FullresultLong, "false"), diff --git a/Duplicati/Library/Main/ResultClasses.cs b/Duplicati/Library/Main/ResultClasses.cs index ce3c73184..0765ed458 100644 --- a/Duplicati/Library/Main/ResultClasses.cs +++ b/Duplicati/Library/Main/ResultClasses.cs @@ -68,6 +68,11 @@ namespace Duplicati.Library.Main ITaskControl TaskControl { get; } } + internal interface IBackendWriterProvider + { + IBackendWriter BackendWriter { get; } + } + internal class BackendWriter : BasicResults, IBackendWriter, IBackendStatstics, IParsedBackendStatistics { public BackendWriter(BasicResults p) : base(p) { } @@ -145,7 +150,7 @@ namespace Duplicati.Library.Main } - internal abstract class BasicResults : IBasicResults, ISetCommonOptions, Logging.ILogDestination, ITaskControlProvider + internal abstract class BasicResults : IBasicResults, ISetCommonOptions, Logging.ILogDestination, ITaskControlProvider, IBackendWriterProvider { /// /// The tag used for logging diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs index da7070238..6ee298520 100644 --- a/Duplicati/Library/Main/Strings.cs +++ b/Duplicati/Library/Main/Strings.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using Duplicati.Library.Localization.Short; @@ -171,6 +171,8 @@ namespace Duplicati.Library.Main.Strings public static string MachineidShort { get { return LC.L(@"Machine ID"); } } public static string MachinenameLong { get { return LC.L(@"The name of the machine running the backup. This can be used to identify the machine when sending mail or running scripts."); } } public static string MachinenameShort { get { return LC.L(@"Machine name"); } } + public static string NextscheduledrunShort { get { return LC.L(@"The time of the next scheduled run"); } } + public static string NextscheduledrunLong { get { return LC.L(@"This property is a reporting option and does not affect the actual scheduled time. Use this option to inform a reporting destination about the next expected time the backup will run."); } } public static string CompressionextensionfileLong(string path) { return LC.L(@"Use this option to point to a text file where each line contains a file extension that indicates a non-compressible file. Files that have an extension found in the file will not be compressed, but simply stored in the archive. The file format ignores any lines that do not start with a period, and considers a space to indicate the end of the extension. A default file is supplied, that also serves as an example. The default file is placed in {0}.", path); } public static string CompressionextensionfileShort { get { return LC.L(@"Manage non-compressible file extensions"); } } public static string BlocksizeLong { get { return LC.L(@"The block size determines how files are fragmented. Choosing a large value will cause a larger overhead on file changes, choosing a small value will cause a large overhead on storage of file lists. Note that the value cannot be changed after remote files are created."); } } @@ -268,7 +270,7 @@ namespace Duplicati.Library.Main.Strings public static string ConcurrencyblockhashersShort { get { return LC.L(@"Specify the number of concurrent hashing processes"); } } public static string ConcurrencycompressorsLong { get { return LC.L(@"Use this option to set the number of processes that perform compression of output data."); } } public static string ConcurrencycompressorsShort { get { return LC.L(@"Specify the number of concurrent compression processes"); } } - public static string ConcurrencyfileprocessorsShort{ get { return LC.L(@"[EXPERIMENTAL]Specify the number of concurrent files to open"); } } + public static string ConcurrencyfileprocessorsShort { get { return LC.L(@"[EXPERIMENTAL]Specify the number of concurrent files to open"); } } public static string ConcurrencyfileprocessorsLong { get { return LC.L(@"Use this option to set the number of concurrent files to open. This could accelerate big backups involving lot of files, such as an initial backup"); } } public static string DisablesyntehticfilelistLong { get { return LC.L(@"If Duplicati detects that the previous backup did not complete, it will generate a filelist that is a merge of the last completed backup and the contents that were uploaded in the incomplete backup session."); } } public static string DisablesyntheticfilelistShort { get { return LC.L(@"Disable synthetic filelist"); } } @@ -355,4 +357,9 @@ namespace Duplicati.Library.Main.Strings public static string SnapshotFailedError(string message) { return LC.L(@"Failed to create a snapshot: {0}", message); } } + internal static class BackendMananger + { + public static string EncryptionModuleNotFound(string name) { return LC.L(@"The encryption module {0} was not found", name); } + } + } diff --git a/Duplicati/Library/Main/Volumes/IParsedVolume.cs b/Duplicati/Library/Main/Volumes/IParsedVolume.cs index 10762d1fb..6dc30ab97 100644 --- a/Duplicati/Library/Main/Volumes/IParsedVolume.cs +++ b/Duplicati/Library/Main/Volumes/IParsedVolume.cs @@ -18,21 +18,18 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; - -namespace Duplicati.Library.Main.Volumes -{ - public interface IParsedVolume - { - RemoteVolumeType FileType { get; } - string Prefix { get; } - string Guid { get; } - DateTime Time { get; } - string CompressionModule { get; } - string EncryptionModule { get; } - Library.Interface.IFileEntry File { get; } - } -} +using System; + +namespace Duplicati.Library.Main.Volumes +{ + public interface IParsedVolume + { + RemoteVolumeType FileType { get; } + string Prefix { get; } + string Guid { get; } + DateTime Time { get; } + string CompressionModule { get; } + string EncryptionModule { get; } + Interface.IFileEntry File { get; } + } +} diff --git a/Duplicati/Library/Main/Volumes/VolumeWriterBase.cs b/Duplicati/Library/Main/Volumes/VolumeWriterBase.cs index f964d1244..c95822b98 100644 --- a/Duplicati/Library/Main/Volumes/VolumeWriterBase.cs +++ b/Duplicati/Library/Main/Volumes/VolumeWriterBase.cs @@ -89,15 +89,6 @@ namespace Duplicati.Library.Main.Volumes sr.Write(ManifestData.GetManifestInstance(m_blocksize, m_blockhash, m_filehash)); } - internal BackendHandler.FileEntryItem CreateFileEntryForUpload(Options options) - { - var fileEntry = new BackendHandler.FileEntryItem(BackendActionType.Put, this.RemoteFilename); - fileEntry.SetLocalfilename(this.LocalFilename); - fileEntry.Encrypt(options); - fileEntry.UpdateHashAndSize(options); - return fileEntry; - } - public virtual void Dispose() { if (m_compression != null) diff --git a/Duplicati/Library/Modules/Builtin/ReportHelper.cs b/Duplicati/Library/Modules/Builtin/ReportHelper.cs index d8c7e4d26..5e9db9a23 100644 --- a/Duplicati/Library/Modules/Builtin/ReportHelper.cs +++ b/Duplicati/Library/Modules/Builtin/ReportHelper.cs @@ -315,6 +315,108 @@ namespace Duplicati.Library.Modules.Builtin protected virtual string ReplaceTemplate(string input, object result, Exception exception, bool subjectline, ResultExportFormat format) => ReplaceTemplate(input, result, exception, subjectline, ResultFormatSerializerProvider.GetSerializer(format)); + /// + /// The operation name template key + /// + private const string OPERATIONNAME = "OperationName"; + /// + /// The remote url template key + /// + private const string REMOTEURL = "RemoteUrl"; + /// + /// The local path template key + /// + private const string LOCALPATH = "LocalPath"; + /// + /// The parsed result template key + /// + private const string PARSEDRESULT = "ParsedResult"; + /// + /// The machine id template key + /// + private const string MACHINE_ID = "machine-id"; + /// + /// The backup id template key + /// + private const string BACKUP_ID = "backup-id"; + /// + /// The backup name template key + /// + private const string BACKUP_NAME = "backup-name"; + /// + /// The machine name template key + /// + private const string MACHINE_NAME = "machine-name"; + /// + /// The operating system template key + /// + private const string OPERATING_SYSTEM = "operating-system"; + /// + /// The installation type template key + /// + private const string INSTALLATION_TYPE = "installation-type"; + /// + /// The destination type template key + /// + private const string DESTINATION_TYPE = "destination-type"; + /// + /// The next scheduled run template key + /// + private const string NEXT_SCHEDULED_RUN = "next-scheduled-run"; + + /// + /// The list of regular template keys + /// + private static readonly IReadOnlySet OPERATION_TEMPLATE_KEYS = new HashSet([ + OPERATIONNAME, REMOTEURL, LOCALPATH, PARSEDRESULT + ], StringComparer.OrdinalIgnoreCase); + + /// + /// The list of extra template keys + /// + private static readonly IReadOnlySet EXTRA_TEMPLATE_KEYS = new HashSet([ + MACHINE_ID, BACKUP_ID, BACKUP_NAME, MACHINE_NAME, + OPERATING_SYSTEM, INSTALLATION_TYPE, DESTINATION_TYPE, NEXT_SCHEDULED_RUN + ], StringComparer.OrdinalIgnoreCase); + + /// + /// Gets the default value for a template key + /// + /// The name of the template key + /// The default value + private string GetDefaultValue(string name) + { + switch (name) + { + case OPERATIONNAME: + return m_operationname; + case REMOTEURL: + return m_remoteurl; + case LOCALPATH: + return m_localpath == null ? "" : string.Join(System.IO.Path.PathSeparator.ToString(), m_localpath); + case PARSEDRESULT: + return m_parsedresultlevel; + case MACHINE_ID: + return AutoUpdater.UpdaterManager.MachineID; + case BACKUP_ID: + return Utility.Utility.ByteArrayAsHexString(Utility.Utility.RepeatedHashWithSalt(m_remoteurl, SALT)); + case BACKUP_NAME: + return System.IO.Path.GetFileNameWithoutExtension(Utility.Utility.getEntryAssembly().Location); + case MACHINE_NAME: + return AutoUpdater.UpdaterManager.MachineName; + case OPERATING_SYSTEM: + return OperatingSystem.IsWindows() ? "Windows" : OperatingSystem.IsLinux() ? "Linux" : OperatingSystem.IsMacOS() ? "MacOS" : "Unknown"; + case INSTALLATION_TYPE: + return AutoUpdater.UpdaterManager.PackageTypeId; + case DESTINATION_TYPE: + // Only return the url scheme, as the rest could contain sensitive information + var ix = m_remoteurl?.IndexOf("://", StringComparison.OrdinalIgnoreCase) ?? -1; + return ix >= 0 && ix < 15 ? m_remoteurl.Substring(0, ix) : "file"; + default: + return null; + } + } + /// /// Helper method to perform template expansion /// @@ -331,55 +433,29 @@ namespace Duplicati.Library.Modules.Builtin { var extra = new Dictionary(); - if (input.IndexOf("%OPERATIONNAME%", StringComparison.OrdinalIgnoreCase) >= 0) - extra["OperationName"] = m_operationname; - if (input.IndexOf("%REMOTEURL%", StringComparison.OrdinalIgnoreCase) >= 0) - extra["RemoteUrl"] = m_remoteurl; - if (input.IndexOf("%LOCALPATH%", StringComparison.OrdinalIgnoreCase) >= 0 && m_localpath != null) - extra["LocalPath"] = string.Join(System.IO.Path.PathSeparator.ToString(), m_localpath); - if (input.IndexOf("%PARSEDRESULT%", StringComparison.OrdinalIgnoreCase) >= 0) - extra["ParsedResult"] = m_parsedresultlevel; - - // If the options contains the key, it is captured by the loop over m_options - // so we only patch it in case it is missing - - if (input.IndexOf("%machine-id%", StringComparison.OrdinalIgnoreCase) >= 0) - { - if (!m_options.ContainsKey("machine-id")) - extra["machine-id"] = Library.AutoUpdater.UpdaterManager.MachineID; - } - - if (input.IndexOf("%backup-id%", StringComparison.OrdinalIgnoreCase) >= 0) - { - if (!m_options.ContainsKey("backup-id")) - extra["backup-id"] = Library.Utility.Utility.ByteArrayAsHexString(Library.Utility.Utility.RepeatedHashWithSalt(m_remoteurl, SALT)); - } - - if (input.IndexOf("%backup-name%", StringComparison.OrdinalIgnoreCase) >= 0) - { - if (!m_options.ContainsKey("backup-name")) - extra["backup-name"] = System.IO.Path.GetFileNameWithoutExtension(Duplicati.Library.Utility.Utility.getEntryAssembly().Location); - } - - if (input.IndexOf("%machine-name%", StringComparison.OrdinalIgnoreCase) >= 0) - { - if (!m_options.ContainsKey("machine-name")) - extra["machine-name"] = Library.AutoUpdater.UpdaterManager.MachineName; - } + // Add the default values, if found in the template + foreach (var key in OPERATION_TEMPLATE_KEYS) + if (input.IndexOf($"%{key}%", StringComparison.OrdinalIgnoreCase) >= 0) + extra[key] = GetDefaultValue(key); + // Add any options that are whitelisted or used in the template foreach (KeyValuePair kv in m_options) - if (input.IndexOf($"%{kv.Key}%", StringComparison.OrdinalIgnoreCase) >= 0) + if (EXTRA_TEMPLATE_KEYS.Contains(kv.Key) || input.IndexOf($"%{kv.Key}%", StringComparison.OrdinalIgnoreCase) >= 0) extra[kv.Key] = kv.Value; + // Add any missing default values + foreach (var key in EXTRA_TEMPLATE_KEYS) + if (!extra.ContainsKey(key)) + extra[key] = GetDefaultValue(key); + return resultFormatSerializer.Serialize(result, exception, LogLines, extra); } else { - input = Regex.Replace(input, "\\%OPERATIONNAME\\%", m_operationname ?? "", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); - input = Regex.Replace(input, "\\%REMOTEURL\\%", m_remoteurl ?? "", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); - input = Regex.Replace(input, "\\%LOCALPATH\\%", m_localpath == null ? "" : string.Join(System.IO.Path.PathSeparator.ToString(), m_localpath), RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); - input = Regex.Replace(input, "\\%PARSEDRESULT\\%", m_parsedresultlevel ?? "", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + foreach (var key in OPERATION_TEMPLATE_KEYS) + input = Regex.Replace(input, $"%{key}%", GetDefaultValue(key) ?? "", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + if (subjectline) { input = Regex.Replace(input, "\\%RESULT\\%", m_parsedresultlevel ?? "", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); @@ -393,18 +469,11 @@ namespace Duplicati.Library.Modules.Builtin foreach (KeyValuePair kv in m_options) input = Regex.Replace(input, "\\%" + kv.Key + "\\%", kv.Value ?? "", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); - if (!m_options.ContainsKey("backup-name")) - input = Regex.Replace(input, "\\%backup-name\\%", System.IO.Path.GetFileNameWithoutExtension(Duplicati.Library.Utility.Utility.getEntryAssembly().Location) ?? "", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); - - if (!m_options.ContainsKey("machine-name")) - input = Regex.Replace(input, "\\%machine-name\\%", Library.AutoUpdater.UpdaterManager.MachineName, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); - - if (!m_options.ContainsKey("backup-id")) - input = Regex.Replace(input, "\\%backup-id\\%", Library.Utility.Utility.ByteArrayAsHexString(Library.Utility.Utility.RepeatedHashWithSalt(m_remoteurl, SALT)), RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); - - if (!m_options.ContainsKey("machine-id")) - input = Regex.Replace(input, "\\%machine-id\\%", Library.AutoUpdater.UpdaterManager.MachineID, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + foreach (var key in EXTRA_TEMPLATE_KEYS) + if (!m_options.ContainsKey(key)) + input = Regex.Replace(input, $"%{key}%", GetDefaultValue(key) ?? "", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + // Remove any remaining template keys input = Regex.Replace(input, "\\%[^\\%]+\\%", ""); return input; } diff --git a/Duplicati/Library/RestAPI/Scheduler.cs b/Duplicati/Library/RestAPI/Scheduler.cs index b356233ae..ba7bd40cf 100644 --- a/Duplicati/Library/RestAPI/Scheduler.cs +++ b/Duplicati/Library/RestAPI/Scheduler.cs @@ -357,8 +357,23 @@ namespace Duplicati.Server } else { + Dictionary taskOptions = null; + try + { + var nextRun = GetNextValidTime(start, + new DateTime( + Math.Max(DateTime.UtcNow.AddSeconds(1).Ticks, start.AddSeconds(1).Ticks), + DateTimeKind.Utc), sc.Repeat, sc.AllowedDays, timeZoneInfo); + + taskOptions = new Dictionary() + { { "next-scheduled-run", Utility.SerializeDateTime(nextRun.ToUniversalTime()) } }; + } + catch + { + } + jobsToRun.Add(Server.Runner.CreateTask( - Duplicati.Server.Serialization.DuplicatiOperation.Backup, entry)); + Serialization.DuplicatiOperation.Backup, entry, taskOptions)); } } } diff --git a/Duplicati/Library/Utility/MachineNameReader.cs b/Duplicati/Library/Utility/MachineNameReader.cs new file mode 100644 index 000000000..370e24e80 --- /dev/null +++ b/Duplicati/Library/Utility/MachineNameReader.cs @@ -0,0 +1,95 @@ +using System; +using System.Management; +using System.Runtime.Versioning; + +#nullable enable + +namespace Duplicati.Library.Utility; + +/// +/// Class for reading the machine name +/// +public static class MachineNameReader +{ + /// + /// Makes a best effort to get the machine name + /// + /// The machine name + public static string GetMachineName() + { + string? machineName = null; + if (OperatingSystem.IsWindows()) + machineName = GetMachineNameWindows(); + else if (OperatingSystem.IsMacOS()) + machineName = GetMachineNameMacOS(); + else if (OperatingSystem.IsLinux()) + machineName = GetMachineNameLinux(); + + return string.IsNullOrWhiteSpace(machineName) + ? Environment.MachineName + : machineName; + } + /// + /// Executes a command and reads the output + /// + /// The command to execute + /// The arguments to pass to the command + /// The output of the command + private static string ExecuteAndReadOutput(string command, string arguments) + { + try + { + var process = new System.Diagnostics.Process + { + StartInfo = new System.Diagnostics.ProcessStartInfo + { + FileName = command, + Arguments = arguments, + RedirectStandardOutput = true, + UseShellExecute = false, + CreateNoWindow = true + } + }; + process.Start(); + process.WaitForExit(TimeSpan.FromSeconds(1)); + if (!process.HasExited) + { + process.Kill(); + return string.Empty; + } + + if (process.ExitCode != 0) + return string.Empty; + + return process.StandardOutput.ReadToEnd().Trim(); + } + catch + { + return string.Empty; + } + } + + /// + /// Gets the machine name if running MacOS + /// + /// The machine name + [SupportedOSPlatform("macos")] + private static string? GetMachineNameMacOS() + => ExecuteAndReadOutput("scutil", "--get ComputerName"); + + /// + /// Gets the machine name if running Windows + /// + /// The machine name + [SupportedOSPlatform("windows")] + private static string? GetMachineNameWindows() + => null; // No special handling for Windows, always uses NetBIOS name + + /// + /// Gets the machine name if running Linux + /// + /// The machine name + [SupportedOSPlatform("linux")] + private static string? GetMachineNameLinux() + => null; // No special handling for Linux, always uses hostname +} diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index 1cfdd5df0..8525e1d27 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -1534,5 +1534,26 @@ namespace Duplicati.Library.Utility collection.Import(pfxPath, password, X509KeyStorageFlags.Exportable | X509KeyStorageFlags.PersistKeySet); return collection; } + + /// + /// Flattens an exception and its inner exceptions + /// + /// The exception to flatten + /// An enumerable of exceptions + public static IEnumerable FlattenException(Exception? ex) + { + if (ex == null) + yield break; + + yield return ex; + + if (ex is AggregateException aex) + foreach (var iex in aex.Flatten().InnerExceptions) + foreach (var iex2 in FlattenException(iex)) + yield return iex2; + + foreach (var iex in FlattenException(ex.InnerException)) + yield return iex; + } } } diff --git a/Duplicati/UnitTest/BorderTests.cs b/Duplicati/UnitTest/BorderTests.cs index 3c1bf59fb..0e4bc93fc 100644 --- a/Duplicati/UnitTest/BorderTests.cs +++ b/Duplicati/UnitTest/BorderTests.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -30,17 +30,6 @@ namespace Duplicati.UnitTest { public class BorderTests : BasicSetupHelper { - private readonly string recreatedDatabaseFile = Path.Combine(BASEFOLDER, "recreated-database.sqlite"); - - [TearDown] - public void TearDown() - { - if (File.Exists(this.recreatedDatabaseFile)) - { - File.Delete(this.recreatedDatabaseFile); - } - } - [Test] [Category("Border")] public void Run10kNoProgress() @@ -317,15 +306,17 @@ namespace Duplicati.UnitTest Assert.IsFalse(r.Verifications.Any(p => p.Value.Any())); } - testopts["dbpath"] = this.recreatedDatabaseFile; + var recreatedDatabaseFile = Path.Combine(BASEFOLDER, "recreated-database.sqlite"); + if (File.Exists(recreatedDatabaseFile)) + File.Delete(recreatedDatabaseFile); + + testopts["dbpath"] = recreatedDatabaseFile; using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) { IRepairResults repairResults = c.Repair(); Assert.AreEqual(0, repairResults.Errors.Count()); - - // TODO: This sometimes results in a "No block hash found for file: C:\projects\duplicati\testdata\backup-data\a-0" warning. - // Because of this, we don't check for warnings here. + Assert.AreEqual(0, repairResults.Warnings.Count()); } using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) diff --git a/Duplicati/UnitTest/CommandLineOperationsTests.cs b/Duplicati/UnitTest/CommandLineOperationsTests.cs index 9876377ae..2b22812ae 100644 --- a/Duplicati/UnitTest/CommandLineOperationsTests.cs +++ b/Duplicati/UnitTest/CommandLineOperationsTests.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -32,7 +32,7 @@ namespace Duplicati.UnitTest { public class CommandLineOperationsTests : BasicSetupHelper { - private static readonly string S3_URL = $"https://testfiles.duplicati.com/"; + public const string S3_URL = $"https://testfiles.duplicati.com/"; /// /// The log tag @@ -79,7 +79,7 @@ namespace Duplicati.UnitTest private void DownloadS3FileIfNewer(string destinationFilePath, string url, int retries = 5) => DownloadS3FileIfNewerAsync(destinationFilePath, url, retries).Await(); - private async Task DownloadS3FileIfNewerAsync(string destinationFilePath, string url, int retries = 5) + public static async Task DownloadS3FileIfNewerAsync(string destinationFilePath, string url, int retries = 5) { do { @@ -92,7 +92,6 @@ namespace Duplicati.UnitTest request.Headers.IfModifiedSince = systemIO.FileGetLastWriteTimeUtc(destinationFilePath); using var response = await httpClient.SendAsync(request); - using var tmpFile = new TempFile(); if (response.StatusCode == System.Net.HttpStatusCode.NotModified) { @@ -101,6 +100,7 @@ namespace Duplicati.UnitTest } else { + using var tmpFile = new TempFile(); Console.WriteLine($"Downloading file from {url} to: {tmpFile}"); var contentStream = await response.Content.ReadAsStreamAsync(); var fileInfo = new FileInfo(tmpFile); @@ -126,6 +126,13 @@ namespace Duplicati.UnitTest throw; await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false); + try + { + System.Net.Dns.GetHostEntry(new System.Uri(url).Host); + } + catch (Exception) + { + } Console.WriteLine($"Retrying download, {retries} retries left."); } } while (retries > 0); diff --git a/Duplicati/UnitTest/DisruptionTests.cs b/Duplicati/UnitTest/DisruptionTests.cs index f8041f1a8..c23a15124 100644 --- a/Duplicati/UnitTest/DisruptionTests.cs +++ b/Duplicati/UnitTest/DisruptionTests.cs @@ -65,8 +65,9 @@ namespace Duplicati.UnitTest { if (path.EndsWith(this.fileSizes[2] + "MB")) { - stopped.TrySetResult(true); + Thread.Sleep(500); controller.Stop(); + stopped.TrySetResult(true); } }; #endif @@ -102,7 +103,8 @@ namespace Duplicati.UnitTest // This allows us to inspect the dlist files without needing the BackendManager (which is inaccessible here) to decrypt them. ["no-encryption"] = "true", - ["disable-file-scanner"] = "true" + ["disable-file-scanner"] = "true", + ["concurrency-fileprocessors"] = "1", }; // Run a full backup. @@ -206,7 +208,8 @@ namespace Duplicati.UnitTest Dictionary options = new Dictionary(this.TestOptions) { ["dblock-size"] = "10mb", - ["disable-file-scanner"] = "true" + ["disable-file-scanner"] = "true", + ["concurrency-fileprocessors"] = "1", }; // First, run two complete backups followed by a partial backup. We will then set the keep-time @@ -300,6 +303,7 @@ namespace Duplicati.UnitTest { ["dblock-size"] = "10mb", ["disable-file-scanner"] = "true", + ["concurrency-fileprocessors"] = "1", }; // Run a full backup. @@ -394,7 +398,8 @@ namespace Duplicati.UnitTest { ["dblock-size"] = "10mb", ["no-local-db"] = "true", - ["disable-file-scanner"] = "true" + ["disable-file-scanner"] = "true", + ["concurrency-fileprocessors"] = "1", }; // Run a full backup. @@ -435,7 +440,8 @@ namespace Duplicati.UnitTest // This test assumes that we can perform 3 backups within 1 minute. ["retention-policy"] = "1m:59s,U:1m", ["no-backend-verification"] = "true", - ["disable-file-scanner"] = "true" + ["disable-file-scanner"] = "true", + ["concurrency-fileprocessors"] = "1" }; DateTime firstBackupTime; @@ -573,6 +579,7 @@ namespace Duplicati.UnitTest // Choose a dblock size that is small enough so that more than one volume is needed. ["dblock-size"] = "10mb", ["disable-file-scanner"] = "true", + ["concurrency-fileprocessors"] = "1" }; // Run a complete backup. @@ -676,7 +683,8 @@ namespace Duplicati.UnitTest { ["dblock-size"] = "10mb", ["disable-synthetic-filelist"] = "true", - ["disable-file-scanner"] = "true" + ["disable-file-scanner"] = "true", + ["concurrency-fileprocessors"] = "1", }; // Run a complete backup. diff --git a/Duplicati/UnitTest/GeneralBlackBoxTesting.cs b/Duplicati/UnitTest/GeneralBlackBoxTesting.cs index 419b87d08..8fd859395 100644 --- a/Duplicati/UnitTest/GeneralBlackBoxTesting.cs +++ b/Duplicati/UnitTest/GeneralBlackBoxTesting.cs @@ -1,30 +1,29 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using NUnit.Framework; using System.Linq; using System.IO; -using System; using System.Collections.Generic; -using System.Net; +using Duplicati.Library.Utility; namespace Duplicati.UnitTest { @@ -70,53 +69,7 @@ namespace Duplicati.UnitTest public void OneTimeSetUp() { this.OneTimeTearDown(); - var url = $"https://testfiles.duplicati.com/{this.zipFilename}"; - var destinationFilePath = this.zipFilepath; - var webRequest = (HttpWebRequest)WebRequest.Create(url); - - using (var wr = (HttpWebResponse)webRequest.GetResponse()) - using (WebClient client = new WebClient()) - { - Console.WriteLine("downloading test file to: {0}, length: {1}", destinationFilePath, wr.ContentLength); - var maxAttempts = 5; - while (maxAttempts-- > 0) - { - try - { - DateTime beginTime = DateTime.Now; - client.DownloadFile(url, destinationFilePath); - long length = new System.IO.FileInfo(destinationFilePath).Length; - Console.WriteLine("downloaded test file: {0}: length {1}, duration {2}", destinationFilePath, length, (DateTime.Now - beginTime).TotalSeconds); - if (length == wr.ContentLength) - { - maxAttempts = -1; - } - else - { - Console.WriteLine("invalid downloaded length {0}, should be {1}...", length, wr.ContentLength); - System.Threading.Thread.Sleep(120000); - } - } - catch (WebException ex) - { - if (ex.Response == null) - { - Console.WriteLine("ex.Response is null !"); - } - else - { - Console.WriteLine("exception {0}", ex); - throw; - } - } - } - - if (maxAttempts == 0) - { - throw new Exception(string.Format("Unable to download test file from {0}", url)); - } - } - + CommandLineOperationsTests.DownloadS3FileIfNewerAsync(this.zipFilepath, $"{CommandLineOperationsTests.S3_URL}{this.zipFilename}").Await(); System.IO.Compression.ZipFile.ExtractToDirectory(this.zipFilepath, BASEFOLDER); } @@ -127,10 +80,6 @@ namespace Duplicati.UnitTest { Directory.Delete(SOURCE_FOLDERS, true); } - if (File.Exists(this.zipFilepath)) - { - File.Delete(this.zipFilepath); - } } [Test] diff --git a/LiveTests/Duplicati.Backend.Tests/FTP/FTPTests.cs b/LiveTests/Duplicati.Backend.Tests/FTP/FTPTests.cs index b60ca858d..80abd0f23 100644 --- a/LiveTests/Duplicati.Backend.Tests/FTP/FTPTests.cs +++ b/LiveTests/Duplicati.Backend.Tests/FTP/FTPTests.cs @@ -50,7 +50,7 @@ public sealed class FtpTests : BaseSftpgoTest .WithEnvironment("SFTPGO_LOADDATA_FROM", "/var/lib/sftpgo/users.json") .WithEnvironment("SFTPGO_LOADDATA_CLEAN", "0") .WithEnvironment("SFTPGO_FTPD__BINDINGS__0__PORT", "2121") - .WithEnvironment("SFTPGO_FTPD__PASSIVE_PORT_RANGE__END", "21000") + .WithEnvironment("SFTPGO_FTPD__PASSIVE_PORT_RANGE__END", "20999") .WithEnvironment("SFTPGO_FTPD__PASSIVE_PORT_RANGE__START", "20000") .WithEnvironment("SFTPGO_FTPD__BINDINGS__0__ACTIVE_CONNECTIONS_SECURITY", "1") .WithEnvironment("SFTPGO_FTPD__BINDINGS__0__CERTIFICATE_FILE", CERTIFICATE_FILE) @@ -60,13 +60,13 @@ public sealed class FtpTests : BaseSftpgoTest .WithOutputConsumer(outputConsumer); - container = Enumerable.Range(20000, 1000) + container = Enumerable.Range(20000, 1001) .Aggregate(container, (current, port) => current.WithPortBinding(port, port)); var builtContainer = container .WithOutputConsumer(outputConsumer) .Build(); - + Console.WriteLine("Starting container"); await builtContainer.StartAsync(); @@ -88,7 +88,7 @@ public sealed class FtpTests : BaseSftpgoTest Console.WriteLine(await outputConsumer.GetStreamsOutput()); Assert.Fail("BackendTester is returning non-zero exit code, check logs for details"); } - + await builtContainer.StopAsync(); } @@ -114,7 +114,7 @@ public sealed class FtpTests : BaseSftpgoTest .WithEnvironment("SFTPGO_LOADDATA_FROM", "/var/lib/sftpgo/users.json") .WithEnvironment("SFTPGO_LOADDATA_CLEAN", "0") .WithEnvironment("SFTPGO_FTPD__BINDINGS__0__PORT", "2121") - .WithEnvironment("SFTPGO_FTPD__PASSIVE_PORT_RANGE__END", "23000") + .WithEnvironment("SFTPGO_FTPD__PASSIVE_PORT_RANGE__END", "22999") .WithEnvironment("SFTPGO_FTPD__PASSIVE_PORT_RANGE__START", "22000") .WithEnvironment("SFTPGO_FTPD__BINDINGS__0__ACTIVE_CONNECTIONS_SECURITY", "1") .WithEnvironment("SFTPGO_FTPD__BINDINGS__0__CERTIFICATE_FILE", CERTIFICATE_FILE) @@ -152,5 +152,5 @@ public sealed class FtpTests : BaseSftpgoTest await builtContainer.StopAsync(); } - + } \ No newline at end of file