diff --git a/Duplicati/Library/Interface/IRestoreCallbackModule.cs b/Duplicati/Library/Interface/IRestoreCallbackModule.cs new file mode 100644 index 000000000..36ac13d7e --- /dev/null +++ b/Duplicati/Library/Interface/IRestoreCallbackModule.cs @@ -0,0 +1,83 @@ +// Copyright (C) 2026, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace Duplicati.Library.Interface +{ + /// + /// Interface for implementing restore-specific callback modules. + /// + /// Modules implementing this interface are notified at key points during a + /// restore operation: when the priority-files list is being prepared (allowing + /// the module to modify it), when the bulk restore starts (after all priority + /// files have been restored), and for each individual file that is restored. + /// + /// The per-file callback () is only invoked when + /// the new restore engine is used (i.e. when --restore-legacy is not + /// set). The and + /// callbacks are invoked for both the legacy + /// and the new restore engines. + /// + public interface IRestoreCallbackModule : IGenericModule + { + /// + /// Called when the priority-files list has been collected from the restore + /// destination, but before any files are restored. The module is free to + /// inspect and modify the list in place (add, remove or reorder entries) + /// to influence which files are restored first. + /// + /// The list of priority file names. The module may modify this list. + /// The 0-based backup version index being restored (0 = newest). + /// The timestamp of the backup version being restored, in UTC. + /// A token to monitor for cancellation requests. + /// A task that represents the asynchronous operation. + Task OnPreparePriorityFilesAsync(IList priorityFiles, long version, DateTime backupTimestamp, CancellationToken cancellationToken); + + /// + /// Called once all priority files have been restored and the bulk restore + /// of the remaining files is about to start. If there are no priority + /// files, this is called before the single bulk restore pass begins. + /// + /// A token to monitor for cancellation requests. + /// A task that represents the asynchronous operation. + Task OnBulkRestoreStartAsync(CancellationToken cancellationToken); + + /// + /// Called for each restore target that has been restored, with the backup version + /// index (0 being the newest backup) the target was restored from, the + /// timestamp of that backup version, and the target path. The callback fires for + /// regular files as well as folders (whose metadata is restored) and alternate data + /// streams; implementations that only care about regular files should filter by + /// path themselves. This is only invoked when the new restore engine is used (i.e. + /// --restore-legacy is not set). + /// + /// The 0-based backup version index the target was restored from (0 = newest). + /// The target path of the restored file, folder or alternate data stream. + /// The timestamp of the backup version the target was restored from, in UTC. + /// A token to monitor for cancellation requests. + /// A task that represents the asynchronous operation. + Task OnFileRestoredAsync(long version, string path, DateTime backupTimestamp, CancellationToken cancellationToken); + } +} diff --git a/Duplicati/Library/Main/Operation/Restore/FileLister.cs b/Duplicati/Library/Main/Operation/Restore/FileLister.cs index 439ae9fed..a5a91d259 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileLister.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileLister.cs @@ -25,6 +25,7 @@ using System.Diagnostics; using System.Linq; using System.Threading.Tasks; using CoCoL; +using Duplicati.Library.Interface; using Duplicati.Library.Main.Database.Local; using Duplicati.Library.Common.IO; @@ -51,8 +52,11 @@ namespace Duplicati.Library.Main.Operation.Restore /// The restore database, which is queried for the file list. /// The restore options /// The restore results - /// The restore destination provider, used to get priority files. - public static Task RunAsync(Channels channels, LocalRestoreDatabase db, Options options, RestoreResults result, IList priorityFiles) + /// The list of priority file names to include. + /// The 0-based backup version index being restored (0 = newest), reported to restore callback modules. + /// The timestamp of the backup version being restored, in UTC, reported to restore callback modules. + /// The loaded generic modules, used to dispatch the bulk-restore-start callback. May be null. + public static Task RunAsync(Channels channels, LocalRestoreDatabase db, Options options, RestoreResults result, IList priorityFiles, long version, DateTime backupTimestamp, IEnumerable? modules) { return AutomationExtensions.RunTask( new @@ -102,31 +106,51 @@ namespace Duplicati.Library.Main.Operation.Restore sw_write_file?.Start(); - // First, send priority files (if any) to be restored first + // Resolve which files are priority files. A priority entry only counts + // if it matches at least one restorable target; entries a restore callback + // module added that match nothing are ignored so the priority-file counter + // below stays consistent with the number of priority FileRequests actually + // emitted. Without this, a non-matching entry would leave the non-priority + // FileProcessors waiting forever for a priority file that never arrives. + List priorityFileList; + IEnumerable remainingFiles; if (priorityFiles.Count > 0) { var priorityFileSet = new HashSet(priorityFiles, StringComparer.OrdinalIgnoreCase); - var priorityFileList = files.Where(f => priorityFileSet.Any(pf => f.TargetPath.EndsWith(pf, StringComparison.OrdinalIgnoreCase))).ToList(); - var remainingFiles = files.Where(f => !priorityFileSet.Any(pf => f.TargetPath.EndsWith(pf, StringComparison.OrdinalIgnoreCase))).ToList(); - - // Send priority files first (marked with IsPriorityFile=true) - foreach (var file in priorityFileList) - { - var priorityFileRequest = new FileRequest(file.ID, file.OriginalPath, file.TargetPath, file.Hash, file.Length, file.BlocksetID, IsPriorityFile: true); - await self.Output.WriteAsync(priorityFileRequest).ConfigureAwait(false); - } - - // Then send remaining files - foreach (var file in remainingFiles) - await self.Output.WriteAsync(file).ConfigureAwait(false); + priorityFileList = files.Where(f => priorityFileSet.Any(pf => f.TargetPath.EndsWith(pf, StringComparison.OrdinalIgnoreCase))).ToList(); + remainingFiles = files.Where(f => !priorityFileSet.Any(pf => f.TargetPath.EndsWith(pf, StringComparison.OrdinalIgnoreCase))); } else { - // No priority files, send all files in order - foreach (var file in files) - await self.Output.WriteAsync(file).ConfigureAwait(false); + priorityFileList = new List(); + remainingFiles = files; } + // The priority-file counter must reflect the number of priority + // FileRequests actually emitted below, not the size of the (module-mutable) + // priority-files list. Resetting the completion source here (the producer) + // is safe: the FileProcessors only await it for non-priority files, which are + // emitted afterwards, and they cannot read a file until it has been written. + FileProcessor.priority_files_remaining = priorityFileList.Count; + FileProcessor.priority_files_completed = new TaskCompletionSource(); + + // When no priority files will be processed, the bulk restore starts + // immediately; notify restore callback modules now. When priority files + // exist, the FileProcessor that finishes the last one notifies the modules. + if (priorityFileList.Count == 0) + await RestoreHandler.InvokeBulkRestoreStartAsync(modules, result.TaskControl.ProgressToken).ConfigureAwait(false); + + // Send priority files first (marked with IsPriorityFile=true) + foreach (var file in priorityFileList) + { + var priorityFileRequest = new FileRequest(file.ID, file.OriginalPath, file.TargetPath, file.Hash, file.Length, file.BlocksetID, IsPriorityFile: true, Version: version, BackupTimestamp: backupTimestamp); + await self.Output.WriteAsync(priorityFileRequest).ConfigureAwait(false); + } + + // Then send remaining files + foreach (var file in remainingFiles) + await self.Output.WriteAsync(file.WithVersion(version, backupTimestamp)).ConfigureAwait(false); + sw_write_file?.Stop(); if (!options.SkipMetadata) @@ -141,14 +165,14 @@ namespace Duplicati.Library.Main.Operation.Restore sw_write_folder?.Start(); foreach (var folder in folders) - await self.Output.WriteAsync(folder).ConfigureAwait(false); + await self.Output.WriteAsync(folder.WithVersion(version, backupTimestamp)).ConfigureAwait(false); sw_write_folder?.Stop(); } // Send the alternate data streams last, so their hosts are restored sw_write_file?.Start(); foreach (var file in adsStreams) - await self.Output.WriteAsync(new FileRequest(file.ID, file.OriginalPath, file.TargetPath, file.Hash, file.Length, file.BlocksetID, IsAlternateDataStream: true)).ConfigureAwait(false); + await self.Output.WriteAsync(new FileRequest(file.ID, file.OriginalPath, file.TargetPath, file.Hash, file.Length, file.BlocksetID, IsAlternateDataStream: true, Version: version, BackupTimestamp: backupTimestamp)).ConfigureAwait(false); sw_write_file?.Stop(); } catch (Exception ex) diff --git a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs index 42c346647..5e7cecbcf 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs @@ -80,7 +80,8 @@ namespace Duplicati.Library.Main.Operation.Restore /// The channel to receive blocks from the block manager. /// The restore options. /// The restore results. - public static Task RunAsync(Channels channels, LocalRestoreDatabase db, IChannel block_request, IChannel> block_response, IRestoreDestinationProvider restoreDestination, Options options, RestoreResults results) + /// The loaded generic modules, used to dispatch restore callbacks. May be null. + public static Task RunAsync(Channels channels, LocalRestoreDatabase db, IChannel block_request, IChannel> block_response, IRestoreDestinationProvider restoreDestination, Options options, RestoreResults results, IEnumerable? modules) { return AutomationExtensions.RunTask( new @@ -125,7 +126,7 @@ namespace Duplicati.Library.Main.Operation.Restore sw_file?.Stop(); // Check if this is a priority file and wait for all priority files to complete before processing non-priority files - await WaitForPriorityFilesIfNeededAsync(file).ConfigureAwait(false); + await WaitForPriorityFilesIfNeededAsync(file, modules, results.TaskControl.ProgressToken).ConfigureAwait(false); Logging.Log.WriteExplicitMessage(LOGTAG, "FileRestored", null, "{0} Restoring file {1}", my_id, file.TargetPath); @@ -548,6 +549,8 @@ namespace Duplicati.Library.Main.Operation.Restore } sw_work_results?.Stop(); + await RestoreHandler.InvokeFileRestoredAsync(modules, file.Version, file.TargetPath, file.BackupTimestamp, results.TaskControl.ProgressToken).ConfigureAwait(false); + Logging.Log.WriteVerboseMessage(LOGTAG, "RestoredFile", $"{my_id} Restored file {{0}}", file.TargetPath); } } @@ -693,9 +696,13 @@ namespace Duplicati.Library.Main.Operation.Restore /// Waits for all priority files to be processed before allowing non-priority files to be processed. /// If the current file is a priority file, it decrements the counter and signals completion when all priority files are done. /// If the current file is not a priority file, it waits until all priority files have been processed. + /// When the last priority file is processed, is + /// invoked to notify restore callback modules that the bulk restore is starting. /// /// The file request being processed. - private static async Task WaitForPriorityFilesIfNeededAsync(FileRequest file) + /// The loaded generic modules, used to dispatch the bulk-restore-start callback. May be null. + /// A token to monitor for cancellation requests. + private static async Task WaitForPriorityFilesIfNeededAsync(FileRequest file, IEnumerable? modules, CancellationToken cancellationToken) { if (file.IsPriorityFile) { @@ -710,7 +717,17 @@ namespace Duplicati.Library.Main.Operation.Restore if (shouldSignal) { - priority_files_completed.SetResult(); + // All priority files have been restored and the bulk restore is starting. + // The first processor to detect completion notifies the restore callback + // modules before signaling the other processors to continue. + try + { + await RestoreHandler.InvokeBulkRestoreStartAsync(modules, cancellationToken).ConfigureAwait(false); + } + finally + { + priority_files_completed.SetResult(); + } } } else diff --git a/Duplicati/Library/Main/Operation/Restore/Interfaces.cs b/Duplicati/Library/Main/Operation/Restore/Interfaces.cs index 5927be71d..e494fcdfd 100644 --- a/Duplicati/Library/Main/Operation/Restore/Interfaces.cs +++ b/Duplicati/Library/Main/Operation/Restore/Interfaces.cs @@ -22,6 +22,7 @@ using System; using System.Buffers; using System.Threading; +using Duplicati.Library.Interface; using Duplicati.Library.Main.Volumes; using Duplicati.Library.Utility; @@ -177,7 +178,9 @@ namespace Duplicati.Library.Main.Operation.Restore /// The BlocksetID of the file. /// Whether this is a priority file that should be processed before other files. /// Whether this request is for an alternate data stream that must be restored after its host file/folder. - public class FileRequest(long ID, string OriginalPath, string TargetPath, string Hash, long Length, long BlocksetID, bool IsPriorityFile = false, bool IsAlternateDataStream = false) + /// The 0-based backup version index this file is being restored from (0 = newest). Defaults to 0. + /// The timestamp of the backup version this file is being restored from, in UTC. Defaults to . + public class FileRequest(long ID, string OriginalPath, string TargetPath, string Hash, long Length, long BlocksetID, bool IsPriorityFile = false, bool IsAlternateDataStream = false, long Version = 0, DateTime BackupTimestamp = default) { public long ID { get; } = ID; public string OriginalPath { get; } = OriginalPath; @@ -187,6 +190,27 @@ namespace Duplicati.Library.Main.Operation.Restore public long BlocksetID { get; } = BlocksetID; public bool IsPriorityFile { get; } = IsPriorityFile; public bool IsAlternateDataStream { get; } = IsAlternateDataStream; + /// + /// The 0-based backup version index this file is being restored from (0 = newest). + /// Used to report the version to modules. + /// + public long Version { get; } = Version; + /// + /// The timestamp of the backup version this file is being restored from, in UTC. + /// Used to report the backup timestamp to modules. + /// + public DateTime BackupTimestamp { get; } = BackupTimestamp; + + /// + /// Returns a copy of this with the + /// and fields set to the supplied values, keeping all + /// other fields unchanged. + /// + /// The 0-based backup version index to set. + /// The backup version timestamp (UTC) to set. + /// A copy of this request with the version and backup timestamp updated. + public FileRequest WithVersion(long version, DateTime backupTimestamp) + => new FileRequest(ID, OriginalPath, TargetPath, Hash, Length, BlocksetID, IsPriorityFile, IsAlternateDataStream, version, backupTimestamp); } } \ No newline at end of file diff --git a/Duplicati/Library/Main/Operation/RestoreHandler.cs b/Duplicati/Library/Main/Operation/RestoreHandler.cs index bf08fd98b..d31b8f373 100644 --- a/Duplicati/Library/Main/Operation/RestoreHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreHandler.cs @@ -228,10 +228,16 @@ namespace Duplicati.Library.Main.Operation m_options.RawOptions["version"] = string.Join(",", Enumerable.Range(0, versions.Length).Select(x => x.ToString())); } + // Resolve the backup version index (0 = newest) and timestamp (UTC) for the + // version being restored, forwarding them to the restore callback modules so + // the engine methods below do not need to look them up. When the caller already + // knows the timestamp (the --restore-all-files path), it is used directly. + var (restoreVersion, restoreBackupTimestamp) = await ResolveRestoreVersionAndTimestampAsync(db, versions, time, cancellationToken).ConfigureAwait(false); + if (m_options.RestoreLegacy) - await DoRunAsync(backendManager, db, filter, restoreDestination, cancellationToken).ConfigureAwait(false); + await DoRunAsync(backendManager, db, filter, restoreDestination, restoreVersion, restoreBackupTimestamp, cancellationToken).ConfigureAwait(false); else - await DoRunNewAsync(backendManager, db, filter, restoreDestination, cancellationToken).ConfigureAwait(false); + await DoRunNewAsync(backendManager, db, filter, restoreDestination, restoreVersion, restoreBackupTimestamp, cancellationToken).ConfigureAwait(false); } finally { @@ -340,6 +346,9 @@ namespace Duplicati.Library.Main.Operation // provider's path validation continues to accept them. var versionDestination = new VersionedRestoreDestinationProvider(restoreDestination, versionTarget); + // Forward the already-known version index so the engine methods and restore + // callback modules receive it without any lookup. The backup timestamp is + // resolved from the fileset list inside RunSingleVersionRestoreAsync. await RunSingleVersionRestoreAsync(backendManager, filter, versionDestination, m_options.Time, [fileset.VersionIndex], cancellationToken) .ConfigureAwait(false); } @@ -666,6 +675,140 @@ namespace Duplicati.Library.Main.Operation } } + /// + /// The tag used for logging restore-callback module invocations. + /// + private static readonly string RESTORE_CALLBACK_LOGTAG = Logging.Log.LogTagFromType() + ".RestoreCallback"; + + /// + /// Invokes on every loaded + /// module, passing the backup version index, the backup + /// timestamp and the restored file's target path. Exceptions are logged and isolated. + /// + /// The loaded generic modules, in their activation order. + /// The 0-based backup version index the file was restored from (0 = newest). + /// The target path of the restored file. + /// The timestamp of the backup version the file was restored from, in UTC. + /// A token to monitor for cancellation requests. + /// A task that represents the asynchronous dispatch operation. + internal static async Task InvokeFileRestoredAsync(IEnumerable? modules, long version, string path, DateTime backupTimestamp, CancellationToken cancellationToken) + { + if (modules == null) + return; + foreach (var mx in modules) + { + if (mx is not IRestoreCallbackModule module) + continue; + try { await module.OnFileRestoredAsync(version, path, backupTimestamp, cancellationToken).ConfigureAwait(false); } + catch (Exception ex) { Logging.Log.WriteWarningMessage(RESTORE_CALLBACK_LOGTAG, $"OnFileRestoredError{mx.Key}", ex, "OnFileRestored callback {0} failed: {1}", mx.Key, ex.Message); } + } + } + + /// + /// Invokes on every loaded + /// module, passing the priority-files list by reference + /// so the modules may modify it, along with the backup version index and timestamp being + /// restored. Exceptions thrown by a module are logged and isolated so a single misbehaving + /// module cannot abort the restore. + /// + /// The loaded generic modules, in their activation order. + /// The priority-files list that modules may modify in place. + /// The 0-based backup version index being restored (0 = newest). + /// The timestamp of the backup version being restored, in UTC. + /// A token to monitor for cancellation requests. + /// A task that represents the asynchronous dispatch operation. + internal static async Task InvokePreparePriorityFilesAsync(IEnumerable? modules, IList priorityFiles, long version, DateTime backupTimestamp, CancellationToken cancellationToken) + { + if (modules == null) + return; + foreach (var mx in modules) + { + if (mx is not IRestoreCallbackModule module) + continue; + try { await module.OnPreparePriorityFilesAsync(priorityFiles, version, backupTimestamp, cancellationToken).ConfigureAwait(false); } + catch (Exception ex) { Logging.Log.WriteWarningMessage(RESTORE_CALLBACK_LOGTAG, $"OnPreparePriorityFilesError{mx.Key}", ex, "OnPreparePriorityFiles callback {0} failed: {1}", mx.Key, ex.Message); } + } + } + + /// + /// Invokes on every loaded + /// module. Exceptions are logged and isolated. + /// + /// The loaded generic modules, in their activation order. + /// A token to monitor for cancellation requests. + /// A task that represents the asynchronous dispatch operation. + internal static async Task InvokeBulkRestoreStartAsync(IEnumerable? modules, CancellationToken cancellationToken) + { + if (modules == null) + return; + foreach (var mx in modules) + { + if (mx is not IRestoreCallbackModule module) + continue; + try { await module.OnBulkRestoreStartAsync(cancellationToken).ConfigureAwait(false); } + catch (Exception ex) { Logging.Log.WriteWarningMessage(RESTORE_CALLBACK_LOGTAG, $"OnBulkRestoreStartError{mx.Key}", ex, "OnBulkRestoreStart callback {0} failed: {1}", mx.Key, ex.Message); } + } + } + + /// + /// Resolves the 0-based backup version index (0 = newest) and the backup timestamp + /// (UTC) for the version being restored, so the value can be forwarded to the restore + /// callback modules rather than reverse-looked-up inside the engine methods. + /// + /// + /// This is a forward resolution: it reads the fileset list (newest first) once and + /// picks the entry matching the requested /. + /// + /// The restore database. + /// The requested version indices (0 = newest), or null for a time-based restore. + /// The restore time, used when is null/empty. + /// The cancellation token to monitor for cancellation requests. + /// A tuple with the 0-based backup version index and the backup timestamp (UTC). + private static async Task<(long version, DateTime backupTimestamp)> ResolveRestoreVersionAndTimestampAsync(LocalRestoreDatabase database, long[]? versions, DateTime time, CancellationToken cancellationToken) + { + try + { + // Filesets ordered newest-first; the 0-based position is the version index. + var filesets = await database.FilesetTimesAsync(cancellationToken) + .ToArrayAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false); + + if (filesets.Length == 0) + return (0, DateTime.MinValue); + + // Explicit version indices take precedence: the caller passes the absolute index. + if (versions is { Length: > 0 }) + { + var requestedIndex = versions[0]; + // On a recreated (no-local-db) database the list may contain fewer entries + // than the absolute index; the single present entry is the requested version. + var timestamp = requestedIndex >= 0 && requestedIndex < filesets.Length + ? filesets[requestedIndex].Value + : filesets[0].Value; + return (requestedIndex, timestamp.ToUniversalTime()); + } + + // Time-based restore: pick the newest fileset at or before the requested time. + if (time.Ticks > 0) + { + var compareTime = time.ToUniversalTime(); + for (int i = 0; i < filesets.Length; i++) + { + if (filesets[i].Value.ToUniversalTime() <= compareTime) + return (i, filesets[i].Value.ToUniversalTime()); + } + } + + // Default: the newest backup (version 0). + return (0, filesets[0].Value.ToUniversalTime()); + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "ResolveRestoreVersionFailed", ex, "Failed to resolve restore version info: {0}", ex.Message); + return (0, DateTime.MinValue); + } + } + /// /// Perform the restore operation. /// This is the new implementation, which utilizes a CSP network of processes to perform the restore. @@ -673,8 +816,10 @@ namespace Duplicati.Library.Main.Operation /// The database containing information about the restore. /// The filter of which files to restore. /// The destination to restore to. + /// The forwarded 0-based backup version index being restored (0 = newest), reported to restore callback modules. + /// The forwarded backup timestamp (UTC) being restored, reported to restore callback modules. /// The cancellation token to monitor for cancellation requests. - private async Task DoRunNewAsync(IBackendManager backendManager, LocalRestoreDatabase database, IFilter? filter, IRestoreDestinationProvider restoreDestination, CancellationToken cancellationToken) + private async Task DoRunNewAsync(IBackendManager backendManager, LocalRestoreDatabase database, IFilter? filter, IRestoreDestinationProvider restoreDestination, long restoreVersion, DateTime restoreBackupTimestamp, CancellationToken cancellationToken) { // Perform initial setup await Utility.UpdateOptionsFromDbAsync(database, m_options, cancellationToken) @@ -724,14 +869,17 @@ namespace Duplicati.Library.Main.Operation Restore.DeadlockTimer.initial_threshold = (int)TimeSpan.FromMinutes(1).TotalMilliseconds * Math.Max(1, (int)(volsize / (10L * 1024L * 1024L))); Restore.FileProcessor.file_processors_restoring_files = m_options.RestoreFileProcessors; - // Initialize priority files synchronization - var priorityFiles = restoreDestination.GetPriorityFiles(); - Restore.FileProcessor.priority_files_remaining = priorityFiles.Count; - Restore.FileProcessor.priority_files_completed = new TaskCompletionSource(); + // Initialize priority files synchronization. + // Wrap in a List<> so restore callback modules can freely modify the list + // (the destination provider may return a fixed-size array). + var priorityFiles = new List(restoreDestination.GetPriorityFiles()); + // Allow restore callback modules to inspect and modify the priority-files list + // before any files are restored, passing the forwarded version and backup timestamp. + await InvokePreparePriorityFilesAsync(m_options.LoadedModules, priorityFiles, restoreVersion, restoreBackupTimestamp, cancellationToken).ConfigureAwait(false); // Create the process network - var filelister = Restore.FileLister.RunAsync(channels, database, m_options, m_result, priorityFiles); - var fileprocessors = Enumerable.Range(0, m_options.RestoreFileProcessors).Select(i => Restore.FileProcessor.RunAsync(channels, database, fileprocessor_requests[i], fileprocessor_responses[i], restoreDestination, m_options, m_result)).ToArray(); + var filelister = Restore.FileLister.RunAsync(channels, database, m_options, m_result, priorityFiles, restoreVersion, restoreBackupTimestamp, m_options.LoadedModules); + var fileprocessors = Enumerable.Range(0, m_options.RestoreFileProcessors).Select(i => Restore.FileProcessor.RunAsync(channels, database, fileprocessor_requests[i], fileprocessor_responses[i], restoreDestination, m_options, m_result, m_options.LoadedModules)).ToArray(); var blockmanager = Restore.BlockManager.RunAsync(channels, database, fileprocessor_requests, fileprocessor_responses, m_options, m_result); var volumecache = Restore.VolumeManager.RunAsync(channels, m_options, m_result); var volumedownloaders = Enumerable.Range(0, m_options.RestoreVolumeDownloaders).Select(i => Restore.VolumeDownloader.RunAsync(channels, database, backendManager, m_options, m_result)).ToArray(); @@ -856,7 +1004,7 @@ namespace Duplicati.Library.Main.Operation /// The filter of which files to restore. /// The destination to restore to. /// The cancellation token to monitor for cancellation requests. - private async Task DoRunAsync(IBackendManager backendManager, LocalRestoreDatabase database, Library.Utility.IFilter? filter, IRestoreDestinationProvider restoreDestination, CancellationToken cancellationToken) + private async Task DoRunAsync(IBackendManager backendManager, LocalRestoreDatabase database, Library.Utility.IFilter? filter, IRestoreDestinationProvider restoreDestination, long restoreVersion, DateTime restoreBackupTimestamp, CancellationToken cancellationToken) { using (var metadatastorage = new RestoreHandlerMetadataStorage()) { @@ -873,8 +1021,13 @@ namespace Duplicati.Library.Main.Operation await FilelistProcessor.VerifyRemoteListAsync(backendManager, m_options, database, m_result.BackendWriter, latestVolumesOnly: false, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly, cancellationToken).ConfigureAwait(false); } - // Get priority files from restore destination - var priorityFiles = restoreDestination.GetPriorityFiles(); + // Get priority files from restore destination. + // Wrap in a List<> so restore callback modules can freely modify the list + // (the destination provider may return a fixed-size array). + var priorityFiles = new List(restoreDestination.GetPriorityFiles()); + // Allow restore callback modules to inspect and modify the priority-files list + // before any files are restored, passing the forwarded version and backup timestamp. + await InvokePreparePriorityFilesAsync(m_options.LoadedModules, priorityFiles, restoreVersion, restoreBackupTimestamp, cancellationToken).ConfigureAwait(false); if (priorityFiles.Count > 0) { @@ -897,13 +1050,16 @@ namespace Duplicati.Library.Main.Operation // Phase 2: Restore remaining files (excluding priority files) Logging.Log.WriteInformationMessage(LOGTAG, "RemainingFilesRestore", "Restoring remaining files (excluding priority files)"); + // All priority files have been restored and the bulk restore is starting. + await InvokeBulkRestoreStartAsync(m_options.LoadedModules, cancellationToken).ConfigureAwait(false); var phase2Filter = BuildExcludePriorityFilter(priorityFiles, filter); await RestoreCoreAsync(backendManager, database, phase2Filter, restoreDestination, metadatastorage, cancellationToken) .ConfigureAwait(false); } else { - // No priority files — single phase + // No priority files — single phase, the bulk restore starts immediately. + await InvokeBulkRestoreStartAsync(m_options.LoadedModules, cancellationToken).ConfigureAwait(false); await RestoreCoreAsync(backendManager, database, filter, restoreDestination, metadatastorage, cancellationToken) .ConfigureAwait(false); } diff --git a/Duplicati/UnitTest/RestoreCallbackModuleTests.cs b/Duplicati/UnitTest/RestoreCallbackModuleTests.cs new file mode 100644 index 000000000..23cfda465 --- /dev/null +++ b/Duplicati/UnitTest/RestoreCallbackModuleTests.cs @@ -0,0 +1,397 @@ +// Copyright (C) 2026, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Duplicati.Library.DynamicLoader; +using Duplicati.Library.Interface; +using Duplicati.Library.Main; +using Duplicati.Library.Main.Operation; +using Duplicati.Library.Main.Operation.Restore; +using NUnit.Framework; +using Assert = NUnit.Framework.Legacy.ClassicAssert; + +#nullable enable + +namespace Duplicati.UnitTest +{ + /// + /// Tests for the restore hook interface and its + /// wiring in / the restore FileProcessor. + /// + [NonParallelizable] + public class RestoreCallbackModuleTests : BasicSetupHelper + { + /// + /// A minimal implementation that records calls + /// on the instance, used for the dispatch-helper unit tests. + /// + private sealed class FakeRestoreCallbackModule : IRestoreCallbackModule, IGenericPriorityModule + { + public string Key { get; set; } = "fake"; + public string DisplayName => "Fake"; + public string Description => "Fake restore callback module"; + public bool LoadAsDefault => false; + public int Priority { get; set; } + public IList SupportedCommands => new List(); + + public List<(string files, long version, DateTime timestamp)> PrepareCalls { get; } = new(); + public int BulkStartCalls { get; set; } + public List<(long version, string path, DateTime timestamp)> FileRestoredCalls { get; } = new(); + + public Action>? OnPrepare { get; set; } + public Action? OnBulk { get; set; } + public Action? OnFile { get; set; } + + public Task OnPreparePriorityFilesAsync(IList priorityFiles, long version, DateTime backupTimestamp, CancellationToken cancellationToken) + { + PrepareCalls.Add((string.Join(",", priorityFiles), version, backupTimestamp)); + OnPrepare?.Invoke(priorityFiles); + return Task.CompletedTask; + } + public Task OnBulkRestoreStartAsync(CancellationToken cancellationToken) + { + BulkStartCalls++; + OnBulk?.Invoke(); + return Task.CompletedTask; + } + public Task OnFileRestoredAsync(long version, string path, DateTime backupTimestamp, CancellationToken cancellationToken) + { + FileRestoredCalls.Add((version, path, backupTimestamp)); + OnFile?.Invoke(version, path); + return Task.CompletedTask; + } + + public void Configure(IDictionary commandlineOptions) { } + } + + /// + /// A module that is NOT a restore callback module, used to verify the dispatch + /// helpers skip non-matching modules. + /// + private sealed class NonCallbackModule : IGenericModule + { + public string Key => "non-callback"; + public string DisplayName => "NonCallback"; + public string Description => ""; + public bool LoadAsDefault => false; + public IList SupportedCommands => new List(); + public void Configure(IDictionary commandlineOptions) { } + } + + [Test] + [Category("RestoreHandler")] + public async Task InvokePreparePriorityFilesCallsModulesAndAllowsMutationAsync() + { + var modA = new FakeRestoreCallbackModule { Key = "a" }; + var modB = new FakeRestoreCallbackModule { Key = "b" }; + modA.OnPrepare = list => list.Add("from-a"); + modB.OnPrepare = list => list.Remove("remove-me"); + + var priorityFiles = new List { "keep", "remove-me" }; + // Mix in a non-callback module to ensure it is skipped. + const long expectedVersion = 7; + var expectedTimestamp = new DateTime(2024, 1, 2, 3, 4, 5, DateTimeKind.Utc); + await RestoreHandler.InvokePreparePriorityFilesAsync(new IGenericModule[] { new NonCallbackModule(), modA, modB }, priorityFiles, expectedVersion, expectedTimestamp, CancellationToken.None); + + // Both callback modules were invoked exactly once, in order. + Assert.AreEqual(1, modA.PrepareCalls.Count); + Assert.AreEqual(1, modB.PrepareCalls.Count); + // The version and backup timestamp were forwarded to the modules. + Assert.AreEqual(expectedVersion, modA.PrepareCalls[0].version); + Assert.AreEqual(expectedTimestamp, modA.PrepareCalls[0].timestamp); + Assert.AreEqual(expectedVersion, modB.PrepareCalls[0].version); + Assert.AreEqual(expectedTimestamp, modB.PrepareCalls[0].timestamp); + // modA saw the original list, modB saw the list after modA mutated it. + Assert.AreEqual("keep,remove-me", modA.PrepareCalls[0].files); + Assert.AreEqual("keep,remove-me,from-a", modB.PrepareCalls[0].files); + + // The shared list reflects both mutations. + Assert.That(priorityFiles, Is.EqualTo(new[] { "keep", "from-a" })); + } + + [Test] + [Category("RestoreHandler")] + public async Task InvokePreparePriorityFilesIsolatedFromThrowingModulesAsync() + { + var throwing = new FakeRestoreCallbackModule { Key = "throw", OnPrepare = _ => throw new InvalidOperationException("boom") }; + var after = new FakeRestoreCallbackModule { Key = "after" }; + + var priorityFiles = new List { "a" }; + await RestoreHandler.InvokePreparePriorityFilesAsync(new IGenericModule[] { throwing, after }, priorityFiles, 0, DateTime.MinValue, CancellationToken.None); + + // The throwing module did not abort dispatch to the next module. + Assert.AreEqual(1, after.PrepareCalls.Count); + } + + [Test] + [Category("RestoreHandler")] + public async Task InvokeBulkRestoreStartCallsAllModulesInOrderAsync() + { + var order = new List(); + var modA = new FakeRestoreCallbackModule { Key = "a", OnBulk = () => order.Add("a") }; + var modB = new FakeRestoreCallbackModule { Key = "b", OnBulk = () => order.Add("b") }; + + await RestoreHandler.InvokeBulkRestoreStartAsync(new IGenericModule[] { new NonCallbackModule(), modA, modB }, CancellationToken.None); + + Assert.AreEqual(1, modA.BulkStartCalls); + Assert.AreEqual(1, modB.BulkStartCalls); + Assert.That(order, Is.EqualTo(new[] { "a", "b" })); + } + + [Test] + [Category("RestoreHandler")] + public async Task InvokeFileRestoredPassesVersionAndPathAsync() + { + var mod = new FakeRestoreCallbackModule(); + const long expectedVersion = 2; + var expectedTimestamp = new DateTime(2023, 6, 15, 12, 0, 0, DateTimeKind.Utc); + await RestoreHandler.InvokeFileRestoredAsync(new IGenericModule[] { new NonCallbackModule(), mod }, expectedVersion, "/some/path/file.txt", expectedTimestamp, CancellationToken.None); + + Assert.AreEqual(1, mod.FileRestoredCalls.Count); + Assert.AreEqual(expectedVersion, mod.FileRestoredCalls[0].version); + Assert.AreEqual("/some/path/file.txt", mod.FileRestoredCalls[0].path); + Assert.AreEqual(expectedTimestamp, mod.FileRestoredCalls[0].timestamp); + } + + [Test] + [Category("RestoreHandler")] + public async Task InvokeFileRestoredIsolatedFromThrowingModulesAsync() + { + var throwing = new FakeRestoreCallbackModule { Key = "throw", OnFile = (_, _) => throw new InvalidOperationException("boom") }; + var after = new FakeRestoreCallbackModule { Key = "after" }; + + await RestoreHandler.InvokeFileRestoredAsync(new IGenericModule[] { throwing, after }, 0, "/p", DateTime.MinValue, CancellationToken.None); + + Assert.AreEqual(1, after.FileRestoredCalls.Count); + } + + [Test] + [Category("RestoreHandler")] + public async Task DispatchHelpersAcceptNullModulesAsync() + { + // Passing null must not throw. + await RestoreHandler.InvokePreparePriorityFilesAsync(null, new List(), 0, DateTime.MinValue, CancellationToken.None); + await RestoreHandler.InvokeBulkRestoreStartAsync(null, CancellationToken.None); + await RestoreHandler.InvokeFileRestoredAsync(null, 0, "/p", DateTime.MinValue, CancellationToken.None); + Assert.Pass(); + } + + // ---------------------------------------------------------------------------------- + // Integration tests: register a real module in the GenericLoader and run a restore. + // ---------------------------------------------------------------------------------- + + /// + /// A restore-callback module that records invocations into static state. Static + /// state is required because creates a + /// fresh instance per operation, so the instance that runs during the restore is not + /// the same instance the test registered. + /// + public class RecordingRestoreCallbackModule : IRestoreCallbackModule + { + public const string KEY = "test-restore-callback"; + + public static readonly object RecordLock = new(); + public static List<(string files, long version, DateTime timestamp)> PrepareCalls { get; } = new(); + public static int BulkStartCalls; + public static List<(long version, string path, DateTime timestamp)> FileRestoredCalls { get; } = new(); + + public static void Reset() + { + lock (RecordLock) + { + PrepareCalls.Clear(); + BulkStartCalls = 0; + FileRestoredCalls.Clear(); + } + } + + public string Key => KEY; + public string DisplayName => "Test Restore Callback"; + public string Description => "Records restore callback invocations for tests"; + public bool LoadAsDefault => false; + public IList SupportedCommands => new List(); + + public void Configure(IDictionary commandlineOptions) { } + + public Task OnPreparePriorityFilesAsync(IList priorityFiles, long version, DateTime backupTimestamp, CancellationToken cancellationToken) + { + lock (RecordLock) + PrepareCalls.Add((string.Join("|", priorityFiles), version, backupTimestamp)); + return Task.CompletedTask; + } + + public Task OnBulkRestoreStartAsync(CancellationToken cancellationToken) + { + lock (RecordLock) + BulkStartCalls++; + return Task.CompletedTask; + } + + public Task OnFileRestoredAsync(long version, string path, DateTime backupTimestamp, CancellationToken cancellationToken) + { + lock (RecordLock) + FileRestoredCalls.Add((version, path, backupTimestamp)); + return Task.CompletedTask; + } + } + + /// + /// Registers into the static + /// so it is discovered and activated during a restore + /// operation (when enabled via --enable-module). Uses reflection because the + /// loader's module table is not publicly writable. + /// + private static void RegisterRecordingModule() + { + var loaderType = typeof(GenericLoader); + var loaderField = loaderType.GetField("_loader", BindingFlags.NonPublic | BindingFlags.Static) + ?? throw new InvalidOperationException("Could not find GenericLoader._loader"); + var lazy = loaderField.GetValue(null); + var loaderSub = lazy!.GetType().GetProperty("Value")!.GetValue(lazy); + var addModule = loaderSub!.GetType().GetMethod("AddModule", BindingFlags.Public | BindingFlags.Instance) + ?? throw new InvalidOperationException("Could not find AddModule on loader"); + addModule.Invoke(loaderSub, new object[] { new RecordingRestoreCallbackModule() }); + } + + [OneTimeSetUp] + public void OneTimeSetup() + { + RegisterRecordingModule(); + } + + [Test] + [Category("RestoreHandler")] + public async Task RestoreCallbacksFireOnNewRestoreAsync() + { + RecordingRestoreCallbackModule.Reset(); + + // Write two files, one of which we mark as a priority file via the restore + // destination provider (priority files are derived from the target folder). + var file1 = Path.Combine(this.DATAFOLDER, "file1.txt"); + var file2 = Path.Combine(this.DATAFOLDER, "file2.txt"); + File.WriteAllText(file1, "content-1"); + File.WriteAllText(file2, "content-2"); + + var backupOptions = new Dictionary(this.TestOptions); + using (var c = new Controller("file://" + this.TARGETFOLDER, backupOptions, null)) + TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER })); + + var restoreOptions = new Dictionary(this.TestOptions) + { + ["restore-path"] = this.RESTOREFOLDER, + ["restore-legacy"] = "false", + ["enable-module"] = RecordingRestoreCallbackModule.KEY, + }; + + var restoreStartedAt = DateTime.UtcNow; + + using (var c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null)) + TestUtils.AssertResults(await c.RestoreAsync(new[] { "*" })); + + // OnPreparePriorityFilesAsync is always called (even with an empty list). + Assert.That(RecordingRestoreCallbackModule.PrepareCalls.Count, Is.GreaterThanOrEqualTo(1), "OnPreparePriorityFilesAsync should be invoked"); + // The reported version is 0 (newest backup) for the single restored version. + Assert.AreEqual(0, RecordingRestoreCallbackModule.PrepareCalls[0].version, "Prepare should report version 0"); + // The reported backup timestamp is the newest backup's timestamp, created just before this restore. + Assert.That(RecordingRestoreCallbackModule.PrepareCalls[0].timestamp, Is.GreaterThan(restoreStartedAt.AddMinutes(-5)).And.LessThan(DateTime.UtcNow.AddMinutes(1)), + "Prepare should report a recent backup timestamp"); + // OnBulkRestoreStartAsync is called exactly once when the bulk restore starts. + Assert.AreEqual(1, RecordingRestoreCallbackModule.BulkStartCalls, "OnBulkRestoreStartAsync should be invoked once"); + // OnFileRestoredAsync is called for each restored file on the new engine. + Assert.That(RecordingRestoreCallbackModule.FileRestoredCalls.Count, Is.GreaterThanOrEqualTo(2), "OnFileRestoredAsync should be invoked for each restored file"); + // The reported version is 0 (newest backup) for the single restored version. + Assert.IsTrue(RecordingRestoreCallbackModule.FileRestoredCalls.All(x => x.version == 0), "Restored files should report version 0"); + // The reported backup timestamp is recent (the backup was created just before this restore). + Assert.IsTrue(RecordingRestoreCallbackModule.FileRestoredCalls.All(x => x.timestamp > restoreStartedAt.AddMinutes(-5) && x.timestamp < DateTime.UtcNow.AddMinutes(1)), + "Restored files should report a recent backup timestamp"); + // The reported paths are within the restore folder. + Assert.IsTrue(RecordingRestoreCallbackModule.FileRestoredCalls.All(x => x.path.StartsWith(this.RESTOREFOLDER, StringComparison.OrdinalIgnoreCase)), + "Restored file paths should be under the restore folder"); + } + + [Test] + [Category("RestoreHandler")] + public async Task OnFileRestoredNotCalledOnLegacyRestoreAsync() + { + RecordingRestoreCallbackModule.Reset(); + + var file1 = Path.Combine(this.DATAFOLDER, "legacy-file.txt"); + File.WriteAllText(file1, "legacy-content"); + + var backupOptions = new Dictionary(this.TestOptions); + using (var c = new Controller("file://" + this.TARGETFOLDER, backupOptions, null)) + TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER })); + + var restoreOptions = new Dictionary(this.TestOptions) + { + ["restore-path"] = this.RESTOREFOLDER, + ["restore-legacy"] = "true", + ["enable-module"] = RecordingRestoreCallbackModule.KEY, + }; + + using (var c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null)) + TestUtils.AssertResults(await c.RestoreAsync(new[] { "*" })); + + // OnPreparePriorityFilesAsync and OnBulkRestoreStartAsync are invoked on the legacy engine too. + Assert.That(RecordingRestoreCallbackModule.PrepareCalls.Count, Is.GreaterThanOrEqualTo(1), "OnPreparePriorityFilesAsync should be invoked on legacy restore"); + Assert.AreEqual(1, RecordingRestoreCallbackModule.BulkStartCalls, "OnBulkRestoreStartAsync should be invoked once on legacy restore"); + // OnFileRestoredAsync must NOT be called when --restore-legacy is set. + Assert.AreEqual(0, RecordingRestoreCallbackModule.FileRestoredCalls.Count, "OnFileRestoredAsync must not be called on legacy restore"); + } + + [Test] + [Category("RestoreHandler")] + public async Task RestoreCallbackReceivesEmptyPriorityListWhenNoPriorityFilesAsync() + { + RecordingRestoreCallbackModule.Reset(); + + var file1 = Path.Combine(this.DATAFOLDER, "nopriority.txt"); + File.WriteAllText(file1, "no-priority-content"); + + var backupOptions = new Dictionary(this.TestOptions); + using (var c = new Controller("file://" + this.TARGETFOLDER, backupOptions, null)) + TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER })); + + var restoreOptions = new Dictionary(this.TestOptions) + { + ["restore-path"] = this.RESTOREFOLDER, + ["restore-legacy"] = "false", + ["enable-module"] = RecordingRestoreCallbackModule.KEY, + }; + + using (var c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null)) + TestUtils.AssertResults(await c.RestoreAsync(new[] { "*" })); + + // With no priority files, OnPreparePriorityFilesAsync is still called once with an empty list, + // and OnBulkRestoreStartAsync is called exactly once (the bulk restore starts immediately). + Assert.AreEqual(1, RecordingRestoreCallbackModule.PrepareCalls.Count, "OnPreparePriorityFilesAsync should be invoked exactly once"); + Assert.AreEqual(string.Empty, RecordingRestoreCallbackModule.PrepareCalls[0].files, "Priority list should be empty"); + Assert.AreEqual(1, RecordingRestoreCallbackModule.BulkStartCalls, "OnBulkRestoreStartAsync should be invoked once"); + Assert.That(RecordingRestoreCallbackModule.FileRestoredCalls.Count, Is.GreaterThanOrEqualTo(1)); + } + } +}