From 27e73cd7f3a4dfa8dba43ed86bab426f23df2e70 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 29 Jun 2026 13:17:53 +0200 Subject: [PATCH] Added support for restoring files across multiple versions --- .../Database/Local/LocalRestoreDatabase.cs | 162 ++++++- Duplicati/Library/Main/Enums.cs | 27 ++ .../Main/Operation/Restore/FileProcessor.cs | 15 + .../Library/Main/Operation/RestoreHandler.cs | 385 +++++++++++++-- Duplicati/Library/Main/Options.cs | 10 + Duplicati/Library/Main/Strings.cs | 2 + Duplicati/UnitTest/RestoreAllFilesTests.cs | 454 ++++++++++++++++++ 7 files changed, 1027 insertions(+), 28 deletions(-) create mode 100644 Duplicati/UnitTest/RestoreAllFilesTests.cs diff --git a/Duplicati/Library/Main/Database/Local/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/Local/LocalRestoreDatabase.cs index 0e1e374a4..41ea73f1d 100644 --- a/Duplicati/Library/Main/Database/Local/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/Local/LocalRestoreDatabase.cs @@ -410,7 +410,7 @@ namespace Duplicati.Library.Main.Database.Local /// A flag indicating whether to disable the restoration of alternate data streams. /// A cancellation token to monitor for cancellation requests. /// A task that, when awaited, returns a tuple containing the count of files to restore and the total size of those files. - public async Task> PrepareRestoreFilelistAsync(DateTime restoretime, long[] versions, IFilter filter, bool disableAdsRestore, CancellationToken token) + public async Task> PrepareRestoreFilelistAsync(DateTime restoretime, long[]? versions, IFilter? filter, bool disableAdsRestore, CancellationToken token) { m_tempfiletable = $"Fileset-{m_temptabsetguid}"; m_tempblocktable = $"Blocks-{m_temptabsetguid}"; @@ -739,6 +739,40 @@ namespace Duplicati.Library.Main.Database.Local return new Tuple(0, 0); } + /// + /// Recomputes the (file count, total size) tuple for the currently prepared restore + /// file list, e.g. after rows have been removed by . + /// + /// A cancellation token to monitor for cancellation requests. + /// A tuple of the file count and total size of the files remaining in the restore list. + public async Task> PrepareRestoreFilelistCountAsync(CancellationToken token) + { + if (m_tempfiletable == null) + return new Tuple(0, 0); + + await using var cmd = m_connection.CreateCommand($@" + SELECT + COUNT(DISTINCT ""{m_tempfiletable}"".""Path""), + SUM(""Blockset"".""Length"") + FROM + ""{m_tempfiletable}"", + ""Blockset"" + WHERE ""{m_tempfiletable}"".""BlocksetID"" = ""Blockset"".""ID"" + ") + .SetTransaction(m_rtr); + + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + var filecount = 0L; + var filesize = 0L; + if (await rd.ReadAsync(token).ConfigureAwait(false)) + { + filecount = rd.ConvertValueToInt64(0, 0); + filesize = rd.ConvertValueToInt64(1, 0); + } + + return new Tuple(filecount, filesize); + } + /// /// Ensures that the parent file or folder entry of any selected alternate data stream /// is also present in the temporary file table for the given fileset. Alternate data @@ -2141,6 +2175,132 @@ namespace Duplicati.Library.Main.Database.Local rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? "", rd.ConvertValueToString(2) ?? "", rd.ConvertValueToInt64(3)); } + /// + /// Creates a persistent scratch table (a regular table in the database file, visible + /// across connections) that records the file hashes already restored in previous + /// versions during a --restore-all-files=unique run. The table is keyed by a + /// caller-supplied unique name. Keeping the restored hashes in the database (rather + /// than in memory) avoids unbounded memory growth when many files/versions are + /// restored. The caller is responsible for dropping the table via + /// when the multi-version run completes. + /// + /// The unique name of the scratch table to create. + /// A cancellation token to monitor for cancellation requests. + public async Task CreateRestoredHashesTableAsync(string tableName, CancellationToken token) + { + await using var cmd = m_connection.CreateCommand().SetTransaction(m_rtr); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tableName}""", token).ConfigureAwait(false); + await cmd.ExecuteNonQueryAsync($@" + CREATE TABLE ""{tableName}"" ( + ""Hash"" TEXT NOT NULL + ) + ", token).ConfigureAwait(false); + await cmd.ExecuteNonQueryAsync($@"CREATE UNIQUE INDEX ""{tableName}_Hash"" ON ""{tableName}"" (""Hash"")", token).ConfigureAwait(false); + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); + } + + /// + /// Drops the persistent scratch table created by + /// . + /// + /// The unique name of the scratch table to drop. + /// A cancellation token to monitor for cancellation requests. + public async Task DropRestoredHashesTableAsync(string tableName, CancellationToken token) + { + await using var cmd = m_connection.CreateCommand($@"DROP TABLE IF EXISTS ""{tableName}""") + .SetTransaction(m_rtr); + await cmd.ExecuteNonQueryAsync(token).ConfigureAwait(false); + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); + } + + /// + /// Marks the data of a file in the prepared restore file list as verified, by setting + /// its DataVerified flag to 1. Used by the new restore path + /// (Restore.FileProcessor) after a file's content has been successfully + /// restored and its hash verified, so that + /// can record only successfully restored files for de-duplication. The legacy restore + /// path sets this flag via the block marker; this method exposes the same update for + /// callers that do not have a block marker. + /// + /// The ID of the file in the prepared restore file list. + /// A cancellation token to monitor for cancellation requests. + public async Task MarkFileDataVerifiedAsync(long targetfileid, CancellationToken token) + { + await using var cmd = m_connection.CreateCommand($@" + UPDATE ""{m_tempfiletable}"" + SET ""DataVerified"" = 1 + WHERE ""ID"" = @TargetFileId + ") + .SetTransaction(m_rtr) + .SetParameterValue("@TargetFileId", targetfileid); + await cmd.ExecuteNonQueryAsync(token).ConfigureAwait(false); + } + + /// + /// Appends the file hashes of the files that were successfully restored (or verified + /// in place) in the just-completed version into the persistent restored-hashes table. + /// Only files whose DataVerified flag is 1 are recorded, so files whose restore + /// failed are not harvested and remain eligible for restore in subsequent versions. + /// Only the file hash is recorded, not metadata. Folder entries (which have no + /// blockset hash) are not recorded. + /// + /// The unique name of the restored-hashes scratch table. + /// A cancellation token to monitor for cancellation requests. + /// The number of hashes appended. + public async Task AddCurrentFilesToRestoredHashesAsync(string tableName, CancellationToken token) + { + if (m_tempfiletable == null) + return 0; + + await using var cmd = m_connection.CreateCommand($@" + INSERT OR IGNORE INTO ""{tableName}"" (""Hash"") + SELECT ""Blockset"".""Fullhash"" + FROM ""{m_tempfiletable}"" + INNER JOIN ""Blockset"" ON ""{m_tempfiletable}"".""BlocksetID"" = ""Blockset"".""ID"" + WHERE ""Blockset"".""Fullhash"" != '' + AND ""{m_tempfiletable}"".""DataVerified"" = 1 + ") + .SetTransaction(m_rtr); + var added = await cmd.ExecuteNonQueryAsync(token).ConfigureAwait(false); + // Commit so the appended hashes are durable and visible to the separate + // connection used by the next version's restore (which reads this table to + // exclude already-restored files). + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); + return added; + } + + /// + /// Removes files from the prepared restore file list whose content hash (the + /// Blockset.Fullhash) is present in the persistent restored-hashes scratch + /// table. Used by the --restore-all-files=unique feature to skip files already + /// restored in a previous version. Only the file hash is considered, not metadata. + /// Folder entries (which have no blockset hash) are never removed by this call. + /// + /// The name of the persistent restored-hashes scratch table. If null or empty, nothing is removed. + /// A cancellation token to monitor for cancellation requests. + /// The number of files removed from the restore list. + public async Task RemoveFilesByRestoredHashesAsync(string restoredHashesTable, CancellationToken token) + { + if (m_tempfiletable == null) + return 0; + if (string.IsNullOrEmpty(restoredHashesTable)) + return 0; + + await using var cmd = m_connection.CreateCommand($@" + DELETE FROM ""{m_tempfiletable}"" + WHERE ""ID"" IN ( + SELECT ""{m_tempfiletable}"".""ID"" + FROM ""{m_tempfiletable}"" + INNER JOIN ""Blockset"" + ON ""{m_tempfiletable}"".""BlocksetID"" = ""Blockset"".""ID"" + INNER JOIN ""{restoredHashesTable}"" + ON ""Blockset"".""Fullhash"" = ""{restoredHashesTable}"".""Hash"" + ) + ") + .SetTransaction(m_rtr); + return await cmd.ExecuteNonQueryAsync(token).ConfigureAwait(false); + } + /// /// Returns a list of files and symlinks to restore. /// diff --git a/Duplicati/Library/Main/Enums.cs b/Duplicati/Library/Main/Enums.cs index 970c99d19..f699338ef 100644 --- a/Duplicati/Library/Main/Enums.cs +++ b/Duplicati/Library/Main/Enums.cs @@ -218,4 +218,31 @@ namespace Duplicati.Library.Main /// BlindlyUpload } + + /// + /// Controls how the --restore-all-files option behaves during a restore. + /// + public enum RestoreAllFilesMode + { + /// + /// The option is not active; restore proceeds as a normal single-version + /// restore (the most recent matching version, or the version selected by + /// --version/--time). + /// + False = 0, + + /// + /// Restore every targeted version into its own timestamp-named subfolder + /// below the restore target. Every (non-filtered) file is restored in + /// every version; no cross-version de-duplication is performed. + /// + True = 1, + + /// + /// Like , but subsequent versions skip any file whose + /// content (file hash) was already restored in a previous version. The + /// uniqueness check is performed on the file hash only, not on metadata. + /// + Unique = 2 + } } diff --git a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs index 7080a7c2b..42c346647 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs @@ -511,6 +511,21 @@ namespace Duplicati.Library.Main.Operation.Restore // TODO legacy restore doesn't count metadata restore as a restored file. + // Mark the file's data as verified in the prepared restore file list. + // The --restore-all-files=unique feature harvests only DataVerified=1 + // files for cross-version de-duplication, so files that failed to + // restore (which throw above and never reach here) are not recorded and + // remain eligible for restore in subsequent versions. A failure to mark + // is non-fatal: it only affects de-dup, not the restore itself. + try + { + await db.MarkFileDataVerifiedAsync(file.ID, results.TaskControl.ProgressToken).ConfigureAwait(false); + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "MarkFileDataVerifiedFailed", ex, "Failed to mark file as data verified: {0}", ex.Message); + } + sw_work_results?.Start(); if (empty_file_or_symlink || bytes_written > 0) { diff --git a/Duplicati/Library/Main/Operation/RestoreHandler.cs b/Duplicati/Library/Main/Operation/RestoreHandler.cs index 8a4e12206..b9b7cc34e 100644 --- a/Duplicati/Library/Main/Operation/RestoreHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreHandler.cs @@ -19,8 +19,11 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +#nullable enable + using System; using System.Collections.Generic; +using System.IO; using System.Linq; using Duplicati.Library.Interface; using Duplicati.Library.Common.IO; @@ -42,9 +45,19 @@ namespace Duplicati.Library.Main.Operation private static readonly string LOGTAG = Logging.Log.LogTagFromType(); private readonly Options m_options; - private byte[] m_blockbuffer; + private byte[]? m_blockbuffer; private readonly RestoreResults m_result; - private static readonly string DIRSEP = Util.DirectorySeparatorString; + + /// + /// When non-null, names a persistent scratch table (in the local database file) that + /// records the file hashes already restored in previous versions during a + /// --restore-all-files=unique run. The per-version file-list preparation + /// excludes files whose hash is present in this table, and each completed version + /// appends its restored hashes to it. The table is created once at the start of the + /// multi-version run and dropped at the end. Keeping this in the database (rather than + /// in memory) avoids unbounded memory growth when many files/versions are involved. + /// + private string? m_restoredHashesTable; public RestoreHandler(Options options, RestoreResults result) { @@ -102,7 +115,7 @@ namespace Duplicati.Library.Main.Operation return tmp.CompressionModule; } - public static RecreateDatabaseHandler.NumberedFilterFilelistDelegate GetNumberedFilelistFilterDelegate(DateTime time, long[] versions, bool singleTimeMatch = false) + public static RecreateDatabaseHandler.NumberedFilterFilelistDelegate? GetNumberedFilelistFilterDelegate(DateTime time, long[]? versions, bool singleTimeMatch = false) { versions ??= []; if (versions.Length == 0 && time.Ticks == 0) @@ -145,7 +158,7 @@ namespace Duplicati.Library.Main.Operation return numbers; } - public async Task RunAsync(string[] paths, IBackendManager backendManager, Library.Utility.IFilter filter, IRestoreDestinationProvider restoreDestination) + public async Task RunAsync(string[] paths, IBackendManager backendManager, Library.Utility.IFilter? filter, IRestoreDestinationProvider restoreDestination) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_Begin); @@ -161,20 +174,43 @@ namespace Duplicati.Library.Main.Operation // If we have both target paths and a filter, combine them into a single filter filter = JoinedFilterExpression.Join(new FilterExpression(paths), filter); - LocalRestoreDatabase db = null; - TempFile tmpdb = null; + // The --restore-all-files option activates a multi-version restore. When active, + // every targeted version is restored into its own timestamp-named subfolder below + // the restore target, instead of restoring a single version. The single-version + // restore logic below is reused verbatim for each version. + if (m_options.RestoreAllFiles is RestoreAllFilesMode.True or RestoreAllFilesMode.Unique) + { + await RunRestoreAllFilesAsync(backendManager, filter, restoreDestination).ConfigureAwait(false); + return; + } + + await RunSingleVersionRestoreAsync(backendManager, filter, restoreDestination, m_options.Time, m_options.Version, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + } + + /// + /// Performs a normal single-version restore, recreating the local database (if needed) + /// for the given time/versions and dispatching to the legacy or new restore flow. + /// + private async Task RunSingleVersionRestoreAsync(IBackendManager backendManager, Library.Utility.IFilter? filter, IRestoreDestinationProvider restoreDestination, DateTime time, long[]? versions, CancellationToken cancellationToken) + { + LocalRestoreDatabase? db = null; + TempFile? tmpdb = null; try { - if (!m_options.NoLocalDb && SystemIO.IO_OS.FileExists(m_options.Dbpath)) + var dbpath = m_options.Dbpath; + if (!m_options.NoLocalDb && SystemIO.IO_OS.FileExists(dbpath)) { - db = await LocalRestoreDatabase.CreateAsync(m_options.Dbpath, null, m_result.TaskControl.ProgressToken) + if (string.IsNullOrWhiteSpace(dbpath)) + throw new InvalidOperationException("Unexpected empty database path"); + db = await LocalRestoreDatabase.CreateAsync(dbpath, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } else { Logging.Log.WriteInformationMessage(LOGTAG, "NoLocalDatabase", "No local database, building a temporary database"); tmpdb = new TempFile(); - RecreateDatabaseHandler.NumberedFilterFilelistDelegate filelistfilter = GetNumberedFilelistFilterDelegate(m_options.Time, m_options.Version); + var filelistfilter = GetNumberedFilelistFilterDelegate(time, versions); db = await LocalRestoreDatabase.CreateAsync(tmpdb, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); m_result.RecreateDatabaseResults = new RecreateDatabaseResults(m_result); @@ -188,14 +224,14 @@ namespace Duplicati.Library.Main.Operation //If we have --version set, we need to adjust, as the db has only the required versions //TODO: Bit of a hack to set options that way - if (m_options.Version != null && m_options.Version.Length > 0) - m_options.RawOptions["version"] = string.Join(",", Enumerable.Range(0, m_options.Version.Length).Select(x => x.ToString())); + if (versions != null && versions.Length > 0) + m_options.RawOptions["version"] = string.Join(",", Enumerable.Range(0, versions.Length).Select(x => x.ToString())); } if (m_options.RestoreLegacy) - await DoRunAsync(backendManager, db, filter, restoreDestination, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await DoRunAsync(backendManager, db, filter, restoreDestination, cancellationToken).ConfigureAwait(false); else - await DoRunNewAsync(backendManager, db, filter, restoreDestination, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await DoRunNewAsync(backendManager, db, filter, restoreDestination, cancellationToken).ConfigureAwait(false); } finally { @@ -205,6 +241,264 @@ namespace Duplicati.Library.Main.Operation } } + /// + /// Implements the --restore-all-files multi-version restore. + /// The set of targeted versions is computed using the same logic as a normal + /// restore (the versions matching --time/--version, newest first). + /// Each targeted version is restored into its own timestamp-named subfolder below + /// the restore target. The first version restores all (non-filtered) files. + /// Subsequent versions apply the usual filter and, when the mode is + /// , additionally skip any file whose + /// content (file hash) was already restored in a previous version. + /// + private async Task RunRestoreAllFilesAsync(IBackendManager backendManager, Library.Utility.IFilter? filter, IRestoreDestinationProvider restoreDestination) + { + var cancellationToken = m_result.TaskControl.ProgressToken; + + // The option requires a definitive restore target. + if (string.IsNullOrEmpty(restoreDestination.TargetDestination)) + throw new UserInformationException("The --restore-all-files option requires --restore-path to be set", "RestoreAllFilesRequiresRestorePath"); + + var dbpath = m_options.Dbpath; + // Enumerating the targeted versions requires the local database. The per-version + // restore below can recreate a database when none exists, but discovering the + // available filesets needs an existing database. + if (m_options.NoLocalDb || string.IsNullOrWhiteSpace(dbpath) || !SystemIO.IO_OS.FileExists(dbpath)) + throw new UserInformationException("The --restore-all-files option requires a local database", "RestoreAllFilesRequiresLocalDb"); + + // Snapshot the original version/time options so they can be restored after the loop. + // The per-version loop below drives selection solely via the per-iteration "version" + // option, so "time" is cleared for the duration of the loop and restored afterwards. + var originalVersionValue = m_options.RawOptions.GetValueOrDefault("version"); + var originalTimeValue = m_options.RawOptions.GetValueOrDefault("time"); + + // Resolve the targeted filesets using the same logic as a normal restore. The + // filesets are returned ordered by timestamp descending (newest first), which + // matches the version-index ordering used by --version (version 0 = newest). + KeyValuePair[] allFilesets; + using (var db = await LocalRestoreDatabase.CreateAsync(dbpath, null, cancellationToken).ConfigureAwait(false)) + { + allFilesets = await db + .FilesetTimesAsync(cancellationToken) + .ToArrayAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false); + } + + // Apply the same time/version selection as GetFilesetIDsAsync would, producing + // the list of filesets to restore, newest first. This selects the full set of + // versions to restore up front, before "time" is cleared for the per-version loop. + var selectedFilesets = SelectTargetedFilesets(allFilesets, m_options.Time, m_options.Version); + + if (selectedFilesets.Count == 0) + throw new UserInformationException("No backup versions matched the restore selection", "NoBackupAtDate"); + + Logging.Log.WriteInformationMessage(LOGTAG, "RestoreAllFiles", "Restoring {0} version(s) with --restore-all-files={1}", selectedFilesets.Count, m_options.RestoreAllFiles); + + // For "unique" mode, the file hashes restored across previous versions are + // accumulated in a persistent scratch table in the local database (not in memory, + // to avoid unbounded growth). The table is created once here and dropped in the + // finally below. Each completed version appends its restored hashes to it, and + // subsequent versions exclude files whose hash is already present. + if (m_options.RestoreAllFiles == RestoreAllFilesMode.Unique) + { + m_restoredHashesTable = $"RestoredHashes-{Library.Utility.Utility.GetHexGuid()}"; + using (var setupDb = await LocalRestoreDatabase.CreateAsync(dbpath, null, cancellationToken).ConfigureAwait(false)) + await setupDb.CreateRestoredHashesTableAsync(m_restoredHashesTable, cancellationToken).ConfigureAwait(false); + } + + // Clear the "time" option for the duration of the per-version loop. The per-version + // restore below selects its fileset via the per-iteration "version" option, but the + // underlying file-list query combines "time" and "version" with OR. If "time" + // remained set, every iteration would also pull in all filesets at or before that + // time, breaking the per-version isolation (each subfolder would contain files from + // multiple versions). "time" is restored in the finally below. + if (originalTimeValue != null) + m_options.RawOptions.Remove("time"); + + try + { + for (var i = 0; i < selectedFilesets.Count; i++) + { + var fileset = selectedFilesets[i]; + var timestampFolder = fileset.Timestamp.ToString("yyyyMMdd-HHmmss"); + var versionTarget = SystemIO.IO_OS.PathCombine(restoreDestination.TargetDestination, timestampFolder); + + Logging.Log.WriteInformationMessage(LOGTAG, "RestoreAllFilesVersion", "Restoring version {0} ({1}) into {2}", i, fileset.Timestamp, versionTarget); + + // Target this single version only. The version index is the position in the + // newest-first fileset list (0 = newest), which is what --version uses. + m_options.RawOptions["version"] = fileset.VersionIndex.ToString(); + + // Build the per-version restore destination. A thin wrapper remaps + // TargetDestination to the timestamp subfolder; all other operations are + // delegated to the original provider. The mapped paths handed to the + // provider are already inside the original target, so the underlying + // provider's path validation continues to accept them. + var versionDestination = new VersionedRestoreDestinationProvider(restoreDestination, versionTarget); + + await RunSingleVersionRestoreAsync(backendManager, filter, versionDestination, m_options.Time, [fileset.VersionIndex], cancellationToken) + .ConfigureAwait(false); + } + } + finally + { + // Drop the persistent restored-hashes scratch table if it was created. Use a + // dedicated timeout token (30s) rather than the restore's cancellation token so + // cleanup still runs if the restore was cancelled, and does not hang forever. + if (!string.IsNullOrEmpty(m_restoredHashesTable)) + { + try + { + using var cleanupCts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + using (var cleanupDb = await LocalRestoreDatabase.CreateAsync(dbpath, null, cleanupCts.Token).ConfigureAwait(false)) + await cleanupDb.DropRestoredHashesTableAsync(m_restoredHashesTable, cleanupCts.Token).ConfigureAwait(false); + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "DropRestoredHashesFailed", ex, "Failed to drop restored-hashes scratch table: {0}", ex.Message); + } + m_restoredHashesTable = null; + } + + // Restore the original options. + if (originalVersionValue != null) + m_options.RawOptions["version"] = originalVersionValue; + else + m_options.RawOptions.Remove("version"); + + if (originalTimeValue != null) + m_options.RawOptions["time"] = originalTimeValue; + else + m_options.RawOptions.Remove("time"); + } + } + + /// + /// Appends the file hashes of the files that were successfully restored (or verified + /// in place) in the just-completed version into the persistent restored-hashes scratch + /// table (). Called by the restore flow after a + /// version's file list is processed (and before the temp tables are dropped), so the + /// next version can skip files with the same content. Only files whose + /// DataVerified flag is 1 are recorded; files whose restore failed are not + /// harvested and remain eligible for restore in subsequent versions. Only the file hash + /// is recorded, not metadata. The hashes are stored in the database rather than in + /// memory to avoid unbounded memory growth across many files/versions. + /// + private async Task HarvestRestoredHashesAsync(LocalRestoreDatabase database, CancellationToken cancellationToken) + { + if (string.IsNullOrEmpty(m_restoredHashesTable)) + return; + + try + { + var added = await database + .AddCurrentFilesToRestoredHashesAsync(m_restoredHashesTable, cancellationToken) + .ConfigureAwait(false); + if (added > 0) + Logging.Log.WriteVerboseMessage(LOGTAG, "RecordedRestoredHashes", "Recorded {0} file hash(es) restored in this version for de-duplication", added); + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "HarvestRestoredHashesFailed", ex, "Failed to harvest restored file hashes for de-duplication: {0}", ex.Message); + } + } + + /// + /// Selects the targeted filesets from the full newest-first fileset list, applying the + /// same time/version selection logic as . + /// Returns the selected filesets as a list of (filesetId, timestamp, versionIndex) + /// triples, newest first. + /// + private static List SelectTargetedFilesets(IReadOnlyList> allFilesets, DateTime time, long[]? versions) + { + var result = new List(); + + // Version indices take precedence: if specific versions are requested, select those + // indices from the newest-first list (matching GetFilelistWhereClauseAsync which + // selects filesets by ID derived from the version index). + if (versions != null && versions.Length > 0) + { + foreach (var v in versions) + { + if (v >= 0 && v < allFilesets.Count) + result.Add(new FilesetSelection(allFilesets[(int)v].Key, allFilesets[(int)v].Value, (int)v)); + else + Logging.Log.WriteWarningMessage(LOGTAG, "SkipInvalidVersion", null, "Skipping invalid version: {0}", v); + } + + if (result.Count > 0) + return result; + } + + // Otherwise, select by time (all filesets at or before the given time), or, when no + // time is set either, the single newest fileset (the normal default restore). + if (time.Ticks > 0) + { + // Compare in UTC to avoid kind mismatches; allFilesets timestamps are local + // (as returned by FilesetTimesAsync) and `time` may be local or UTC. + var compareTime = time.ToUniversalTime(); + for (var idx = 0; idx < allFilesets.Count; idx++) + { + var fs = allFilesets[idx]; + if (fs.Value.ToUniversalTime() <= compareTime) + result.Add(new FilesetSelection(fs.Key, fs.Value, idx)); + } + } + else + { + // No version or time filter: target every fileset. The --restore-all-files + // option is "restore all files across all versions", so without an explicit + // version/time selection every available version is targeted. + for (var idx = 0; idx < allFilesets.Count; idx++) + result.Add(new FilesetSelection(allFilesets[idx].Key, allFilesets[idx].Value, idx)); + } + + return result; + } + + /// + /// A thin wrapper that reports a per-version + /// (the timestamp subfolder) while delegating every + /// operation to the original provider. The restore path-mapping logic maps all target + /// paths into the per-version subfolder, and those mapped paths are inside the + /// original target, so the underlying provider's path validation still accepts them. + /// + private sealed class VersionedRestoreDestinationProvider : IRestoreDestinationProvider + { + private readonly IRestoreDestinationProvider _inner; + private readonly string _targetDestination; + + public VersionedRestoreDestinationProvider(IRestoreDestinationProvider inner, string targetDestination) + { + _inner = inner; + _targetDestination = targetDestination; + } + + public string TargetDestination => _targetDestination; + public Task Initialize(CancellationToken cancel) => _inner.Initialize(cancel); + public Task Finalize(Action? progressCallback, CancellationToken cancel) => _inner.Finalize(progressCallback, cancel); + public Task Test(CancellationToken cancellationToken) => _inner.Test(cancellationToken); + public Task CreateFolderIfNotExists(string path, CancellationToken cancel) => _inner.CreateFolderIfNotExists(path, cancel); + public Task FileExists(string path, CancellationToken cancel) => _inner.FileExists(path, cancel); + public Task OpenWrite(string path, CancellationToken cancel) => _inner.OpenWrite(path, cancel); + public Task OpenRead(string path, CancellationToken cancel) => _inner.OpenRead(path, cancel); + public Task OpenReadWrite(string path, CancellationToken cancel) => _inner.OpenReadWrite(path, cancel); + public Task GetFileLength(string path, CancellationToken cancel) => _inner.GetFileLength(path, cancel); + public Task HasReadOnlyAttribute(string path, CancellationToken cancel) => _inner.HasReadOnlyAttribute(path, cancel); + public Task ClearReadOnlyAttribute(string path, CancellationToken cancel) => _inner.ClearReadOnlyAttribute(path, cancel); + public Task WriteMetadata(string path, Dictionary metadata, bool restoreSymlinkMetadata, bool restorePermissions, CancellationToken cancel) => _inner.WriteMetadata(path, metadata, restoreSymlinkMetadata, restorePermissions, cancel); + public Task DeleteFolder(string path, CancellationToken cancel) => _inner.DeleteFolder(path, cancel); + public Task DeleteFile(string path, CancellationToken cancel) => _inner.DeleteFile(path, cancel); + public IList GetPriorityFiles() => _inner.GetPriorityFiles(); + public void Dispose() => _inner.Dispose(); + } + + /// + /// A selected fileset: its database ID, its timestamp, and its version index (position + /// in the newest-first fileset list, where 0 is the newest). + /// + private readonly record struct FilesetSelection(long FilesetId, DateTime Timestamp, int VersionIndex); + private static async Task PatchWithBlocklistAsync(LocalRestoreDatabase database, BlockVolumeReader blocks, Options options, RestoreResults result, byte[] blockbuffer, RestoreHandlerMetadataStorage metadatastorage, IRestoreDestinationProvider restoreDestination, CancellationToken cancellationToken) { var blocksize = options.Blocksize; @@ -376,7 +670,7 @@ namespace Duplicati.Library.Main.Operation /// The filter of which files to restore. /// The destination to restore to. /// The cancellation token to monitor for cancellation requests. - private async Task DoRunNewAsync(IBackendManager backendManager, LocalRestoreDatabase database, IFilter filter, IRestoreDestinationProvider restoreDestination, CancellationToken cancellationToken) + private async Task DoRunNewAsync(IBackendManager backendManager, LocalRestoreDatabase database, IFilter? filter, IRestoreDestinationProvider restoreDestination, CancellationToken cancellationToken) { // Perform initial setup await Utility.UpdateOptionsFromDbAsync(database, m_options, cancellationToken) @@ -403,7 +697,7 @@ namespace Duplicati.Library.Main.Operation // Prepare the block and file list and create the directory structure m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateFileList); using (new Logging.Timer(LOGTAG, "PrepareBlockList", "PrepareBlockList")) - await PrepareBlockAndFileListAsync(database, m_options, filter, restoreDestination, m_result) + await PrepareBlockAndFileListAsync(database, m_options, filter, restoreDestination, m_result, m_restoredHashesTable) .ConfigureAwait(false); m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateTargetFolders); using (new Logging.Timer(LOGTAG, "CreateDirectory", "CreateDirectory")) @@ -509,6 +803,10 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteInformationMessage(LOGTAG, "NoFilesNeededRestore", null, "Restore completed but all files were already present"); } + // Harvest restored file hashes for --restore-all-files=unique before the temp + // tables are dropped, so subsequent versions can skip files with the same content. + await HarvestRestoredHashesAsync(database, cancellationToken).ConfigureAwait(false); + // Drop the temp tables await database.DropRestoreTableAsync(cancellationToken).ConfigureAwait(false); await backendManager.WaitForEmptyAsync(database, m_result.TaskControl.ProgressToken).ConfigureAwait(false); @@ -537,7 +835,7 @@ namespace Duplicati.Library.Main.Operation /// The list of priority file names to exclude. /// The original filter to combine with. /// A filter that matches the original filter but excludes priority files. - private static Library.Utility.IFilter BuildExcludePriorityFilter(IList priorityFiles, Library.Utility.IFilter originalFilter) + private static Library.Utility.IFilter? BuildExcludePriorityFilter(IList priorityFiles, Library.Utility.IFilter? originalFilter) { var excludePatterns = priorityFiles.Select(pf => "*" + pf); var excludeFilter = new FilterExpression(excludePatterns, false); @@ -554,7 +852,7 @@ namespace Duplicati.Library.Main.Operation /// The filter of which files to restore. /// The destination to restore to. /// The cancellation token to monitor for cancellation requests. - private async Task DoRunAsync(IBackendManager backendManager, LocalRestoreDatabase database, Library.Utility.IFilter filter, IRestoreDestinationProvider restoreDestination, CancellationToken cancellationToken) + private async Task DoRunAsync(IBackendManager backendManager, LocalRestoreDatabase database, Library.Utility.IFilter? filter, IRestoreDestinationProvider restoreDestination, CancellationToken cancellationToken) { using (var metadatastorage = new RestoreHandlerMetadataStorage()) { @@ -632,12 +930,12 @@ namespace Duplicati.Library.Main.Operation /// The destination to restore to. /// Shared metadata storage across phases. /// The cancellation token to monitor for cancellation requests. - private async Task RestoreCoreAsync(IBackendManager backendManager, LocalRestoreDatabase database, Library.Utility.IFilter filter, IRestoreDestinationProvider restoreDestination, RestoreHandlerMetadataStorage metadatastorage, CancellationToken cancellationToken) + private async Task RestoreCoreAsync(IBackendManager backendManager, LocalRestoreDatabase database, Library.Utility.IFilter? filter, IRestoreDestinationProvider restoreDestination, RestoreHandlerMetadataStorage metadatastorage, CancellationToken cancellationToken) { //Figure out what files are to be patched, and what blocks are needed m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateFileList); using (new Logging.Timer(LOGTAG, "PrepareBlockList", "PrepareBlockList")) - await PrepareBlockAndFileListAsync(database, m_options, filter, restoreDestination, m_result) + await PrepareBlockAndFileListAsync(database, m_options, filter, restoreDestination, m_result, m_restoredHashesTable) .ConfigureAwait(false); //Make the entire output setup @@ -646,6 +944,9 @@ namespace Duplicati.Library.Main.Operation await CreateDirectoryStructureAsync(database, restoreDestination, string.IsNullOrEmpty(restoreDestination.TargetDestination), m_options, m_result) .ConfigureAwait(false); + if (m_blockbuffer == null) + throw new InvalidOperationException("Block buffer has not been initialized"); + //If we are patching an existing target folder, do not touch stuff that is already updated m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_ScanForExistingFiles); using (var blockhasher = HashFactory.CreateHasher(m_options.BlockHashAlgorithm)) @@ -819,6 +1120,10 @@ namespace Duplicati.Library.Main.Operation else if (fileErrors > 0) Logging.Log.WriteInformationMessage(LOGTAG, "RestoreFailures", "Failed to restore {0} files", fileErrors); + // Harvest restored file hashes for --restore-all-files=unique before the temp + // tables are dropped, so subsequent versions can skip files with the same content. + await HarvestRestoredHashesAsync(database, cancellationToken).ConfigureAwait(false); + // Drop the temp tables await database.DropRestoreTableAsync(cancellationToken).ConfigureAwait(false); await backendManager.WaitForEmptyAsync(database, cancellationToken).ConfigureAwait(false); @@ -826,13 +1131,12 @@ namespace Duplicati.Library.Main.Operation public static async Task ApplyMetadataAsync(string path, System.IO.Stream stream, Options options, IRestoreDestinationProvider restoreDestination, CancellationToken cancellationToken) { - // TODO This has been modified to return a bool indicating if anything was written to properly report the number of files restored. The legacy restore doesn't check this, which produces an error in the CI where it reports that no files have been restored, even though one have. It's in Duplicati/UnitTests/SymLinkTests.cs the test SymLinkTests.SymLinkExists() that fails on the very last assert that there are 0 warnings. using (var tr = new System.IO.StreamReader(stream)) using (var jr = new Newtonsoft.Json.JsonTextReader(tr)) { - var metadata = new Newtonsoft.Json.JsonSerializer().Deserialize>(jr); + var metadata = new Newtonsoft.Json.JsonSerializer().Deserialize>(jr); // If this is dry-run, we stop after having deserialized the metadata - if (options.Dryrun) + if (metadata == null || options.Dryrun) return false; return await restoreDestination.WriteMetadata(path, metadata, options.RestoreSymlinkMetadata, options.RestorePermissions, cancellationToken).ConfigureAwait(false); @@ -890,6 +1194,8 @@ namespace Duplicati.Library.Main.Operation patched = true; if (!options.Dryrun) { + if (targetstream == null) + throw new InvalidOperationException("Did not expect stream to be null"); targetstream.Position = block.Offset; await targetstream.WriteAsync(blockbuffer, 0, size); } @@ -1008,6 +1314,9 @@ namespace Duplicati.Library.Main.Operation metadatastorage.Add(targetpath, new System.IO.MemoryStream(blockbuffer, 0, size)); else { + if (file == null) + throw new InvalidOperationException("Did not expect file to be null"); + file.Position = targetblock.Offset; await file.WriteAsync(blockbuffer, 0, size); } @@ -1059,7 +1368,7 @@ namespace Duplicati.Library.Main.Operation await blockmarker.CommitAsync(result.TaskControl.ProgressToken).ConfigureAwait(false); } - private static async Task PrepareBlockAndFileListAsync(LocalRestoreDatabase database, Options options, Library.Utility.IFilter filter, IRestoreDestinationProvider restoreDestination, RestoreResults result) + private static async Task PrepareBlockAndFileListAsync(LocalRestoreDatabase database, Options options, Library.Utility.IFilter? filter, IRestoreDestinationProvider restoreDestination, RestoreResults result, string? restoredHashesTable) { // Create a temporary table FILES by selecting the files from fileset that matches a specific operation id // Delete all entries from the temp table that are excluded by the filter(s) @@ -1068,6 +1377,28 @@ namespace Duplicati.Library.Main.Operation var c = await database .PrepareRestoreFilelistAsync(options.Time, options.Version, filter, options.DisableAdsRestore, result.TaskControl.ProgressToken) .ConfigureAwait(false); + + // When --restore-all-files=unique is active, remove files already restored in + // a previous version. The uniqueness check is on the file hash only. The set of + // already-restored hashes is kept in a persistent database table (named by + // restoredHashesTable) so it does not grow unbounded in memory. + if (!string.IsNullOrEmpty(restoredHashesTable) && c.Item1 > 0) + { + using (new Logging.Timer(LOGTAG, "RemoveRestoredHashes", "Removing files already restored in a previous version")) + { + var removed = await database + .RemoveFilesByRestoredHashesAsync(restoredHashesTable, result.TaskControl.ProgressToken) + .ConfigureAwait(false); + if (removed > 0) + Logging.Log.WriteVerboseMessage(LOGTAG, "SkippedAlreadyRestored", "Skipped {0} file(s) already restored in a previous version", removed); + + // Recompute the file count after removal. + c = await database + .PrepareRestoreFilelistCountAsync(result.TaskControl.ProgressToken) + .ConfigureAwait(false); + } + } + result.OperationProgressUpdater.UpdatefileCount(c.Item1, c.Item2, true); // If the selection is completely empty (no files, folders, or symlinks), @@ -1248,14 +1579,14 @@ namespace Duplicati.Library.Main.Operation // Parallelize file hash calculation on rename. Running read-only on same array should not cause conflicts or races. // Actually, in future always calculate the file hash and mark the file data as already verified. - System.Threading.Tasks.Task calcFileHashTask = null; + var calcFileHashTask = Task.CompletedTask; if (calcFileHash) - calcFileHashTask = System.Threading.Tasks.Task.Run( + calcFileHashTask = Task.Run( () => filehasher.TransformBlock(blockbuffer, 0, size, blockbuffer, 0)); var key = Convert.ToBase64String(blockhasher.ComputeHash(blockbuffer, 0, size)); - if (calcFileHashTask != null) await calcFileHashTask.ConfigureAwait(false); // wait because blockbuffer will be overwritten. + await calcFileHashTask.ConfigureAwait(false); // wait because blockbuffer will be overwritten. if (key == targetblock.Hash) { @@ -1278,7 +1609,7 @@ namespace Duplicati.Library.Main.Operation if (calcFileHash) // now check if files are identical { filehasher.TransformFinalBlock(blockbuffer, 0, 0); - var filekey = Convert.ToBase64String(filehasher.Hash); + var filekey = Convert.ToBase64String(filehasher.Hash ?? throw new InvalidDataException("Unexpected null hash result")); fullfilehashmatch = (filekey == targetfilehash); } diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index 54f0c2281..f7221ff56 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -596,6 +596,7 @@ namespace Duplicati.Library.Main new CommandLineArgument("restore-cache-max", CommandLineArgument.ArgumentType.Size, Strings.Options.RestoreCacheMaxShort, Strings.Options.RestoreCacheMaxLong, DEFAULT_RESTORE_CACHE_MAX), new CommandLineArgument("restore-cache-evict", CommandLineArgument.ArgumentType.Integer, Strings.Options.RestoreCacheEvictShort, Strings.Options.RestoreCacheEvictLong, DEFAULT_RESTORE_CACHE_EVICT.ToString()), new CommandLineArgument("restore-file-processors", CommandLineArgument.ArgumentType.Integer, Strings.Options.RestoreFileprocessorsShort, Strings.Options.RestoreFileprocessorsLong, DEFAULT_RESTORE_FILE_PROCESSORS.ToString()), + new CommandLineArgument("restore-all-files", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.RestoreAllFilesShort, Strings.Options.RestoreAllFilesLong("restore-path"), nameof(RestoreAllFilesMode.False), null, Enum.GetNames(typeof(RestoreAllFilesMode))), new CommandLineArgument("restore-legacy", CommandLineArgument.ArgumentType.Boolean, Strings.Options.RestoreLegacyShort, Strings.Options.RestoreLegacyLong, "false"), new CommandLineArgument("restore-preallocate-size", CommandLineArgument.ArgumentType.Boolean, Strings.Options.RestorePreallocateSizeShort, Strings.Options.RestorePreallocateSizeLong, "false"), new CommandLineArgument("restore-volume-cache-hint", CommandLineArgument.ArgumentType.Size, Strings.Options.RestoreVolumeCacheHintShort, Strings.Options.RestoreVolumeCacheHintLong, DEFAULT_RESTORE_VOLUME_CACHE_HINT), @@ -1408,6 +1409,15 @@ namespace Duplicati.Library.Main /// public string? Restorepath => m_options.GetValueOrDefault("restore-path"); + /// + /// Gets the --restore-all-files mode controlling multi-version restore behavior. + /// When set to or , + /// every targeted version is restored into its own timestamp-named subfolder below + /// . performs a normal + /// single-version restore. + /// + public RestoreAllFilesMode RestoreAllFiles => GetEnum("restore-all-files", RestoreAllFilesMode.False); + /// /// Gets the index file usage method /// diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs index 0c7be2e19..41116f5b5 100644 --- a/Duplicati/Library/Main/Strings.cs +++ b/Duplicati/Library/Main/Strings.cs @@ -382,6 +382,8 @@ namespace Duplicati.Library.Main.Strings public static string RestoreFileprocessorsLong { get { return LC.L("Use this option to set the number of concurrent FileProcessors processes used during restore. A FileProcessor processes one file at a time, and increasing the number of FileProcessors may improve restore performance."); } } public static string RestoreLegacyShort { get { return LC.L("Use legacy restore method"); } } public static string RestoreLegacyLong { get { return LC.L("Use this option to use the legacy restore method. The legacy restore method is slower than the new method, but may be more reliable in some cases."); } } + public static string RestoreAllFilesShort { get { return LC.L(@"Restore all targeted versions into timestamp folders"); } } + public static string RestoreAllFilesLong(string restorepathoptionname) => LC.L(@$"Use this option to restore every targeted backup version, instead of just the most recent matching one. Each version is restored into its own subfolder below the restore target, named with the backup timestamp (e.g. 20241224-130255). When set to ""{RestoreAllFilesMode.True}"", all (non-filtered) files are restored in every version. When set to ""{RestoreAllFilesMode.Unique}"", subsequent versions skip any file whose content (file hash) was already restored in a previous version. Setting this to ""{RestoreAllFilesMode.False}"" is the same as not setting the option. This option requires --{restorepathoptionname} to be set."); public static string RestorePreallocateSizeShort { get { return LC.L("Preallocate size of restored files"); } } public static string RestorePreallocateSizeLong { get { return LC.L("Use this option to toggle whether to set the size of the restored files before they are written to disk. This can help to reduce fragmentation and improve performance on some filesystems."); } } public static string RestoreVolumeCacheHintShort { get { return LC.L("Hints to the desired maximum size of the restore volume cache"); } } diff --git a/Duplicati/UnitTest/RestoreAllFilesTests.cs b/Duplicati/UnitTest/RestoreAllFilesTests.cs new file mode 100644 index 000000000..4285f680a --- /dev/null +++ b/Duplicati/UnitTest/RestoreAllFilesTests.cs @@ -0,0 +1,454 @@ +// Copyright (C) 2026, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Duplicati.Library.Main; +using NUnit.Framework; +using Assert = NUnit.Framework.Legacy.ClassicAssert; + +#nullable enable + +namespace Duplicati.UnitTest +{ + /// + /// Tests for the --restore-all-files restore option, which restores every + /// targeted backup version into its own timestamp-named subfolder below the restore + /// target. See for the supported modes. + /// + public class RestoreAllFilesTests : BasicSetupHelper + { + /// + /// The timestamp format used for per-version restore subfolders. + /// + private static readonly string VersionFolderFormat = "yyyyMMdd-HHmmss"; + + /// + /// Creates three backup versions with distinct, predictable contents and returns the + /// expected per-version contents. Versions are spaced at least one second apart so that + /// their fileset timestamps (and thus the per-version subfolder names) are distinct. + /// + private async Task> CreateThreeVersionsAsync() + { + var file1Path = Path.Combine(this.DATAFOLDER, "file1.txt"); + var file2Path = Path.Combine(this.DATAFOLDER, "file2.txt"); + + var versions = new List<(DateTime, string, string)>(); + + // Version 2 (oldest): file1 and file2 with initial content. + File.WriteAllText(file1Path, "v0-file1"); + File.WriteAllText(file2Path, "v0-file2"); + using (var c = new Controller("file://" + this.TARGETFOLDER, new Dictionary(this.TestOptions), null)) + TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER })); + versions.Add((DateTime.Now, "v0-file1", "v0-file2")); + await Task.Delay(1100); + + // Version 1: only file1 changes (file2 content unchanged => same hash as v0). + File.WriteAllText(file1Path, "v1-file1"); + using (var c = new Controller("file://" + this.TARGETFOLDER, new Dictionary(this.TestOptions), null)) + TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER })); + versions.Add((DateTime.Now, "v1-file1", "v0-file2")); + await Task.Delay(1100); + + // Version 0 (newest): both files change. + File.WriteAllText(file1Path, "v2-file1"); + File.WriteAllText(file2Path, "v2-file2"); + using (var c = new Controller("file://" + this.TARGETFOLDER, new Dictionary(this.TestOptions), null)) + TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER })); + versions.Add((DateTime.Now, "v2-file1", "v2-file2")); + + // versions[0] = oldest, versions[2] = newest. Return newest-first to match + // Duplicati's version indexing (version 0 = newest). + versions.Reverse(); + return versions; + } + + [Test] + [Category("RestoreHandler")] + public async Task RestoreAllFilesFalseBehavesLikeNormalRestoreAsync() + { + await CreateThreeVersionsAsync(); + + // --restore-all-files=false must behave like the option not being set. To prove + // equivalence, run the same restore twice into two separate folders (once with the + // option set to "false", once without the option) and assert the output trees are + // identical. + var withOptionFolder = Path.Combine(this.RESTOREFOLDER, "with-option"); + var withoutOptionFolder = Path.Combine(this.RESTOREFOLDER, "without-option"); + Directory.CreateDirectory(withOptionFolder); + Directory.CreateDirectory(withoutOptionFolder); + + var withOption = new Dictionary(this.TestOptions) + { + ["restore-path"] = withOptionFolder, + ["restore-all-files"] = "false" + }; + using (var c = new Controller("file://" + this.TARGETFOLDER, withOption, null)) + TestUtils.AssertResults(await c.RestoreAsync(new[] { "*" })); + + var withoutOption = new Dictionary(this.TestOptions) + { + ["restore-path"] = withoutOptionFolder + }; + using (var c = new Controller("file://" + this.TARGETFOLDER, withoutOption, null)) + TestUtils.AssertResults(await c.RestoreAsync(new[] { "*" })); + + // No timestamp subfolders should exist in the with-option folder; files are + // restored directly. + var subFolders = Directory.GetDirectories(withOptionFolder) + .Where(d => DateTime.TryParseExact(Path.GetFileName(d), VersionFolderFormat, null, System.Globalization.DateTimeStyles.None, out _)) + .ToArray(); + Assert.AreEqual(0, subFolders.Length, "No timestamp subfolders expected for restore-all-files=false"); + + // The two output trees must be identical. + AssertTreesEqual(withOptionFolder, withoutOptionFolder); + + var restoredFile1 = Path.Combine(withOptionFolder, "file1.txt"); + var restoredFile2 = Path.Combine(withOptionFolder, "file2.txt"); + Assert.IsTrue(File.Exists(restoredFile1), "file1.txt should be restored"); + Assert.IsTrue(File.Exists(restoredFile2), "file2.txt should be restored"); + Assert.AreEqual("v2-file1", File.ReadAllText(restoredFile1), "Newest file1 content expected"); + Assert.AreEqual("v2-file2", File.ReadAllText(restoredFile2), "Newest file2 content expected"); + } + + /// + /// Recursively asserts that two directory trees contain the same files with the same + /// contents. Used to prove that two restores produced identical output. + /// + private static void AssertTreesEqual(string expectedRoot, string actualRoot) + { + var expectedFiles = Directory.GetFiles(expectedRoot, "*", SearchOption.AllDirectories) + .Select(p => Path.GetRelativePath(expectedRoot, p)) + .OrderBy(p => p) + .ToArray(); + var actualFiles = Directory.GetFiles(actualRoot, "*", SearchOption.AllDirectories) + .Select(p => Path.GetRelativePath(actualRoot, p)) + .OrderBy(p => p) + .ToArray(); + Assert.AreEqual(expectedFiles.Length, actualFiles.Length, + $"File count mismatch between '{expectedRoot}' and '{actualRoot}'"); + for (var i = 0; i < expectedFiles.Length; i++) + { + Assert.AreEqual(expectedFiles[i], actualFiles[i], $"File path mismatch at index {i}"); + Assert.AreEqual( + File.ReadAllText(Path.Combine(expectedRoot, expectedFiles[i])), + File.ReadAllText(Path.Combine(actualRoot, actualFiles[i])), + $"Content mismatch for '{expectedFiles[i]}'"); + } + } + + [Test] + [Category("RestoreHandler")] + public async Task RestoreAllFilesTrueRestoresEveryVersionIntoTimestampFoldersAsync() + { + var versions = await CreateThreeVersionsAsync(); + + var restoreOptions = new Dictionary(this.TestOptions) + { + ["restore-path"] = this.RESTOREFOLDER, + ["restore-all-files"] = "true" + }; + + using (var c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null)) + TestUtils.AssertResults(await c.RestoreAsync(new[] { "*" })); + + var subFolders = Directory.GetDirectories(this.RESTOREFOLDER) + .Where(d => DateTime.TryParseExact(Path.GetFileName(d), VersionFolderFormat, null, System.Globalization.DateTimeStyles.None, out _)) + .OrderBy(d => Path.GetFileName(d)) + .ToArray(); + Assert.AreEqual(3, subFolders.Length, "Expected three timestamp subfolders (one per version)"); + + // Each subfolder contains both files from its version (true mode => no de-dup). + for (var i = 0; i < subFolders.Length; i++) + { + var folder = subFolders[i]; + // subFolders are ordered oldest -> newest; versions list is newest-first, + // so the oldest folder (index 0) corresponds to versions[2] (oldest). + var versionInfo = versions[versions.Count - 1 - i]; + var file1 = Path.Combine(folder, "file1.txt"); + var file2 = Path.Combine(folder, "file2.txt"); + Assert.IsTrue(File.Exists(file1), $"file1.txt missing in {Path.GetFileName(folder)}"); + Assert.IsTrue(File.Exists(file2), $"file2.txt missing in {Path.GetFileName(folder)}"); + Assert.AreEqual(versionInfo.File1Content, File.ReadAllText(file1), $"file1.txt content mismatch in {Path.GetFileName(folder)}"); + Assert.AreEqual(versionInfo.File2Content, File.ReadAllText(file2), $"file2.txt content mismatch in {Path.GetFileName(folder)}"); + } + } + + [Test] + [Category("RestoreHandler")] + public async Task RestoreAllFilesUniqueSkipsAlreadyRestoredContentAsync() + { + var versions = await CreateThreeVersionsAsync(); + + var restoreOptions = new Dictionary(this.TestOptions) + { + ["restore-path"] = this.RESTOREFOLDER, + ["restore-all-files"] = "unique" + }; + + using (var c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null)) + TestUtils.AssertResults(await c.RestoreAsync(new[] { "*" })); + + var subFolders = Directory.GetDirectories(this.RESTOREFOLDER) + .Where(d => DateTime.TryParseExact(Path.GetFileName(d), VersionFolderFormat, null, System.Globalization.DateTimeStyles.None, out _)) + .OrderBy(d => Path.GetFileName(d)) + .ToArray(); + Assert.AreEqual(3, subFolders.Length, "Expected three timestamp subfolders (one per version)"); + + // subFolders[0] = oldest, subFolders[2] = newest. The first version restored is + // the newest (versions[0]); it contains both files. Each subsequent (older) version + // restores only files whose content hash was not restored in a previous version. + // + // Content timeline (version -> file -> content): + // newest (v0): file1="v2-file1", file2="v2-file2" + // middle (v1): file1="v1-file1", file2="v0-file2" + // oldest (v2): file1="v0-file1", file2="v0-file2" + // + // Unique de-dup, processed newest-first, with the set of hashes already restored: + // | Version | file1 content | file2 content | file1 restored? | file2 restored? | + // |---------|---------------|---------------|-----------------|-----------------| + // | newest | v2-file1 | v2-file2 | yes (new) | yes (new) | + // | middle | v1-file1 | v0-file2 | yes (new) | yes (new) | + // | oldest | v0-file1 | v0-file2 | yes (new) | no (dup of middle's file2) | + // + // After the middle version, {v2-file1, v2-file2, v1-file1, v0-file2} are recorded. + // The oldest version's file2 ("v0-file2") is already in that set, so it is skipped. + + // Map folder -> version content. Folders oldest->newest map to versions newest-first reversed: + var folderVersionContent = new List<(string Folder, string File1Content, string File2Content, string VersionLabel)> + { + (subFolders[2], versions[0].File1Content, versions[0].File2Content, "newest"), + (subFolders[1], versions[1].File1Content, versions[1].File2Content, "middle"), + (subFolders[0], versions[2].File1Content, versions[2].File2Content, "oldest"), + }; + + // Newest folder: both files restored (first version, nothing de-duped yet). + var newest = folderVersionContent[0]; + Assert.IsTrue(File.Exists(Path.Combine(newest.Folder, "file1.txt")), "newest: file1 should be restored"); + Assert.IsTrue(File.Exists(Path.Combine(newest.Folder, "file2.txt")), "newest: file2 should be restored"); + Assert.AreEqual(newest.File1Content, File.ReadAllText(Path.Combine(newest.Folder, "file1.txt"))); + Assert.AreEqual(newest.File2Content, File.ReadAllText(Path.Combine(newest.Folder, "file2.txt"))); + + // Middle folder: file1 has a new hash (v1-file1) => restored. file2 has content + // v0-file2, whose hash was not restored in the newest version (which had v2-file2), + // so it is new => restored. + var middle = folderVersionContent[1]; + Assert.IsTrue(File.Exists(Path.Combine(middle.Folder, "file1.txt")), "middle: file1 should be restored (new content)"); + Assert.IsTrue(File.Exists(Path.Combine(middle.Folder, "file2.txt")), "middle: file2 should be restored (content not seen before)"); + Assert.AreEqual(middle.File1Content, File.ReadAllText(Path.Combine(middle.Folder, "file1.txt"))); + Assert.AreEqual(middle.File2Content, File.ReadAllText(Path.Combine(middle.Folder, "file2.txt"))); + + // Oldest folder: file1 has a new hash (v0-file1) => restored. file2 has content + // v0-file2, whose hash was already restored in the middle folder => SKIPPED. + var oldest = folderVersionContent[2]; + Assert.IsTrue(File.Exists(Path.Combine(oldest.Folder, "file1.txt")), "oldest: file1 should be restored (new content)"); + Assert.AreEqual(oldest.File1Content, File.ReadAllText(Path.Combine(oldest.Folder, "file1.txt"))); + Assert.IsFalse(File.Exists(Path.Combine(oldest.Folder, "file2.txt")), "oldest: file2 should be skipped (content already restored in middle folder)"); + } + + [Test] + [Category("RestoreHandler")] + public async Task RestoreAllFilesRequiresRestorePathAsync() + { + await CreateThreeVersionsAsync(); + + // No --restore-path set. The restore should fail because --restore-all-files + // requires a definitive target. In unittest mode the UserInformationException is + // propagated, so assert that the expected exception is thrown. + var restoreOptions = new Dictionary(this.TestOptions) + { + ["restore-all-files"] = "true" + }; + // Remove any default restore-path if present. + restoreOptions.Remove("restore-path"); + + using (var c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null)) + { + Assert.ThrowsAsync( + async () => await c.RestoreAsync(new[] { "*" }), + "Expected --restore-all-files without --restore-path to throw"); + } + } + + [Test] + [Category("RestoreHandler")] + public async Task RestoreAllFilesRespectsVersionSelectionAsync() + { + var versions = await CreateThreeVersionsAsync(); + + // Target only versions 0 and 2 (newest and oldest) via --version=0,2. + var restoreOptions = new Dictionary(this.TestOptions) + { + ["restore-path"] = this.RESTOREFOLDER, + ["restore-all-files"] = "true", + ["version"] = "0,2" + }; + + using (var c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null)) + TestUtils.AssertResults(await c.RestoreAsync(new[] { "*" })); + + var subFolders = Directory.GetDirectories(this.RESTOREFOLDER) + .Where(d => DateTime.TryParseExact(Path.GetFileName(d), VersionFolderFormat, null, System.Globalization.DateTimeStyles.None, out _)) + .ToArray(); + Assert.AreEqual(2, subFolders.Length, "Expected two timestamp subfolders for the two selected versions"); + + // The two folders correspond to the newest (version 0) and oldest (version 2) versions. + var orderedFolders = subFolders.OrderBy(d => Path.GetFileName(d)).ToArray(); + var oldestFolder = orderedFolders[0]; + var newestFolder = orderedFolders[1]; + + Assert.AreEqual(versions[0].File1Content, File.ReadAllText(Path.Combine(newestFolder, "file1.txt")), "newest folder file1"); + Assert.AreEqual(versions[0].File2Content, File.ReadAllText(Path.Combine(newestFolder, "file2.txt")), "newest folder file2"); + Assert.AreEqual(versions[2].File1Content, File.ReadAllText(Path.Combine(oldestFolder, "file1.txt")), "oldest folder file1"); + Assert.AreEqual(versions[2].File2Content, File.ReadAllText(Path.Combine(oldestFolder, "file2.txt")), "oldest folder file2"); + } + + [Test] + [Category("RestoreHandler")] + public async Task RestoreAllFilesWithTimeDoesNotPollutePerVersionFoldersAsync() + { + // Regression test: when --time and --restore-all-files are combined, each + // per-version iteration must restore exactly that single version's files into its + // own subfolder. Previously the --time option was not cleared between iterations, + // so the underlying file-list query (which combines --time and --version with OR) + // pulled in all filesets at or before --time on every iteration, polluting each + // subfolder with files from multiple versions. + var versions = await CreateThreeVersionsAsync(); + + // Use --time set to the middle version's post-backup timestamp. This selects the + // middle and oldest versions (both at or before --time); the newest version is after + // the cutoff and excluded. Each targeted version is restored into its own subfolder + // with exactly its own files. Use the invariant sortable format (yyyy-MM-ddTHH:mm:ss) + // which Timeparser.ParseTimeInterval accepts via DateTime.TryParse with AssumeLocal. + var middleTime = versions[1].Time; + var timeSpec = middleTime.ToString("s", System.Globalization.CultureInfo.InvariantCulture); + + var restoreOptions = new Dictionary(this.TestOptions) + { + ["restore-path"] = this.RESTOREFOLDER, + ["restore-all-files"] = "true", + ["time"] = timeSpec + }; + + using (var c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null)) + TestUtils.AssertResults(await c.RestoreAsync(new[] { "*" })); + + var subFolders = Directory.GetDirectories(this.RESTOREFOLDER) + .Where(d => DateTime.TryParseExact(Path.GetFileName(d), VersionFolderFormat, null, System.Globalization.DateTimeStyles.None, out _)) + .OrderBy(d => Path.GetFileName(d)) + .ToArray(); + Assert.AreEqual(2, subFolders.Length, "Expected two timestamp subfolders (middle and oldest, both at or before --time)"); + + // Each subfolder must contain exactly its own version's two files -- no cross-version + // pollution. The older folder maps to the oldest version, the newer folder to the + // middle version (both at or before --time). + var orderedFolders = subFolders.OrderBy(d => Path.GetFileName(d)).ToArray(); + var oldestFolder = orderedFolders[0]; + var middleFolder = orderedFolders[1]; + + Assert.AreEqual(versions[2].File1Content, File.ReadAllText(Path.Combine(oldestFolder, "file1.txt")), "oldest folder file1"); + Assert.AreEqual(versions[2].File2Content, File.ReadAllText(Path.Combine(oldestFolder, "file2.txt")), "oldest folder file2"); + Assert.AreEqual(versions[1].File1Content, File.ReadAllText(Path.Combine(middleFolder, "file1.txt")), "middle folder file1"); + Assert.AreEqual(versions[1].File2Content, File.ReadAllText(Path.Combine(middleFolder, "file2.txt")), "middle folder file2"); + + // Each folder must contain exactly two files (file1.txt and file2.txt) -- if the + // --time bug were present, a folder could contain duplicate/extra files from the + // other version. Assert no extra files leaked in. + foreach (var folder in subFolders) + { + var files = Directory.GetFiles(folder); + Assert.AreEqual(2, files.Length, $"Folder {Path.GetFileName(folder)} should contain exactly 2 files, found {files.Length}: {string.Join(", ", files.Select(Path.GetFileName))}"); + } + } + + [Test] + [Category("RestoreHandler")] + public async Task RestoreAllFilesTrueLegacyRestoresEveryVersionIntoTimestampFoldersAsync() + { + // --restore-legacy exercises the DoRunAsync path, which has its own + // HarvestRestoredHashesAsync wiring. Ensure the multi-version restore works there too. + var versions = await CreateThreeVersionsAsync(); + + var restoreOptions = new Dictionary(this.TestOptions) + { + ["restore-path"] = this.RESTOREFOLDER, + ["restore-all-files"] = "true", + ["restore-legacy"] = "true" + }; + + using (var c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null)) + TestUtils.AssertResults(await c.RestoreAsync(new[] { "*" })); + + var subFolders = Directory.GetDirectories(this.RESTOREFOLDER) + .Where(d => DateTime.TryParseExact(Path.GetFileName(d), VersionFolderFormat, null, System.Globalization.DateTimeStyles.None, out _)) + .OrderBy(d => Path.GetFileName(d)) + .ToArray(); + Assert.AreEqual(3, subFolders.Length, "Expected three timestamp subfolders (one per version) with --restore-legacy"); + + for (var i = 0; i < subFolders.Length; i++) + { + var folder = subFolders[i]; + var versionInfo = versions[versions.Count - 1 - i]; + var file1 = Path.Combine(folder, "file1.txt"); + var file2 = Path.Combine(folder, "file2.txt"); + Assert.IsTrue(File.Exists(file1), $"file1.txt missing in {Path.GetFileName(folder)} (legacy)"); + Assert.IsTrue(File.Exists(file2), $"file2.txt missing in {Path.GetFileName(folder)} (legacy)"); + Assert.AreEqual(versionInfo.File1Content, File.ReadAllText(file1), $"file1.txt content mismatch in {Path.GetFileName(folder)} (legacy)"); + Assert.AreEqual(versionInfo.File2Content, File.ReadAllText(file2), $"file2.txt content mismatch in {Path.GetFileName(folder)} (legacy)"); + } + } + + [Test] + [Category("RestoreHandler")] + public async Task RestoreAllFilesUniqueCleansUpScratchTableAsync() + { + // The unique mode tracks restored hashes in a persistent scratch table in the local + // database (not in memory). Verify the table is dropped after the run so it does not + // leak into the database file. + await CreateThreeVersionsAsync(); + + var restoreOptions = new Dictionary(this.TestOptions) + { + ["restore-path"] = this.RESTOREFOLDER, + ["restore-all-files"] = "unique" + }; + + using (var c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null)) + TestUtils.AssertResults(await c.RestoreAsync(new[] { "*" })); + + // Inspect the local database for any leftover "RestoredHashes-*" scratch tables. + using var connection = new Microsoft.Data.Sqlite.SqliteConnection($"Data Source={this.DBFILE}"); + await connection.OpenAsync(); + await using var cmd = connection.CreateCommand(); + cmd.CommandText = "SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'RestoredHashes-%'"; + await using var reader = await cmd.ExecuteReaderAsync(); + var leftoverTables = new List(); + while (await reader.ReadAsync()) + leftoverTables.Add(reader.GetString(0)); + + Assert.AreEqual(0, leftoverTables.Count, + $"The restored-hashes scratch table should be dropped after the run, but found: {string.Join(", ", leftoverTables)}"); + } + } +}