From e7b998f909988b34392b2d6f0cc78fecbb7d07ee Mon Sep 17 00:00:00 2001 From: aureliandevel Date: Sat, 28 Feb 2026 20:14:22 +1100 Subject: [PATCH] Enhance restore volume caching: implement disk pressure eviction logic and add related options --- .../Main/Operation/Restore/VolumeManager.cs | 58 ++++++++++++++++-- Duplicati/Library/Main/Options.cs | 24 ++++++-- Duplicati/Library/Main/ResultClasses.cs | 19 ++++++ Duplicati/Library/Main/Strings.cs | 4 +- Duplicati/UnitTest/OptionsTests.cs | 30 +++++++++ Duplicati/UnitTest/RestoreHandlerTests.cs | 61 ++++++++++++++++++- 6 files changed, 184 insertions(+), 12 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/VolumeManager.cs b/Duplicati/Library/Main/Operation/Restore/VolumeManager.cs index 663e68aee..3b5eae95b 100644 --- a/Duplicati/Library/Main/Operation/Restore/VolumeManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/VolumeManager.cs @@ -25,6 +25,7 @@ using Duplicati.Library.Utility; using System; using System.Collections.Generic; using System.Diagnostics; +using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -45,6 +46,11 @@ namespace Duplicati.Library.Main.Operation.Restore /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(); + /// + /// Number of disk-pressure evictions that must occur before a mid-run warning is emitted. + /// + private const int CACHE_PRESSURE_WARNING_THRESHOLD = 5; + /// /// Helper class to read from either of two channels. /// This is a workaround for the fact that CoCoL seems to deadlock on ReadFroAnyAsync @@ -108,7 +114,12 @@ namespace Duplicati.Library.Main.Operation.Restore async self => { // The maximum number of volumes to have in cache at once. If this is exceeded, we'll try to evict the least recently used volume that is not actively in use. + // -1 = unlimited (disk-space-aware), 0 = disabled, >0 = hard cap in bytes. long cache_max = options.RestoreVolumeCacheHint; + // Minimum free space (bytes) to maintain in temp dir — used only in unlimited mode. + long cache_min_free = options.RestoreVolumeCacheMinFree; + // Temp directory path used for DriveInfo queries in unlimited mode. + string temp_dir = options.TempDir; // Cache of volume readers. Dictionary cache = []; // Current size of the cache in bytes. @@ -117,6 +128,13 @@ namespace Duplicati.Library.Main.Operation.Restore List cache_last_touched = []; // Dictionary to keep track of active downloads. Used for grouping requests to the same volume. Dictionary> in_flight_downloads = []; + // Disk-pressure counters (unlimited cache mode only). + HashSet previously_evicted_volume_ids = []; + HashSet all_accessed_volume_ids = []; + long disk_pressure_evictions = 0; + long disk_pressure_redownloads = 0; + long total_volumes_accessed = 0; + bool cache_exhausted_warned = false; Stopwatch? sw_cache_set = options.InternalProfiling ? new() : null; Stopwatch? sw_cache_evict = options.InternalProfiling ? new() : null; @@ -196,6 +214,10 @@ namespace Duplicati.Library.Main.Operation.Restore } else { + if (all_accessed_volume_ids.Add(request.VolumeID)) + total_volumes_accessed++; + if (previously_evicted_volume_ids.Contains(request.VolumeID)) + disk_pressure_redownloads++; await self.DownloadRequest.WriteAsync(request.VolumeID).ConfigureAwait(false); in_flight_downloads[request.VolumeID] = [request]; } @@ -211,9 +233,15 @@ namespace Duplicati.Library.Main.Operation.Restore { sw_cache_set?.Start(); volume.Reference(in_flight_downloads[volume_id].Count); - if (cache_max > 0) + if (cache_max == 0) { - // Check if adding another volume would exceed cache limits. + // Caching disabled — dispose immediately. + Logging.Log.WriteExplicitMessage(LOGTAG, "VolumeRequest", "Not caching volume {0} (caching disabled)", volume_id); + volume.Dispose(); + } + else if (cache_max > 0) + { + // Hard-cap LRU: evict until the new volume fits. while (cache_size > 0 && (cache_size + volume.Size) > cache_max) { // TODO switch based of the eviction strategy. @@ -230,8 +258,26 @@ namespace Duplicati.Library.Main.Operation.Restore } else { - Logging.Log.WriteExplicitMessage(LOGTAG, "VolumeRequest", "Not caching volume {0} ({1} + {2} > {3})", volume_id, cache_size, volume.Size, cache_max); - volume.Dispose(); + // Unlimited mode (cache_max < 0): evict LRU while free space is below the minimum. + var available_free_space = new DriveInfo(temp_dir).AvailableFreeSpace; + while (available_free_space < cache_min_free && cache_last_touched.Count > 0) + { + var evict_id = cache_last_touched[0]; + evict_lru(); + previously_evicted_volume_ids.Add(evict_id); + disk_pressure_evictions++; + if (disk_pressure_evictions == CACHE_PRESSURE_WARNING_THRESHOLD) + Logging.Log.WriteWarningMessage(LOGTAG, "CachePressure", null, "Restore volume cache has begun evicting cached volumes due to low disk space in '{0}'. Restore performance may be degraded.", temp_dir); + available_free_space = new DriveInfo(temp_dir).AvailableFreeSpace; + } + if (!cache_exhausted_warned && cache_last_touched.Count == 0 && available_free_space < cache_min_free) + { + cache_exhausted_warned = true; + Logging.Log.WriteWarningMessage(LOGTAG, "CacheExhausted", null, "Restore volume cache is empty but disk space in '{0}' is still below the configured minimum. Performance impact is likely.", temp_dir); + } + Logging.Log.WriteExplicitMessage(LOGTAG, "VolumeRequest", "Caching volume {0} in unlimited mode (free space: {1})", volume_id, available_free_space); + cache[volume_id] = volume; + cache_size += volume.Size; } cache_last_touched.Add(volume_id); sw_cache_set?.Stop(); @@ -255,6 +301,10 @@ namespace Duplicati.Library.Main.Operation.Restore { Logging.Log.WriteVerboseMessage(LOGTAG, "RetiredProcess", null, "Volume manager retired"); + results.CachePressureEvictions = disk_pressure_evictions; + results.CachePressureRedownloads = disk_pressure_redownloads; + results.TotalVolumesAccessed = total_volumes_accessed; + if (options.InternalProfiling) { Logging.Log.WriteProfilingMessage(LOGTAG, "InternalTimings", $"CacheSet: {sw_cache_set?.ElapsedMilliseconds}ms, CacheEvict: {sw_cache_evict?.ElapsedMilliseconds}ms, Query: {sw_query?.ElapsedMilliseconds}ms, Backend: {sw_backend?.ElapsedMilliseconds}ms, Request: {sw_request?.ElapsedMilliseconds}ms, Wakeup: {sw_wakeup?.ElapsedMilliseconds}ms"); diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index c7cd25533..1c28040a3 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -164,10 +164,15 @@ namespace Duplicati.Library.Main private static readonly int DEFAULT_RESTORE_FILE_PROCESSORS = Math.Max(1, Environment.ProcessorCount / 2); /// - /// The default value for the maximum size of the restore volume cache is 100 times the volume size. E.g. for 50MB volumes, the default cache size is 5GB. + /// The default value for the restore volume cache hint. When empty, the cache is unlimited (disk-space-aware eviction via ). /// private const string DEFAULT_RESTORE_VOLUME_CACHE_HINT = ""; + /// + /// The default minimum free disk space to maintain in the temp directory during restore (1 GB). + /// + private const string DEFAULT_RESTORE_VOLUME_CACHE_MIN_FREE = "1gb"; + /// /// The default value for the number of volume decryptors during restore /// @@ -575,6 +580,7 @@ namespace Duplicati.Library.Main new CommandLineArgument("restore-legacy", CommandLineArgument.ArgumentType.Boolean, Strings.Options.RestoreLegacyShort, Strings.Options.RestoreLegacyLong, "false"), new CommandLineArgument("restore-preallocate-size", CommandLineArgument.ArgumentType.Boolean, Strings.Options.RestorePreallocateSizeShort, Strings.Options.RestorePreallocateSizeLong, "false"), new CommandLineArgument("restore-volume-cache-hint", CommandLineArgument.ArgumentType.Size, Strings.Options.RestoreVolumeCacheHintShort, Strings.Options.RestoreVolumeCacheHintLong, DEFAULT_RESTORE_VOLUME_CACHE_HINT), + new CommandLineArgument("restore-volume-cache-min-free", CommandLineArgument.ArgumentType.Size, Strings.Options.RestoreVolumeCacheMinFreeShort, Strings.Options.RestoreVolumeCacheMinFreeLong, DEFAULT_RESTORE_VOLUME_CACHE_MIN_FREE), new CommandLineArgument("restore-volume-decompressors", CommandLineArgument.ArgumentType.Integer, Strings.Options.RestoreVolumeDecompressorsShort, Strings.Options.RestoreVolumeDecompressorsLong, DEFAULT_RESTORE_VOLUME_DECOMPRESSORS.ToString()), new CommandLineArgument("restore-volume-decryptors", CommandLineArgument.ArgumentType.Integer, Strings.Options.RestoreVolumeDecryptorsShort, Strings.Options.RestoreVolumeDecryptorsLong, DEFAULT_RESTORE_VOLUME_DECRYPTORS.ToString()), new CommandLineArgument("restore-volume-downloaders", CommandLineArgument.ArgumentType.Integer, Strings.Options.RestoreVolumeDownloadersShort, Strings.Options.RestoreVolumeDownloadersLong, DEFAULT_RESTORE_VOLUME_DOWNLOADERS.ToString()), @@ -1805,9 +1811,10 @@ namespace Duplicati.Library.Main /// public string? MacOSPhotoLibraryPath => GetString("photos-library-path", null); - // - // Gets the size of the volume cache used during restore, in MB. - // + /// + /// Gets the maximum size of the restore volume cache in bytes. + /// Returns -1 when unset (unlimited/disk-space-aware mode), 0 to disable caching, or a positive byte count for a hard cap. + /// public long RestoreVolumeCacheHint { get @@ -1815,12 +1822,19 @@ namespace Duplicati.Library.Main m_options.TryGetValue("restore-volume-cache-hint", out var s); if (string.IsNullOrEmpty(s)) - return VolumeSize * 100; // Default to 100 times the volume size + return -1L; // Unlimited: disk-space-aware eviction bounded by RestoreVolumeCacheMinFree return GetSize("restore-volume-cache-hint", "mb", $"{VolumeSize * 100}b"); } } + /// + /// Gets the minimum free disk space (in bytes) to maintain in the temp directory during restore. + /// Only used when returns -1 (unlimited mode). + /// + public long RestoreVolumeCacheMinFree + => GetSize("restore-volume-cache-min-free", "gb", DEFAULT_RESTORE_VOLUME_CACHE_MIN_FREE); + /// /// Gets the number of volume decryptors to use in the restore process /// diff --git a/Duplicati/Library/Main/ResultClasses.cs b/Duplicati/Library/Main/ResultClasses.cs index d99e40d49..f445700e7 100644 --- a/Duplicati/Library/Main/ResultClasses.cs +++ b/Duplicati/Library/Main/ResultClasses.cs @@ -539,6 +539,25 @@ namespace Duplicati.Library.Main public long DeletedSymlinks { get; internal set; } public string RestorePath { get; internal set; } + /// + /// Number of LRU evictions triggered by low disk space in the temp directory + /// (disk-pressure path, unlimited cache mode only). + /// + public long CachePressureEvictions { get; internal set; } + public bool ShouldSerializeCachePressureEvictions() => CachePressureEvictions > 0; + + /// + /// Number of volumes re-downloaded after being evicted due to disk pressure. + /// + public long CachePressureRedownloads { get; internal set; } + public bool ShouldSerializeCachePressureRedownloads() => CachePressureRedownloads > 0; + + /// + /// Total number of distinct dblock volumes accessed during the restore. + /// + public long TotalVolumesAccessed { get; internal set; } + public bool ShouldSerializeTotalVolumesAccessed() => TotalVolumesAccessed > 0; + public override OperationMode MainOperation { get { return OperationMode.Restore; } } public IRecreateDatabaseResults RecreateDatabaseResults { get; internal set; } diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs index 751c328c2..99211c506 100644 --- a/Duplicati/Library/Main/Strings.cs +++ b/Duplicati/Library/Main/Strings.cs @@ -374,7 +374,9 @@ namespace Duplicati.Library.Main.Strings public static string RestorePreallocateSizeShort { get { return LC.L("Preallocate size of restored files"); } } public static string RestorePreallocateSizeLong { get { return LC.L("Use this option to toggle whether to set the size of the restored files before they are written to disk. This can help to reduce fragmentation and improve performance on some filesystems."); } } public static string RestoreVolumeCacheHintShort { get { return LC.L("Hints to the desired maximum size of the restore volume cache"); } } - public static string RestoreVolumeCacheHintLong { get { return LC.L("Use this option to hint the maximum desired size of the restore volume cache. The restore volume cache is used to store the volumes from the remote storage on local disk. When this number is exceeded, the client tries to evict the least recently used volume(s). This means that the number of volumes kept on disk can exceed this value if they are still being downloaded or are in use."); } } + public static string RestoreVolumeCacheHintLong { get { return LC.L("Use this option to hint the maximum desired size of the restore volume cache. The restore volume cache is used to store the volumes from the remote storage on local disk. When this number is exceeded, the client tries to evict the least recently used volume(s). This means that the number of volumes kept on disk can exceed this value if they are still being downloaded or are in use. A bare number is interpreted as megabytes. When left unset, the cache is unlimited and disk-space-aware eviction is used instead (see restore-volume-cache-min-free). Set to 0 to disable caching entirely."); } } + public static string RestoreVolumeCacheMinFreeShort { get { return LC.L("Minimum free disk space to maintain in the temp directory during restore"); } } + public static string RestoreVolumeCacheMinFreeLong { get { return LC.L("When restore-volume-cache-hint is unset (unlimited mode), the restore volume cache evicts least-recently-used volumes whenever the free space in the temp directory drops below this value. A bare number is interpreted as gigabytes. Has no effect when restore-volume-cache-hint is set to a specific size or to 0."); } } public static string RestoreVolumeDecompressorsShort { get { return LC.L("Number of concurrent FileDecompressor processes used during restore"); } } public static string RestoreVolumeDecompressorsLong { get { return LC.L("Use this option to set the number of concurrent FileDecompressor processes used during restore. A FileDecompressor processes one volume at a time, and increasing the number of FileDecompressors may improve restore performance if the bottleneck is decompression."); } } public static string RestoreVolumeDecryptorsShort { get { return LC.L("Number of concurrent FileDecryptor processes used during restore"); } } diff --git a/Duplicati/UnitTest/OptionsTests.cs b/Duplicati/UnitTest/OptionsTests.cs index 011d49750..de6569ef7 100644 --- a/Duplicati/UnitTest/OptionsTests.cs +++ b/Duplicati/UnitTest/OptionsTests.cs @@ -229,5 +229,35 @@ namespace Duplicati.UnitTest Assert.That(options.IndexfilePolicy, Is.EqualTo(Options.IndexFileStrategy.Lookup)); } + + [Test] + [Category("Options")] + public void RestoreVolumeCacheHintDefaultsToUnlimited() + { + var options = CreateOptions(); + + Assert.That(options.RestoreVolumeCacheHint, Is.EqualTo(-1L)); + } + + [Test] + [Category("Options")] + public void RestoreVolumeCacheMinFreeDefaultsToOneGigabyte() + { + var options = CreateOptions(); + + Assert.That(options.RestoreVolumeCacheMinFree, Is.EqualTo(1L * 1024 * 1024 * 1024)); + } + + [Test] + [Category("Options")] + public void RestoreVolumeCacheMinFreeBareNumberTreatedAsGigabytes() + { + var options = CreateOptions(new Dictionary + { + { "restore-volume-cache-min-free", "2" } + }); + + Assert.That(options.RestoreVolumeCacheMinFree, Is.EqualTo(2L * 1024 * 1024 * 1024)); + } } } diff --git a/Duplicati/UnitTest/RestoreHandlerTests.cs b/Duplicati/UnitTest/RestoreHandlerTests.cs index b6ee78c7e..e2a8d2d07 100644 --- a/Duplicati/UnitTest/RestoreHandlerTests.cs +++ b/Duplicati/UnitTest/RestoreHandlerTests.cs @@ -166,12 +166,13 @@ namespace Duplicati.UnitTest [Test] [Category("RestoreHandler")] - public async System.Threading.Tasks.Task RestoreVolumeCache([Values("0b", "1mb", "5mb", "1gb")] string cache_size, [Values("0", "1", null)] string? channel_size) + public async System.Threading.Tasks.Task RestoreVolumeCache([Values("0b", "1mb", "5mb", "1gb", null)] string? cache_size, [Values("0", "1", null)] string? channel_size) { var opts = TestOptions; opts["dblock-size"] = "1mb"; opts["blocksize"] = "1kb"; - opts["restore-volume-cache-hint"] = cache_size; + if (cache_size != null) + opts["restore-volume-cache-hint"] = cache_size; if (channel_size != null) opts["restore-channel-buffer-size"] = channel_size; opts["restore-legacy"] = "false"; @@ -281,5 +282,61 @@ namespace Duplicati.UnitTest TestUtils.AssertResults(res_restore); Assert.AreEqual(original_contents, File.ReadAllBytes(file1Path)); } + + [Test] + [Category("RestoreHandler")] + public async System.Threading.Tasks.Task RestoreVolumeCacheDiskPressure() + { + var opts = TestOptions; + opts["dblock-size"] = "1mb"; + opts["blocksize"] = "1kb"; + // No restore-volume-cache-hint → unlimited mode (-1 sentinel). + // Set restore-volume-cache-min-free to an absurdly large value so + // every volume arrival triggers disk-pressure eviction. + opts["restore-volume-cache-min-free"] = "999tb"; + opts["restore-legacy"] = "false"; + opts["restore-path"] = RESTOREFOLDER; + + // Write enough data to create at least 10 dblock volumes (dblock-size=1mb), + // so that the eviction loop is entered multiple times and the CachePressure + // warning threshold (5 evictions) is reliably crossed. + Random rng = new(); + for (int i = 0; i < 20; i++) + { + var data = new byte[512 * 1024]; // 512 KB each → ~10 MB total → ~10 dblock volumes + rng.NextBytes(data); + File.WriteAllBytes(Path.Combine(this.DATAFOLDER, $"file{i}"), data); + } + + using var c = new Controller("file://" + this.TARGETFOLDER, opts, null); + TestUtils.AssertResults(c.Backup([this.DATAFOLDER])); + + var timeout_task = System.Threading.Tasks.Task.Delay(TimeSpan.FromSeconds(120)); + RestoreResults? result = null; + + var restore_task = System.Threading.Tasks.Task.Run(() => + { + result = (RestoreResults)c.Restore(["*"]); + }); + + var t = await System.Threading.Tasks.Task.WhenAny(timeout_task, restore_task); + if (t == timeout_task) + { + c.Abort(); + await restore_task; + Assert.Fail("Restore timed out"); + } + else + { + t.GetAwaiter().GetResult(); + } + + Assert.IsNotNull(result); + Assert.That(result!.CachePressureEvictions, Is.GreaterThan(0), "Expected disk-pressure evictions with 999tb min-free"); + Assert.That(result.TotalVolumesAccessed, Is.GreaterThan(0), "Expected at least one volume to be accessed"); + Assert.That(result.Warnings.Count(), Is.GreaterThanOrEqualTo(1), "Expected at least one CachePressure warning"); + + TestUtils.AssertDirectoryTreesAreEquivalent(this.DATAFOLDER, this.RESTOREFOLDER, true, "Restoring with disk pressure eviction"); + } } } \ No newline at end of file