Kenneth Skovhede
179d1ceb47
This PR has a large blast radius because it takes the final step and bumps up the Controller to be fully async. We have historically done a piece-by-piece update, so all operations were already async but the controller interface was kept synchronous. With this update, the controller is now fully async and all tests are updated. Most places where the new C# compiler warns about function names not ending in `Async` were also adressed, giving a massive refactor change. Functionally, no changes are done.
441 lines
20 KiB
C#
441 lines
20 KiB
C#
// Copyright (C) 2026, The Duplicati Team
|
|
// https://duplicati.com, hello@duplicati.com
|
|
//
|
|
// Permission is hereby granted, free of charge, to any person obtaining a
|
|
// copy of this software and associated documentation files (the "Software"),
|
|
// to deal in the Software without restriction, including without limitation
|
|
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
// and/or sell copies of the Software, and to permit persons to whom the
|
|
// Software is furnished to do so, subject to the following conditions:
|
|
//
|
|
// The above copyright notice and this permission notice shall be included in
|
|
// all copies or substantial portions of the Software.
|
|
//
|
|
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
|
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
// DEALINGS IN THE SOFTWARE.
|
|
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.IO;
|
|
using System.Linq;
|
|
using System.Threading.Tasks;
|
|
using Duplicati.Library.Common.IO;
|
|
using Duplicati.Library.Interface;
|
|
using Duplicati.Library.Main;
|
|
using NUnit.Framework;
|
|
using Assert = NUnit.Framework.Legacy.ClassicAssert;
|
|
|
|
#nullable enable
|
|
|
|
namespace Duplicati.UnitTest
|
|
{
|
|
public class RestoreHandlerTests : BasicSetupHelper
|
|
{
|
|
|
|
[Test]
|
|
[Category("RestoreHandler")]
|
|
public async Task DisablePipedStreamingAsync()
|
|
{
|
|
var filePath = Path.Combine(this.DATAFOLDER, "file");
|
|
File.WriteAllBytes(filePath, new byte[] { 0 });
|
|
|
|
var options = new Dictionary<string, string>(this.TestOptions);
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
|
|
await c.BackupAsync(new[] { this.DATAFOLDER });
|
|
|
|
var restoreOptions = new Dictionary<string, string>(this.TestOptions) { ["restore-path"] = this.RESTOREFOLDER };
|
|
// This is now the default behavior, so we cannot explicitly disable it
|
|
//restoreOptions["disable-piped-streaming"] = "true";
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
|
|
{
|
|
var restoreResults = await c.RestoreAsync(new[] { filePath });
|
|
Assert.AreEqual(0, restoreResults.Errors.Count());
|
|
Assert.AreEqual(0, restoreResults.Warnings.Count());
|
|
}
|
|
|
|
var restoredFilePath = Path.Combine(this.RESTOREFOLDER, "file");
|
|
Assert.IsTrue(File.Exists(restoredFilePath));
|
|
}
|
|
|
|
[Test]
|
|
[Category("RestoreHandler")]
|
|
public async Task RestoreEmptyFileAsync()
|
|
{
|
|
var folderPath = Path.Combine(this.DATAFOLDER, "folder");
|
|
Directory.CreateDirectory(folderPath);
|
|
var filePath = Path.Combine(folderPath, "empty_file");
|
|
File.WriteAllBytes(filePath, new byte[] { });
|
|
|
|
var options = new Dictionary<string, string>(this.TestOptions);
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
|
|
{
|
|
var backupResults = await c.BackupAsync(new[] { this.DATAFOLDER });
|
|
Assert.AreEqual(0, backupResults.Errors.Count());
|
|
Assert.AreEqual(0, backupResults.Warnings.Count());
|
|
}
|
|
|
|
// Issue #4148 described a situation where the folders containing the empty file were not recreated properly.
|
|
var restoreOptions = new Dictionary<string, string>(this.TestOptions)
|
|
{
|
|
["restore-path"] = this.RESTOREFOLDER,
|
|
["dont-compress-restore-paths"] = "true"
|
|
};
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
|
|
{
|
|
var restoreResults = await c.RestoreAsync(new[] { filePath });
|
|
Assert.AreEqual(0, restoreResults.Errors.Count());
|
|
// TODO The expected warning is expected, as the 'dont-compress-restore-paths' option results in a warning about a folder not being created before restoring a file.
|
|
Assert.AreEqual(1, restoreResults.Warnings.Count());
|
|
}
|
|
|
|
// We need to strip the root part of the path. Otherwise, Path.Combine will simply return the second argument
|
|
// if it's determined to be an absolute path.
|
|
var rootString = SystemIO.IO_OS.GetPathRoot(filePath);
|
|
var newPathPart = filePath.Substring(rootString.Length);
|
|
if (OperatingSystem.IsWindows())
|
|
{
|
|
// On Windows, the drive letter is included in the path when the dont-compress-restore-paths option is used.
|
|
// The drive letter is assumed to be the first character of the path root (e.g., C:\).
|
|
newPathPart = Path.Combine(rootString.Substring(0, 1), filePath.Substring(rootString.Length));
|
|
}
|
|
|
|
var restoredFilePath = Path.Combine(restoreOptions["restore-path"], newPathPart);
|
|
Assert.IsTrue(File.Exists(restoredFilePath));
|
|
}
|
|
|
|
[Test]
|
|
[Category("RestoreHandler")]
|
|
public async Task RestoreInheritanceBreaksAsync()
|
|
{
|
|
if (!OperatingSystem.IsWindows())
|
|
{
|
|
return;
|
|
}
|
|
|
|
var folderPath = Path.Combine(this.DATAFOLDER, "folder");
|
|
Directory.CreateDirectory(folderPath);
|
|
var filePath = Path.Combine(folderPath, "file");
|
|
File.WriteAllBytes(filePath, new byte[] { 0 });
|
|
|
|
// Protect access rules on the file.
|
|
var fileSecurity = new FileInfo(filePath).GetAccessControl();
|
|
fileSecurity.SetAccessRuleProtection(true, true);
|
|
new FileInfo(filePath).SetAccessControl(fileSecurity);
|
|
|
|
var options = new Dictionary<string, string>(this.TestOptions);
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
|
|
TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER }));
|
|
|
|
// First, restore without restoring permissions.
|
|
var restoreOptions = new Dictionary<string, string>(this.TestOptions) { ["restore-path"] = this.RESTOREFOLDER };
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
|
|
{
|
|
TestUtils.AssertResults(await c.RestoreAsync(new[] { filePath }));
|
|
|
|
var restoredFilePath = Path.Combine(this.RESTOREFOLDER, "file");
|
|
Assert.IsTrue(File.Exists(restoredFilePath));
|
|
|
|
var restoredFileSecurity = new FileInfo(restoredFilePath).GetAccessControl();
|
|
Assert.IsFalse(restoredFileSecurity.AreAccessRulesProtected);
|
|
|
|
// Remove the restored file so that the later restore avoids the "Restore completed
|
|
// without errors but no files were restored" warning.
|
|
File.Delete(restoredFilePath);
|
|
}
|
|
|
|
// Restore with restoring permissions.
|
|
restoreOptions["overwrite"] = "true";
|
|
restoreOptions["restore-permissions"] = "true";
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
|
|
{
|
|
TestUtils.AssertResults(await c.RestoreAsync(new[] { filePath }));
|
|
|
|
var restoredFilePath = Path.Combine(this.RESTOREFOLDER, "file");
|
|
Assert.IsTrue(File.Exists(restoredFilePath));
|
|
|
|
var restoredFileSecurity = new FileInfo(restoredFilePath).GetAccessControl();
|
|
Assert.IsTrue(restoredFileSecurity.AreAccessRulesProtected);
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
[Category("RestoreHandler")]
|
|
public async Task RestoreVolumeCacheAsync([Values("0b", "1mb", "5mb", "1gb", null)] string? cache_size, [Values("0", "1", null)] string? channel_size)
|
|
{
|
|
var opts = TestOptions;
|
|
opts["dblock-size"] = "1mb";
|
|
opts["blocksize"] = "1kb";
|
|
if (cache_size != null)
|
|
opts["restore-volume-cache-hint"] = cache_size;
|
|
if (channel_size != null)
|
|
opts["restore-channel-buffer-size"] = channel_size;
|
|
opts["restore-legacy"] = "false";
|
|
opts["restore-file-processors"] = "4";
|
|
opts["restore-volume-decryptors"] = "4";
|
|
opts["restore-volume-decompressors"] = "4";
|
|
opts["restore-volume-downloaders"] = "4";
|
|
opts["restore-path"] = RESTOREFOLDER;
|
|
|
|
// Write a bunch of files
|
|
Random rng = new();
|
|
byte[] data = new byte[1024];
|
|
for (int i = 0; i < 1000; i++)
|
|
{
|
|
rng.NextBytes(data);
|
|
var filePath = Path.Combine(this.DATAFOLDER, $"file{i}");
|
|
File.WriteAllBytes(filePath, data);
|
|
}
|
|
|
|
using var c = new Controller("file://" + this.TARGETFOLDER, opts, null);
|
|
TestUtils.AssertResults(await c.BackupAsync([this.DATAFOLDER]));
|
|
|
|
// Start a 30 second timeout
|
|
var timeout_task = System.Threading.Tasks.Task.Delay(TimeSpan.FromSeconds(30));
|
|
|
|
var restore_task = System.Threading.Tasks.Task.Run(async () =>
|
|
{
|
|
TestUtils.AssertResults(await c.RestoreAsync(["*"]));
|
|
});
|
|
|
|
var t = await System.Threading.Tasks.Task.WhenAny(timeout_task, restore_task);
|
|
|
|
if (t == timeout_task)
|
|
{
|
|
await c.AbortAsync().ConfigureAwait(false);
|
|
await restore_task; // Ensure we wait for the restore task to complete
|
|
Assert.Fail("Restore timed out");
|
|
}
|
|
else if (t == restore_task)
|
|
// Throw any exceptions it might have
|
|
await t.ConfigureAwait(false);
|
|
|
|
TestUtils.AssertDirectoryTreesAreEquivalent(this.DATAFOLDER, this.RESTOREFOLDER, true, "Restoring with different volume cache sizes");
|
|
}
|
|
|
|
[Test]
|
|
[Category("RestoreHandler")]
|
|
public async Task RestoreInternalProfilingLogsCacheUsageAsync()
|
|
{
|
|
var sourceFilePath = Path.Combine(this.DATAFOLDER, "profiled-restore.bin");
|
|
File.WriteAllBytes(sourceFilePath, new byte[256 * 1024]);
|
|
|
|
var backupOptions = new Dictionary<string, string>(this.TestOptions)
|
|
{
|
|
["dblock-size"] = "100kb",
|
|
["blocksize"] = "10kb"
|
|
};
|
|
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, backupOptions, null))
|
|
TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER }));
|
|
|
|
var restoreOptions = new Dictionary<string, string>(backupOptions)
|
|
{
|
|
["restore-path"] = this.RESTOREFOLDER,
|
|
["restore-legacy"] = "false",
|
|
["restore-with-local-blocks"] = "false",
|
|
["restore-volume-cache-hint"] = "1mb",
|
|
["internal-profiling"] = "true"
|
|
};
|
|
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
|
|
TestUtils.AssertResults(await c.RestoreAsync(new[] { "*" }));
|
|
|
|
var logContents = File.ReadAllText(this.LOGFILE);
|
|
Assert.That(logContents, Does.Contain("Max used cache size:"));
|
|
}
|
|
|
|
[Test]
|
|
[Category("RestoreHandler")]
|
|
public async Task RestoreWithoutLocalDataAsync([Values("true", "false")] string noLocalDb, [Values("true", "false")] string patchWithLocalBlocks)
|
|
{
|
|
var file1Path = Path.Combine(this.DATAFOLDER, "file1");
|
|
File.WriteAllBytes(file1Path, new byte[] { 1, 2, 3 });
|
|
|
|
var file2Path = Path.Combine(this.DATAFOLDER, "file2");
|
|
File.WriteAllBytes(file2Path, new byte[] { 3, 4, 6 });
|
|
|
|
var folderPath = Path.Combine(this.DATAFOLDER, "folder");
|
|
Directory.CreateDirectory(folderPath);
|
|
systemIO.FileCopy(file1Path, Path.Combine(folderPath, "file1 copy"), true);
|
|
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, new Dictionary<string, string>(this.TestOptions), null))
|
|
TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER }));
|
|
|
|
var restoreOptions = new Dictionary<string, string>(this.TestOptions)
|
|
{
|
|
["restore-path"] = this.RESTOREFOLDER,
|
|
["no-local-db"] = noLocalDb,
|
|
["restore-with-local-blocks"] = patchWithLocalBlocks
|
|
};
|
|
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
|
|
TestUtils.AssertResults(await c.RestoreAsync(new[] { "*" }));
|
|
|
|
TestUtils.AssertDirectoryTreesAreEquivalent(this.DATAFOLDER, this.RESTOREFOLDER, true, "Restoring without local data");
|
|
}
|
|
|
|
[Test]
|
|
[Category("RestoreHandler")]
|
|
public async Task RestoreThreeVersionsWithoutLocalDbAsync([Values(0, 1, 2)] int version)
|
|
{
|
|
// Create three versions with different file contents
|
|
var file1Path = Path.Combine(this.DATAFOLDER, "file1.txt");
|
|
var file2Path = Path.Combine(this.DATAFOLDER, "file2.txt");
|
|
|
|
// Version 0 (oldest): Create initial files
|
|
File.WriteAllText(file1Path, "version 0 content");
|
|
File.WriteAllText(file2Path, "version 0 file2");
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, new Dictionary<string, string>(this.TestOptions), null))
|
|
TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER }));
|
|
|
|
// Version 1: Modify file1, keep file2
|
|
File.WriteAllText(file1Path, "version 1 content");
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, new Dictionary<string, string>(this.TestOptions), null))
|
|
TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER }));
|
|
|
|
// Version 2 (newest): Modify both files
|
|
File.WriteAllText(file1Path, "version 2 content");
|
|
File.WriteAllText(file2Path, "version 2 file2");
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, new Dictionary<string, string>(this.TestOptions), null))
|
|
TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER }));
|
|
|
|
// Prepare expected content based on version being restored
|
|
// Note: In Duplicati, version 0 is the MOST RECENT backup
|
|
string expectedFile1Content;
|
|
string expectedFile2Content;
|
|
switch (version)
|
|
{
|
|
case 0: // Most recent (version 2 in our creation order)
|
|
expectedFile1Content = "version 2 content";
|
|
expectedFile2Content = "version 2 file2";
|
|
break;
|
|
case 1: // Second most recent (version 1 in our creation order)
|
|
expectedFile1Content = "version 1 content";
|
|
expectedFile2Content = "version 0 file2";
|
|
break;
|
|
case 2: // Oldest (version 0 in our creation order)
|
|
expectedFile1Content = "version 0 content";
|
|
expectedFile2Content = "version 0 file2";
|
|
break;
|
|
default:
|
|
throw new ArgumentException("Invalid version");
|
|
}
|
|
|
|
// Restore with --version and --no-local-db
|
|
var restoreOptions = new Dictionary<string, string>(this.TestOptions)
|
|
{
|
|
["restore-path"] = this.RESTOREFOLDER,
|
|
["version"] = version.ToString(),
|
|
["no-local-db"] = "true"
|
|
};
|
|
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
|
|
TestUtils.AssertResults(await c.RestoreAsync(new[] { "*" }));
|
|
|
|
// Verify restored files match the expected version
|
|
var restoredFile1Path = Path.Combine(this.RESTOREFOLDER, "file1.txt");
|
|
var restoredFile2Path = Path.Combine(this.RESTOREFOLDER, "file2.txt");
|
|
|
|
Assert.IsTrue(File.Exists(restoredFile1Path), "Restored file1 should exist");
|
|
Assert.IsTrue(File.Exists(restoredFile2Path), "Restored file2 should exist");
|
|
Assert.AreEqual(expectedFile1Content, File.ReadAllText(restoredFile1Path), $"File1 content mismatch for version {version}");
|
|
Assert.AreEqual(expectedFile2Content, File.ReadAllText(restoredFile2Path), $"File2 content mismatch for version {version}");
|
|
}
|
|
|
|
[Test]
|
|
[Category("RestoreHandler")]
|
|
public async Task RestoreOtherProcessIsUsingFileAsync()
|
|
{
|
|
var file1Path = Path.Combine(this.DATAFOLDER, "file1");
|
|
byte[] original_contents = [1, 2, 3];
|
|
File.WriteAllBytes(file1Path, original_contents);
|
|
|
|
var opts = new Dictionary<string, string>(this.TestOptions);
|
|
opts["overwrite"] = "true";
|
|
|
|
using var c = new Controller("file://" + this.TARGETFOLDER, opts, null);
|
|
|
|
var res_backup = await c.BackupAsync([this.DATAFOLDER]);
|
|
TestUtils.AssertResults(res_backup);
|
|
|
|
File.WriteAllBytes(file1Path, [4, 5, 6]);
|
|
|
|
using (var fs = new FileStream(file1Path, FileMode.Open, FileAccess.Read, FileShare.None))
|
|
{
|
|
var res_failing = await c.RestoreAsync(["*"]);
|
|
Assert.AreEqual(4, res_failing.Errors.Count());
|
|
var first_error = res_failing.Errors.First();
|
|
Assert.IsTrue(
|
|
first_error.Contains("IOException: The process cannot access the file")
|
|
&&
|
|
first_error.EndsWith(" because it is being used by another process.")
|
|
);
|
|
}
|
|
|
|
var res_restore = await c.RestoreAsync(["*"]);
|
|
TestUtils.AssertResults(res_restore);
|
|
Assert.AreEqual(original_contents, File.ReadAllBytes(file1Path));
|
|
}
|
|
|
|
[Test]
|
|
[Category("RestoreHandler")]
|
|
public async Task RestoreVolumeCacheDiskPressureAsync()
|
|
{
|
|
var opts = TestOptions;
|
|
opts["dblock-size"] = "1mb";
|
|
opts["blocksize"] = "1kb";
|
|
// No restore-volume-cache-hint → unlimited mode (-1 sentinel).
|
|
// Set restore-volume-cache-min-free to an absurdly large value so
|
|
// every volume arrival triggers disk-pressure eviction.
|
|
opts["restore-volume-cache-min-free"] = "999tb";
|
|
opts["restore-legacy"] = "false";
|
|
opts["restore-path"] = RESTOREFOLDER;
|
|
|
|
// Write enough data to create at least 10 dblock volumes (dblock-size=1mb),
|
|
// so that the eviction loop is entered multiple times and the CachePressure
|
|
// warning threshold (5 evictions) is reliably crossed.
|
|
Random rng = new();
|
|
for (int i = 0; i < 20; i++)
|
|
{
|
|
var data = new byte[512 * 1024]; // 512 KB each → ~10 MB total → ~10 dblock volumes
|
|
rng.NextBytes(data);
|
|
File.WriteAllBytes(Path.Combine(this.DATAFOLDER, $"file{i}"), data);
|
|
}
|
|
|
|
using var c = new Controller("file://" + this.TARGETFOLDER, opts, null);
|
|
TestUtils.AssertResults(await c.BackupAsync([this.DATAFOLDER]));
|
|
|
|
var timeout_task = System.Threading.Tasks.Task.Delay(TimeSpan.FromSeconds(120));
|
|
RestoreResults? result = null;
|
|
|
|
var restore_task = System.Threading.Tasks.Task.Run(async () =>
|
|
{
|
|
result = (RestoreResults)await c.RestoreAsync(["*"]);
|
|
});
|
|
|
|
var t = await System.Threading.Tasks.Task.WhenAny(timeout_task, restore_task);
|
|
if (t == timeout_task)
|
|
{
|
|
await c.AbortAsync().ConfigureAwait(false);
|
|
await restore_task;
|
|
Assert.Fail("Restore timed out");
|
|
}
|
|
else
|
|
{
|
|
await t.ConfigureAwait(false);
|
|
}
|
|
|
|
Assert.IsNotNull(result);
|
|
Assert.That(result!.CachePressureEvictions, Is.GreaterThan(0), "Expected disk-pressure evictions with 999tb min-free");
|
|
Assert.That(result.TotalVolumesAccessed, Is.GreaterThan(0), "Expected at least one volume to be accessed");
|
|
Assert.That(result.Warnings.Count(), Is.GreaterThanOrEqualTo(1), "Expected at least one CachePressure warning");
|
|
|
|
TestUtils.AssertDirectoryTreesAreEquivalent(this.DATAFOLDER, this.RESTOREFOLDER, true, "Restoring with disk pressure eviction");
|
|
}
|
|
}
|
|
} |