From df76a77ee23ce7dbd91e07fc0e2a27d4fda0afda Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 8 Jul 2024 07:32:35 +0200 Subject: [PATCH 001/362] Added tool to generate test data --- Tools/TestDataGenerator/.gitignore | 1 + Tools/TestDataGenerator/.vscode/launch.json | 27 +++ Tools/TestDataGenerator/.vscode/tasks.json | 17 ++ Tools/TestDataGenerator/Commands/Create.cs | 227 ++++++++++++++++++ Tools/TestDataGenerator/Commands/Shared.cs | 132 ++++++++++ Tools/TestDataGenerator/Commands/Update.cs | 195 +++++++++++++++ Tools/TestDataGenerator/Program.cs | 10 + Tools/TestDataGenerator/README.md | 7 + .../TestDataGenerator.csproj | 15 ++ Tools/TestDataGenerator/TestDataGenerator.sln | 25 ++ 10 files changed, 656 insertions(+) create mode 100644 Tools/TestDataGenerator/.gitignore create mode 100644 Tools/TestDataGenerator/.vscode/launch.json create mode 100644 Tools/TestDataGenerator/.vscode/tasks.json create mode 100644 Tools/TestDataGenerator/Commands/Create.cs create mode 100644 Tools/TestDataGenerator/Commands/Shared.cs create mode 100644 Tools/TestDataGenerator/Commands/Update.cs create mode 100644 Tools/TestDataGenerator/Program.cs create mode 100644 Tools/TestDataGenerator/README.md create mode 100644 Tools/TestDataGenerator/TestDataGenerator.csproj create mode 100644 Tools/TestDataGenerator/TestDataGenerator.sln diff --git a/Tools/TestDataGenerator/.gitignore b/Tools/TestDataGenerator/.gitignore new file mode 100644 index 000000000..1269488f7 --- /dev/null +++ b/Tools/TestDataGenerator/.gitignore @@ -0,0 +1 @@ +data diff --git a/Tools/TestDataGenerator/.vscode/launch.json b/Tools/TestDataGenerator/.vscode/launch.json new file mode 100644 index 000000000..b265cd745 --- /dev/null +++ b/Tools/TestDataGenerator/.vscode/launch.json @@ -0,0 +1,27 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Generate Test Data", + "type": "coreclr", + "request": "launch", + "preLaunchTask": "build", + "program": "${workspaceFolder}/bin/Debug/net8.0/TestDataGenerator.dll", + "args": ["create", "./data", "--file-count=1000", "--max-folder-count=100", "--max-fan-out=3"], + "cwd": "${workspaceFolder}", + "console": "internalConsole", + "stopAtEntry": false + }, + { + "name": "Update Test Data", + "type": "coreclr", + "request": "launch", + "preLaunchTask": "build", + "program": "${workspaceFolder}/bin/Debug/net8.0/TestDataGenerator.dll", + "args": ["update", "./data", "--new-files=10", "--updated-files=10", "--deleted-files=10"], + "cwd": "${workspaceFolder}", + "console": "internalConsole", + "stopAtEntry": false + } + ] +} \ No newline at end of file diff --git a/Tools/TestDataGenerator/.vscode/tasks.json b/Tools/TestDataGenerator/.vscode/tasks.json new file mode 100644 index 000000000..f2835a246 --- /dev/null +++ b/Tools/TestDataGenerator/.vscode/tasks.json @@ -0,0 +1,17 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "build", + "command": "dotnet", + "type": "process", + "args": [ + "build", + "${workspaceFolder}/TestDataGenerator.sln", + "/property:GenerateFullPaths=true", + "/consoleloggerparameters:NoSummary;ForceNoAlign" + ], + "problemMatcher": "$msCompile" + } + ] +} \ No newline at end of file diff --git a/Tools/TestDataGenerator/Commands/Create.cs b/Tools/TestDataGenerator/Commands/Create.cs new file mode 100644 index 000000000..4eed98130 --- /dev/null +++ b/Tools/TestDataGenerator/Commands/Create.cs @@ -0,0 +1,227 @@ +using System.CommandLine; +using System.CommandLine.NamingConventionBinder; +using static TestDataGenerator.Commands.Shared; + +namespace TestDataGenerator.Commands; + +/// +/// Create test data in a folder +/// +public static class Create +{ + /// + /// The input parameters for the command + /// + /// The folder to create files in + /// The number of files to create in the target folder + /// The maximum total size of the files to create in the target folder + /// The maximum size of each file to create in the target folder + /// The minimum size of each file to create in the target folder + /// The percentage of data that should be zeroed out in each file + /// The maximum number of subfolders to create in each folder + /// The maximum depth of subfolders to create in the target folder + /// The maximum number of folders to create in the target folder + /// The maximum length of each path segment in the target folder + record CommandInput( + DirectoryInfo TargetFolder, + int FileCount, + long MaxTotalSize, + long MaxFileSize, + long MinFileSize, + int SparseFactor, + int MaxFanOut, + int MaxDepth, + int MaxFolderCount, + int MaxPathSegmentLength); + + /// + /// Creates the command + /// + /// The command + public static Command CreateCommand() + { + var command = new Command("create", "Create test data in a folder"); + + var targetFolderOption = new Argument("target-folder", "The folder to create files in"); + targetFolderOption.SetDefaultValue(new DirectoryInfo(Directory.GetCurrentDirectory())); + command.AddArgument(targetFolderOption); + + var fileCountOption = new Option("--file-count", "The number of files to create in the target folder"); + fileCountOption.SetDefaultValue(200000); + command.AddOption(fileCountOption); + + var maxTotalSizeOption = new Option("--max-total-size", "The maximum total size of the files to create in the target folder"); + maxTotalSizeOption.SetDefaultValue(1024 * 1024 * 1024L); + command.AddOption(maxTotalSizeOption); + + var maxFileSizeOption = new Option("--max-file-size", "The maximum size of each file to create in the target folder"); + maxFileSizeOption.SetDefaultValue(1024 * 1024L); + command.AddOption(maxFileSizeOption); + + var minFileSizeOption = new Option("--min-file-size", "The minimum size of each file to create in the target folder"); + minFileSizeOption.SetDefaultValue(0L); + command.AddOption(minFileSizeOption); + + var sparseFactorOption = new Option("--sparse-factor", "The percentage of data that should be zeroed out in each file"); + sparseFactorOption.SetDefaultValue(10); + command.AddOption(sparseFactorOption); + + var maxFanOutOption = new Option("--max-fan-out", "The maximum number of subfolders to create in each folder"); + maxFanOutOption.SetDefaultValue(10); + command.AddOption(maxFanOutOption); + + var maxDepthOption = new Option("--max-depth", "The maximum depth of subfolders to create in the target folder"); + maxDepthOption.SetDefaultValue(5); + command.AddOption(maxDepthOption); + + var maxFolderCountOption = new Option("--max-folder-count", "The maximum number of folders to create in the target folder"); + maxFolderCountOption.SetDefaultValue(10000); + command.AddOption(maxFolderCountOption); + + var maxPathSegmentLengthOption = new Option("--max-path-segment-length", "The maximum length of each path segment in the target folder"); + maxPathSegmentLengthOption.SetDefaultValue(15); + command.AddOption(maxPathSegmentLengthOption); + + command.Handler = CommandHandler.Create(Execute); + + return command; + } + + /// + /// Executes the command + /// + /// The input parameters + private static void Execute(CommandInput input) + { + if (input.TargetFolder.Exists && input.TargetFolder.GetFileSystemInfos().Length > 0) + throw new Exception($"The target folder {input.TargetFolder.FullName} already exists"); + + if (input.MaxFileSize == 0) + throw new Exception("The maximum file size must be greater than zero"); + + if (input.MaxTotalSize == 0) + throw new Exception("The maximum total size must be greater than zero"); + + if (input.MinFileSize > input.MaxFileSize) + throw new Exception("The minimum file size must be less than or equal to the maximum file size"); + + if (input.FileCount == 0) + throw new Exception("The file count must be greater than zero"); + + if (input.MaxFolderCount == 0) + throw new Exception("The maximum folder count must be greater than zero"); + + if (input.MaxFanOut == 0) + throw new Exception("The maximum fan-out must be greater than zero"); + + if (input.MaxDepth == 0) + throw new Exception("The maximum depth must be greater than zero"); + + if (input.MaxPathSegmentLength == 0) + throw new Exception("The maximum path segment length must be greater than zero"); + + Console.WriteLine($"Creating test data in {input.TargetFolder.FullName}"); + var rnd = new Random(); + var folders = GeneratePathStructure(rnd, input.TargetFolder.FullName, input.MaxDepth, input.MaxFanOut, input.MaxPathSegmentLength); + while (folders.Count < input.MaxFolderCount) + { + folders.AddRange(GeneratePathStructure(rnd, input.TargetFolder.FullName, input.MaxDepth, input.MaxFanOut, input.MaxPathSegmentLength)); + folders = folders.Distinct().ToList(); + } + + while (folders.Count > input.MaxFolderCount) + folders.RemoveAt(folders.Count - 1); + + folders.Add(input.TargetFolder.FullName); + + var files = DistributeFiles(rnd, folders, GenerateFileNames(rnd, input.FileCount, input.MaxPathSegmentLength)); + + var filesWithSizes = files.Select(x => (x, rnd.NextInt64(input.MinFileSize, input.MaxFileSize))).ToList(); + var totalSize = filesWithSizes.Sum(x => x.Item2); + + while (totalSize > input.MaxTotalSize) + { + var index = rnd.Next(filesWithSizes.Count); + var newSize = rnd.NextInt64(input.MinFileSize, filesWithSizes[index].Item2); + totalSize -= filesWithSizes[index].Item2 - newSize; + filesWithSizes[index] = (filesWithSizes[index].Item1, newSize); + } + + Console.WriteLine($"Creating {folders.Count - 1} folders and {files.Count} files ({SizeToHumanReadable(totalSize)})"); + if (!input.TargetFolder.Exists) + input.TargetFolder.Create(); + + var lastUpdate = DateTime.UtcNow; + var foldersCreated = 0L; + + var updateInterval = TimeSpan.FromSeconds(10); + + foreach (var folder in folders) + { + Directory.CreateDirectory(folder); + foldersCreated++; + if ((DateTime.UtcNow - lastUpdate) > updateInterval) + { + Console.WriteLine($"Created {foldersCreated} of {folders.Count} folders"); + lastUpdate = DateTime.UtcNow; + } + } + + var filesCreated = 0L; + var fileSizeCreated = 0L; + + foreach (var (file, size) in filesWithSizes) + { + try + { + using var fs = new FileStream(file, FileMode.Create, FileAccess.Write); + if (size > 0) + { + fs.SetLength(size); + fs.Position = 0; + + WriteRandomData(rnd, fs, size, input.SparseFactor); + } + + filesCreated++; + fileSizeCreated += size; + if ((DateTime.UtcNow - lastUpdate) > updateInterval) + { + Console.WriteLine($"Created {filesCreated} of {files.Count} files ({SizeToHumanReadable(fileSizeCreated)} of {SizeToHumanReadable(totalSize)})"); + lastUpdate = DateTime.UtcNow; + } + } + catch (Exception ex) + { + Console.WriteLine($"Error creating file {file}: {ex.Message}"); + } + } + + } + + /// + /// Generates a list of random folder paths + /// + /// The random number generator to use + /// The prefix to use for the folder paths + /// The maximum depth of the folder structure + /// The maximum number of subfolders to create in each folder + /// The maximum length of each path segment + private static List GeneratePathStructure(Random rnd, string prefix, int maxDepth, int maxFanOut, int maxSegmentLength) + { + var folders = new List(); + var folderCount = rnd.Next(1, maxFanOut + 1); + + if (maxDepth == 0) + return GeneratePathSegments(rnd, folderCount, maxSegmentLength).Select(x => Path.Combine(prefix, x)).ToList(); + + for (var i = 0; i < folderCount; i++) + { + var folderName = GetPathSegment(rnd, maxSegmentLength); + folders.Add(Path.Combine(prefix, folderName)); + folders.AddRange(GeneratePathStructure(rnd, Path.Combine(prefix, folderName), maxDepth - 1, maxFanOut, maxSegmentLength)); + } + + return folders; + } +} diff --git a/Tools/TestDataGenerator/Commands/Shared.cs b/Tools/TestDataGenerator/Commands/Shared.cs new file mode 100644 index 000000000..ac9afad60 --- /dev/null +++ b/Tools/TestDataGenerator/Commands/Shared.cs @@ -0,0 +1,132 @@ +using System.Text; + +namespace TestDataGenerator.Commands; + +/// +/// Shared utility methods for the test data generator +/// +public static class Shared +{ + /// + /// Distributes the files in the list to the folders in the list + /// + /// The random number generator to use + /// The list of folders to distribute the files to + /// The list of files to distribute + /// The list of files distributed to the folders + public static List DistributeFiles(Random rnd, List folders, List list) + { + var expanded = new List(); + foreach (var file in list) + expanded.Add(Path.Combine(folders[rnd.Next(folders.Count)], file)); + + return expanded; + } + + /// + /// Converts a size in bytes to a human-readable string + /// + /// The size in bytes + /// The human-readable string + public static string SizeToHumanReadable(long size) + { + if (size < 1024) + return $"{size} B"; + if (size < 1024 * 1000) + return $"{size / 1024m:F2} KiB"; + if (size < 1024 * 1024 * 1000) + return $"{size / 1024 / 1024m:F2} MiB"; + if (size < 1024L * 1024 * 1024 * 1000) + return $"{size / 1024 / 1024 / 1024m:F2} GiB"; + return $"{size / 1024 / 1024 / 1024 / 1024m:F2} TiB"; + } + + /// + /// The list of file extensions to use for random file names + /// + private static readonly IReadOnlyList FileExtensions = new List([".bin", ".dat", ".binary", ".data", ".test", ""]); + + /// + /// The characters to use for random path segments + /// + private const string PathSegmentChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + + /// + /// Generates a random path segment + /// + /// The random number generator to use + /// The maximum length of the path segment + /// The random path segment + public static string GetPathSegment(Random rnd, int maxLength) + { + var length = rnd.Next(1, maxLength + 1); + var sb = new StringBuilder(length); + for (int j = 0; j < length; j++) + sb.Append(PathSegmentChars[rnd.Next(PathSegmentChars.Length)]); + return sb.ToString(); + } + + /// + /// Generates a list of random path segments + /// + /// The random number generator to use + /// The number of path segments to generate + /// The maximum length of each path segment + /// The list of path segments + public static List GeneratePathSegments(Random rnd, int count, int maxLength) + { + var names = new List(count); + for (int i = 0; i < count; i++) + names.Add(GetPathSegment(rnd, maxLength)); + return names; + } + + /// + /// Generates a list of random file names + /// + /// The random number generator to use + /// The number of file names to generate + /// The maximum length of each file name + /// The list of file names + public static List GenerateFileNames(Random rnd, int count, int maxLength) + { + var names = new List(count); + for (int i = 0; i < count; i++) + names.Add(GetPathSegment(rnd, maxLength) + FileExtensions[rnd.Next(FileExtensions.Count)]); + return names; + } + + /// + /// Writes random data to a stream + /// + /// The random number generator to use + /// The stream to write to + /// The size of the data to write + /// The factor to use for sparse data + /// + /// The sparse factor is a percentage value that determines how many bytes are skipped in the stream. + /// For example, a sparse factor of 10 means that 10% of the data will be not be overwritten. + /// + public static void WriteRandomData(Random rnd, Stream fs, long size, int sparseFactor) + { + var buffer = new byte[4096]; + while (size > 0) + { + var chunkSize = rnd.Next(1, (int)Math.Min(size, buffer.Length)); + var sp = buffer.AsSpan(chunkSize); + if (rnd.Next(100) < sparseFactor) + { + // Skip the bytes + fs.Position += sp.Length; + } + else + { + rnd.NextBytes(sp); + fs.Write(sp); + } + + size -= sp.Length; + } + } + +} \ No newline at end of file diff --git a/Tools/TestDataGenerator/Commands/Update.cs b/Tools/TestDataGenerator/Commands/Update.cs new file mode 100644 index 000000000..5066ac50e --- /dev/null +++ b/Tools/TestDataGenerator/Commands/Update.cs @@ -0,0 +1,195 @@ +using System.CommandLine; +using System.CommandLine.NamingConventionBinder; +using static TestDataGenerator.Commands.Shared; + +namespace TestDataGenerator.Commands; + +/// +/// Update test data in a folder +/// +public static class Update +{ + /// + /// The input parameters for the command + /// + /// The folder to update files in + /// The number of new files to create in the target folder + /// The number of files to update in the target folder + /// The number of files to delete in the target folder + /// The number of files to rename in the target folder + /// The maximum size of each file to create in the target folder + /// The minimum size of each file to create in the target folder + /// The percentage of data that should be zeroed out in each file + /// The percentage of files to update in the target folder + /// The maximum length of each path segment in the target folder + record CommandInput( + DirectoryInfo TargetFolder, + int NewFiles, + int UpdatedFiles, + int DeletedFiles, + int RenameFiles, + long MaxFileSize, + long MinFileSize, + int SparseFactor, + int UpdateFactor, + int MaxPathSegmentLength); + + /// + /// Creates the command + /// + /// The command + public static Command CreateCommand() + { + var command = new Command("update", "Update test data in a folder"); + + var targetFolderOption = new Argument("target-folder", "The folder to update files in"); + targetFolderOption.SetDefaultValue(new DirectoryInfo(Directory.GetCurrentDirectory())); + command.AddArgument(targetFolderOption); + + var newFilesOption = new Option("--new-files", "The number of new files to create in the target folder"); + newFilesOption.SetDefaultValue(1000); + command.AddOption(newFilesOption); + + var updatedFilesOption = new Option("--updated-files", "The number of files to update in the target folder"); + updatedFilesOption.SetDefaultValue(1000); + command.AddOption(updatedFilesOption); + + var deletedFilesOption = new Option("--deleted-files", "The number of files to delete in the target folder"); + deletedFilesOption.SetDefaultValue(1000); + command.AddOption(deletedFilesOption); + + var renameFilesOption = new Option("--rename-files", "The number of files to rename in the target folder"); + renameFilesOption.SetDefaultValue(1000); + command.AddOption(renameFilesOption); + + var maxFileSizeOption = new Option("--max-file-size", "The maximum size of each file to create in the target folder"); + maxFileSizeOption.SetDefaultValue(1024 * 1024L); + command.AddOption(maxFileSizeOption); + + var minFileSizeOption = new Option("--min-file-size", "The minimum size of each file to create in the target folder"); + minFileSizeOption.SetDefaultValue(1024L); + command.AddOption(minFileSizeOption); + + var sparseFactorOption = new Option("--sparse-factor", "The percentage of data that should be zeroed out in each file"); + sparseFactorOption.SetDefaultValue(10); + command.AddOption(sparseFactorOption); + + var updateFactorOption = new Option("--update-factor", "The percentage of files to update in the target folder"); + updateFactorOption.SetDefaultValue(10); + command.AddOption(updateFactorOption); + + var maxPathSegmentLengthOption = new Option("--max-path-segment-length", "The maximum length of each path segment in the target folder"); + maxPathSegmentLengthOption.SetDefaultValue(15); + command.AddOption(maxPathSegmentLengthOption); + + command.Handler = CommandHandler.Create(Execute); + + return command; + } + + /// + /// Executes the command + /// + /// The input parameters + private static void Execute(CommandInput input) + { + if (!input.TargetFolder.Exists) + throw new DirectoryNotFoundException($"The target folder {input.TargetFolder.FullName} does not exist"); + + if (input.MaxFileSize == 0) + throw new Exception("The maximum file size must be greater than zero"); + + Console.WriteLine($"Updating test data in {input.TargetFolder.FullName}"); + + var folders = new List(); + var files = new List(); + + foreach (var folder in input.TargetFolder.EnumerateDirectories("*", SearchOption.AllDirectories)) + { + folders.Add(folder.FullName); + foreach (var file in folder.EnumerateFiles()) + files.Add(file.FullName); + } + + Console.WriteLine($"Found {folders.Count} folders and {files.Count} files"); + + folders.Add(input.TargetFolder.FullName); + + var rnd = new Random(); + var newFileNames = DistributeFiles(rnd, folders, GenerateFileNames(rnd, input.NewFiles, input.MaxPathSegmentLength)); + + foreach (var file in newFileNames) + { + Console.WriteLine($"Creating new file {file}"); + + var size = rnd.NextInt64((int)input.MinFileSize, (int)input.MaxFileSize); + using var fs = new FileStream(file, FileMode.Create, FileAccess.Write); + fs.SetLength(size); + if (size > 0) + { + fs.Position = 0; + WriteRandomData(rnd, fs, size, input.SparseFactor); + } + } + + var updatedFileNames = new HashSet(); + while (updatedFileNames.Count < input.UpdatedFiles) + { + var file = files[rnd.Next(files.Count)]; + if (!updatedFileNames.Contains(file)) + updatedFileNames.Add(file); + + if (updatedFileNames.Count == files.Count) + break; + } + + foreach (var file in updatedFileNames) + { + Console.WriteLine($"Updating file {file}"); + + var size = rnd.NextInt64(input.MinFileSize, input.MaxFileSize); + using var fs = new FileStream(file, FileMode.Open, FileAccess.Write); + fs.SetLength(size); + if (size > 0) + { + fs.Position = 0; + WriteRandomData(rnd, fs, size, 100 - input.UpdateFactor); + } + } + + var deletedFileNames = new HashSet(); + while (deletedFileNames.Count < input.DeletedFiles) + { + var file = files[rnd.Next(files.Count)]; + if (!deletedFileNames.Contains(file)) + deletedFileNames.Add(file); + + if (deletedFileNames.Count == files.Count) + break; + } + + foreach (var file in deletedFileNames) + { + Console.WriteLine($"Deleting file {file}"); + File.Delete(file); + } + + var renamedFileNames = new HashSet(); + while (renamedFileNames.Count < input.RenameFiles) + { + var file = files[rnd.Next(files.Count)]; + if (!renamedFileNames.Contains(file)) + renamedFileNames.Add(file); + + if (renamedFileNames.Count == files.Count) + break; + } + + foreach (var file in renamedFileNames) + { + var newName = Path.Combine(folders[rnd.Next(folders.Count)], GenerateFileNames(rnd, 1, input.MaxPathSegmentLength).First()); + Console.WriteLine($"Renaming file {file} to {newName}"); + File.Move(file, newName); + } + } +} diff --git a/Tools/TestDataGenerator/Program.cs b/Tools/TestDataGenerator/Program.cs new file mode 100644 index 000000000..6f43b554d --- /dev/null +++ b/Tools/TestDataGenerator/Program.cs @@ -0,0 +1,10 @@ +using System.CommandLine; + +var rootCommand = new RootCommand("Test Data Generator") +{ + TestDataGenerator.Commands.Create.CreateCommand(), + TestDataGenerator.Commands.Update.CreateCommand() +}; + +return rootCommand.Invoke(args); + diff --git a/Tools/TestDataGenerator/README.md b/Tools/TestDataGenerator/README.md new file mode 100644 index 000000000..f01ba7e88 --- /dev/null +++ b/Tools/TestDataGenerator/README.md @@ -0,0 +1,7 @@ +# Test data generator + +This tool is intended for use by developer who need to generate a data set for testing Duplicati. + +The tool can create a set of random data at a target folder using the `create` command, and make alterations using the `update` command. + +Use the `help` command to examine the supported properties and adjust to make the desired test dataset. diff --git a/Tools/TestDataGenerator/TestDataGenerator.csproj b/Tools/TestDataGenerator/TestDataGenerator.csproj new file mode 100644 index 000000000..04c2e9be8 --- /dev/null +++ b/Tools/TestDataGenerator/TestDataGenerator.csproj @@ -0,0 +1,15 @@ + + + + Exe + net8.0 + enable + enable + + + + + + + + diff --git a/Tools/TestDataGenerator/TestDataGenerator.sln b/Tools/TestDataGenerator/TestDataGenerator.sln new file mode 100644 index 000000000..6cbfc34b3 --- /dev/null +++ b/Tools/TestDataGenerator/TestDataGenerator.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.5.002.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestDataGenerator", "TestDataGenerator.csproj", "{547C0549-040B-4F4F-A410-351FC0176F28}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {547C0549-040B-4F4F-A410-351FC0176F28}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {547C0549-040B-4F4F-A410-351FC0176F28}.Debug|Any CPU.Build.0 = Debug|Any CPU + {547C0549-040B-4F4F-A410-351FC0176F28}.Release|Any CPU.ActiveCfg = Release|Any CPU + {547C0549-040B-4F4F-A410-351FC0176F28}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {F3FB6242-FFDF-42A7-A78F-BB1FE67C12D8} + EndGlobalSection +EndGlobal From deab874499486185b457de2ec0fe609b2861a879 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 13 Jan 2025 22:23:00 +0100 Subject: [PATCH 002/362] Better `wix.exe` support --- ReleaseBuilder/Build/Command.CreatePackage.cs | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/ReleaseBuilder/Build/Command.CreatePackage.cs b/ReleaseBuilder/Build/Command.CreatePackage.cs index e7ef0f9ba..0f3bc59a3 100644 --- a/ReleaseBuilder/Build/Command.CreatePackage.cs +++ b/ReleaseBuilder/Build/Command.CreatePackage.cs @@ -298,22 +298,31 @@ public static partial class Command _ => throw new Exception($"Architeture not supported: {target.ArchString}") }; - List wixArgs = [ - Program.Configuration.Commands.Wix!, - "--define", $"HarvestPath={sourceFiles}", - "--arch", msiArch, - "--output", msiFile, - Path.Combine(resourcesSubDir, "Shortcuts.wxs"), - binFiles, - Path.Combine(resourcesSubDir, "Duplicati.wxs") - ]; + var argPrefix = OperatingSystem.IsWindows() ? "-" : "--"; - // Add wixl specific extensions - if (!OperatingSystem.IsWindows()) - wixArgs.InsertRange(1, [ + string[] wixArgs = + OperatingSystem.IsWindows() ? + [ + Program.Configuration.Commands.Wix!, + "-define", $"HarvestPath={sourceFiles}", + "-arch", msiArch, + "-out", msiFile, + Path.Combine(resourcesSubDir, "Shortcuts.wxs"), + binFiles, + Path.Combine(resourcesSubDir, "Duplicati.wxs") + ] + : + [ + Program.Configuration.Commands.Wix!, "--ext", "ui", "--extdir", Path.Combine(resourcesDir, "WixUIExtension"), - ]); + "--define", $"HarvestPath={sourceFiles}", + "--arch", msiArch, + "--output", msiFile, + Path.Combine(resourcesSubDir, "Shortcuts.wxs"), + binFiles, + Path.Combine(resourcesSubDir, "Duplicati.wxs") + ]; await ProcessHelper.Execute(wixArgs, workingDirectory: buildRoot); From 76bced0fb7bce75afc015044794eb8343b026660 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 14 Jan 2025 20:00:15 +0100 Subject: [PATCH 003/362] More `wix.exe` fixing --- ReleaseBuilder/Build/Command.CreatePackage.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/ReleaseBuilder/Build/Command.CreatePackage.cs b/ReleaseBuilder/Build/Command.CreatePackage.cs index 0f3bc59a3..6138af521 100644 --- a/ReleaseBuilder/Build/Command.CreatePackage.cs +++ b/ReleaseBuilder/Build/Command.CreatePackage.cs @@ -304,6 +304,7 @@ public static partial class Command OperatingSystem.IsWindows() ? [ Program.Configuration.Commands.Wix!, + "build", "-define", $"HarvestPath={sourceFiles}", "-arch", msiArch, "-out", msiFile, From 4716738014cdfc41b31c6cf606548bfd35c43b83 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 16 Jan 2025 09:30:57 +0100 Subject: [PATCH 004/362] Fixed an issue with the fileId --- ReleaseBuilder/WixHeatBuilder.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ReleaseBuilder/WixHeatBuilder.cs b/ReleaseBuilder/WixHeatBuilder.cs index ba58137de..acf123566 100644 --- a/ReleaseBuilder/WixHeatBuilder.cs +++ b/ReleaseBuilder/WixHeatBuilder.cs @@ -38,7 +38,7 @@ public static class WixHeatBuilder public static string CreateWixFilelist(string sourceFolder, string version, string folderPrefix = "$(var.HarvestPath)", string directoryRefName = "INSTALLLOCATION", string componentGroupId = "DUPLICATIBIN", Func? fileIdGenerator = null) { var itemIds = new Dictionary(); - fileIdGenerator ??= (x) => Path.GetRelativePath(sourceFolder, x).Replace("\\", "_").Replace("/", "_").Replace(":", "_").Replace(" ", "_"); + fileIdGenerator ??= (x) => Path.GetRelativePath(sourceFolder, x).Replace("\\", "_").Replace("/", "_").Replace(":", "_").Replace(" ", "_").Replace("-", "_"); Func pathTransformer = (x) => $"{folderPrefix}{Path.GetRelativePath(sourceFolder, x)}"; var doc = new XmlDocument(); From 54dde6e221495e1400d8f09be9be33291d4f4dd7 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 16 Jan 2025 09:51:44 +0100 Subject: [PATCH 005/362] Support different namespaces for WiX v4 --- ReleaseBuilder/Build/Command.CreatePackage.cs | 43 ++++++++++++++----- ReleaseBuilder/WixHeatBuilder.cs | 4 +- 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/ReleaseBuilder/Build/Command.CreatePackage.cs b/ReleaseBuilder/Build/Command.CreatePackage.cs index 6138af521..76dbb3a6e 100644 --- a/ReleaseBuilder/Build/Command.CreatePackage.cs +++ b/ReleaseBuilder/Build/Command.CreatePackage.cs @@ -264,6 +264,10 @@ public static partial class Command /// A task representing the asynchronous operation. static async Task BuildMsiPackage(string baseDir, string buildRoot, string msiFile, PackageTarget target, RuntimeConfig rtcfg) { + var isWindows = OperatingSystem.IsWindows(); + const string originalNamespace = "http://schemas.microsoft.com/wix/2006/wi"; + const string wixv4Namespace = "http://wixtoolset.org/schemas/v4/wxs"; + var resourcesDir = Path.Combine(baseDir, "ReleaseBuilder", "Resources", "Windows"); var resourcesSubDir = Path.Combine(resourcesDir, target.Interface switch @@ -288,7 +292,11 @@ public static partial class Command if (File.Exists(binFiles)) File.Delete(binFiles); - File.WriteAllText(binFiles, WixHeatBuilder.CreateWixFilelist(sourceFiles, version: rtcfg.ReleaseInfo.Version.ToString())); + File.WriteAllText(binFiles, WixHeatBuilder.CreateWixFilelist( + sourceFiles, + version: rtcfg.ReleaseInfo.Version.ToString(), + wixNs: isWindows ? wixv4Namespace : originalNamespace + )); var msiArch = target.Arch switch { @@ -298,22 +306,36 @@ public static partial class Command _ => throw new Exception($"Architeture not supported: {target.ArchString}") }; - var argPrefix = OperatingSystem.IsWindows() ? "-" : "--"; + // Prepare the Wix arguments, different for WiX (Windows) and wixl (Linux/MacOS) + string[] wixArgs; - string[] wixArgs = - OperatingSystem.IsWindows() ? - [ + if (isWindows) + { + // Update namespace in the .wxs files for WiX v4 + var shortcutsTempfile = Path.Combine(buildTmp, "Shortcuts.wxs"); + var entryTempfile = Path.Combine(buildTmp, "Duplicati.wxs"); + + File.WriteAllText(shortcutsTempfile, File.ReadAllText(Path.Combine(resourcesSubDir, "Shortcuts.wxs")) + .Replace(originalNamespace, wixv4Namespace)); + + File.WriteAllText(entryTempfile, File.ReadAllText(Path.Combine(resourcesSubDir, "Duplicati.wxs")) + .Replace(originalNamespace, wixv4Namespace)); + + wixArgs = [ Program.Configuration.Commands.Wix!, "build", "-define", $"HarvestPath={sourceFiles}", "-arch", msiArch, "-out", msiFile, - Path.Combine(resourcesSubDir, "Shortcuts.wxs"), + shortcutsTempfile, binFiles, - Path.Combine(resourcesSubDir, "Duplicati.wxs") - ] - : - [ + entryTempfile + ]; + } + else + { + // wixl needs the UI extension + wixArgs = [ Program.Configuration.Commands.Wix!, "--ext", "ui", "--extdir", Path.Combine(resourcesDir, "WixUIExtension"), @@ -324,6 +346,7 @@ public static partial class Command binFiles, Path.Combine(resourcesSubDir, "Duplicati.wxs") ]; + } await ProcessHelper.Execute(wixArgs, workingDirectory: buildRoot); diff --git a/ReleaseBuilder/WixHeatBuilder.cs b/ReleaseBuilder/WixHeatBuilder.cs index acf123566..bab4a38f3 100644 --- a/ReleaseBuilder/WixHeatBuilder.cs +++ b/ReleaseBuilder/WixHeatBuilder.cs @@ -35,14 +35,14 @@ public static class WixHeatBuilder /// The name of the component group /// A function to generate file IDs. /// The wix file xml contents - public static string CreateWixFilelist(string sourceFolder, string version, string folderPrefix = "$(var.HarvestPath)", string directoryRefName = "INSTALLLOCATION", string componentGroupId = "DUPLICATIBIN", Func? fileIdGenerator = null) + public static string CreateWixFilelist(string sourceFolder, string version, string folderPrefix = "$(var.HarvestPath)", string directoryRefName = "INSTALLLOCATION", string componentGroupId = "DUPLICATIBIN", string wixNs = "http://schemas.microsoft.com/wix/2006/wi", Func? fileIdGenerator = null) { var itemIds = new Dictionary(); fileIdGenerator ??= (x) => Path.GetRelativePath(sourceFolder, x).Replace("\\", "_").Replace("/", "_").Replace(":", "_").Replace(" ", "_").Replace("-", "_"); Func pathTransformer = (x) => $"{folderPrefix}{Path.GetRelativePath(sourceFolder, x)}"; var doc = new XmlDocument(); - doc.LoadXml(""); + doc.LoadXml($""); var root = doc.DocumentElement!; var fragment = doc.CreateElement("Fragment"); root.AppendChild(fragment); From 22840d49d9c5ceec9e40d3a6877f8ac56fe4fee9 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 16 Jan 2025 09:56:54 +0100 Subject: [PATCH 006/362] Created a more robust identifier generator --- ReleaseBuilder/WixHeatBuilder.cs | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/ReleaseBuilder/WixHeatBuilder.cs b/ReleaseBuilder/WixHeatBuilder.cs index bab4a38f3..d16df263f 100644 --- a/ReleaseBuilder/WixHeatBuilder.cs +++ b/ReleaseBuilder/WixHeatBuilder.cs @@ -18,6 +18,7 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +using System.Text.RegularExpressions; using System.Xml; namespace ReleaseBuilder; @@ -27,6 +28,27 @@ namespace ReleaseBuilder; /// public static class WixHeatBuilder { + + /// + /// Converts a string to a valid Wix identifier. + /// + /// The input string. + /// The WiX identifier. + public static string ConvertToIdentifier(string input) + { + if (string.IsNullOrEmpty(input)) + throw new ArgumentException("Input string cannot be null or empty."); + + // Remove invalid characters (only allow A-Z, a-z, 0-9, _, .) + string cleanedInput = Regex.Replace(input, @"[^A-Za-z0-9_.]", "_"); + + // Ensure the identifier starts with a letter or underscore + if (!char.IsLetter(cleanedInput[0]) && cleanedInput[0] != '_') + cleanedInput = "_" + cleanedInput; + + return cleanedInput; + } + /// /// Creates a Wix filelist from a directory /// @@ -38,7 +60,7 @@ public static class WixHeatBuilder public static string CreateWixFilelist(string sourceFolder, string version, string folderPrefix = "$(var.HarvestPath)", string directoryRefName = "INSTALLLOCATION", string componentGroupId = "DUPLICATIBIN", string wixNs = "http://schemas.microsoft.com/wix/2006/wi", Func? fileIdGenerator = null) { var itemIds = new Dictionary(); - fileIdGenerator ??= (x) => Path.GetRelativePath(sourceFolder, x).Replace("\\", "_").Replace("/", "_").Replace(":", "_").Replace(" ", "_").Replace("-", "_"); + fileIdGenerator ??= (x) => ConvertToIdentifier(Path.GetRelativePath(sourceFolder, x)); Func pathTransformer = (x) => $"{folderPrefix}{Path.GetRelativePath(sourceFolder, x)}"; var doc = new XmlDocument(); From eac8cf62ebe5a82c63c081a5a9d98b9dd91e627d Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 16 Jan 2025 21:24:18 +0100 Subject: [PATCH 007/362] Fixed length issue for wix --- ReleaseBuilder/WixHeatBuilder.cs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/ReleaseBuilder/WixHeatBuilder.cs b/ReleaseBuilder/WixHeatBuilder.cs index d16df263f..79fdcb1cf 100644 --- a/ReleaseBuilder/WixHeatBuilder.cs +++ b/ReleaseBuilder/WixHeatBuilder.cs @@ -18,6 +18,8 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +using System.Security.Cryptography; +using System.Text; using System.Text.RegularExpressions; using System.Xml; @@ -40,7 +42,14 @@ public static class WixHeatBuilder throw new ArgumentException("Input string cannot be null or empty."); // Remove invalid characters (only allow A-Z, a-z, 0-9, _, .) - string cleanedInput = Regex.Replace(input, @"[^A-Za-z0-9_.]", "_"); + var cleanedInput = Regex.Replace(input, @"[^A-Za-z0-9_.]", "_"); + + if (cleanedInput.Length > 72) + { + var hash = BitConverter.ToString(SHA256.HashData(Encoding.UTF8.GetBytes(cleanedInput))) + .Replace("-", string.Empty)[..12]; + cleanedInput = cleanedInput[..(72 - hash.Length)] + hash; + } // Ensure the identifier starts with a letter or underscore if (!char.IsLetter(cleanedInput[0]) && cleanedInput[0] != '_') From d3f6bd4b2b02dc2f612db42683cac9aaf6db0893 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 17 Jan 2025 08:57:02 +0100 Subject: [PATCH 008/362] Added preliminary remote sync tool --- Duplicati.sln | 9 ++ Tools/RemoteSynchronization/Program.cs | 97 +++++++++++++++++++ .../RemoteSynchronization.csproj | 13 +++ 3 files changed, 119 insertions(+) create mode 100644 Tools/RemoteSynchronization/Program.cs create mode 100644 Tools/RemoteSynchronization/RemoteSynchronization.csproj diff --git a/Duplicati.sln b/Duplicati.sln index 879a03397..47c8cfa25 100644 --- a/Duplicati.sln +++ b/Duplicati.sln @@ -166,6 +166,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.Crashlog" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.Backend.CIFS", "Duplicati\Library\Backend\CIFS\Duplicati.Library.Backend.CIFS.csproj", "{836E0557-B40C-4DC7-9A2A-5C062F9ACC6B}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tools", "Tools", "{15388C37-9218-4818-972E-738EEA8F1602}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RemoteSynchronization", "Tools\RemoteSynchronization\RemoteSynchronization.csproj", "{D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -472,6 +476,10 @@ Global {836E0557-B40C-4DC7-9A2A-5C062F9ACC6B}.Debug|Any CPU.Build.0 = Debug|Any CPU {836E0557-B40C-4DC7-9A2A-5C062F9ACC6B}.Release|Any CPU.ActiveCfg = Release|Any CPU {836E0557-B40C-4DC7-9A2A-5C062F9ACC6B}.Release|Any CPU.Build.0 = Release|Any CPU + {D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -532,6 +540,7 @@ Global {CA1A0EE8-DFFF-42AF-B15D-EF538357A2DE} = {E1A9B303-F281-45C5-A4F6-CADD9DE3F3C4} {8ACA2736-3C69-4FA5-BCF9-5EDF50CAF332} = {566EBBDA-19A4-4056-A615-D901D57D2439} {836E0557-B40C-4DC7-9A2A-5C062F9ACC6B} = {E1A9B303-F281-45C5-A4F6-CADD9DE3F3C4} + {D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D} = {15388C37-9218-4818-972E-738EEA8F1602} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {8B40BAFE-D862-4397-9495-8F5EAF5CE80C} diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs new file mode 100644 index 000000000..bd19630fe --- /dev/null +++ b/Tools/RemoteSynchronization/Program.cs @@ -0,0 +1,97 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Duplicati.Library.Main; +using Duplicati.Library.Common.IO; +using Duplicati.Library.Interface; +using System.IO; +using System.Threading; + + +namespace RemoteSynchronization +{ + class Program + { + static void Main(string[] args) + { + var home = Environment.GetEnvironmentVariable("HOME"); + var base_path = $"{home}/git/duplicati-carl/rsync"; + var l1 = $"{base_path}/l1"; + var l2 = $"{base_path}/l2"; + var l1r = $"{base_path}/l1_restore"; + var l2r = $"{base_path}/l2_restore"; + var backup_path = $"{home}/tmp/adaptivecpp"; + + Dictionary options = new (){ + ["passphrase"] = "1234" + }; + + // Create the directories if they do not exist + foreach (var p in new string[] { base_path, l1, l2, l1r, l2r }) + { + if (!SystemIO.IO_OS.DirectoryExists(p)) + SystemIO.IO_OS.DirectoryCreate(p); + } + + // Backup the first level + using (var c = new Controller($"file://{l1}", options, null)) + { + var results = c.Backup([backup_path]); + Console.WriteLine($"Backed up {results.AddedFiles} files to {l1}"); + } + + // Sync the first level to the second level + long count = 0; + using (var b1 = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend($"file://{l1}", options)) + { + var b1s = b1 as IStreamingBackend; + System.Diagnostics.Debug.Assert(b1s != null); + using (var b2 = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend($"file://{l2}", options)) + { + var b2s = b2 as IStreamingBackend; + System.Diagnostics.Debug.Assert(b2s != null); + var files1 = b1s.List(); + Console.WriteLine($"Found {files1.Count()} remote files in {l1}"); + var files2 = b2s.List(); + Console.WriteLine($"Found {files2.Count()} remote files in {l2}"); + if (files1.Count() >= files2.Count()) + { + foreach (var f1 in files1) + { + if (!files2.Any(f2 => f2.Name == f1.Name)) + { + var s = new MemoryStream(); + var s1 = b1s.GetAsync(f1.Name, s, CancellationToken.None); + s1.Wait(); + var s2 = b2s.PutAsync(f1.Name, s, CancellationToken.None); + s2.Wait(); + count++; + } + } + } + else + { + Console.WriteLine("Error: l1 has less files than l2 - something is wrong"); + } + } + } + Console.WriteLine($"Synced {count} files from {l1} to {l2}"); + + // Try to restore the first level + options["restore-path"] = l1r; + using (var c = new Controller($"file://{l1}", options, null)) + { + var results = c.Restore([Path.Combine(backup_path, "*")]); + Console.WriteLine($"Restored {results.RestoredFiles} files to {options["restore-path"]}"); + } + + // Try to restore the second level + options["restore-path"] = l2r; + using (var c = new Controller($"file://{l2}", options, null)) + { + var results = c.Restore([]); + Console.WriteLine($"Restored {results.RestoredFiles} files to {options["restore-path"]}"); + } + } + } +} diff --git a/Tools/RemoteSynchronization/RemoteSynchronization.csproj b/Tools/RemoteSynchronization/RemoteSynchronization.csproj new file mode 100644 index 000000000..d1e587209 --- /dev/null +++ b/Tools/RemoteSynchronization/RemoteSynchronization.csproj @@ -0,0 +1,13 @@ + + + + Exe + net8.0 + disable + enable + + + + + + From 2e502ea7be252614a895912ba773b82da34dd349 Mon Sep 17 00:00:00 2001 From: "Marcelo C." Date: Fri, 17 Jan 2025 14:00:14 -0300 Subject: [PATCH 009/362] Help Command unification All CLI tools and executables now accept a unified set of help command variants: "help", "/help","--help", "usage", "/usage", "--usage", "/h", "-h" Affected executables: Duplicati.Agent Duplicati.CommandLine Duplicati.CommandLine.AutoUpdater Duplicati.CommandLine.BackendTester Duplicati.CommandLine.BackendTool Duplicati.CommandLine.ConfigurationImporter Duplicati.CommandLine.SecretTool Duplicati.CommandLine.Snapshots Duplicati.GUI.TrayIcon Duplicati.Server Duplicati.Service Duplicati.WindowsService Note that SharpAesCrypt command is not included in this because the parameter parsing is part of the SharpAesCrypt code base. --- Duplicati/Agent/Program.cs | 39 ++++++------- .../CommandLine/BackendTester/Program.cs | 43 ++++++++------- Duplicati/CommandLine/BackendTool/Program.cs | 41 +++++++------- Duplicati/CommandLine/CLI/Program.cs | 43 ++++++++------- Duplicati/CommandLine/SecretTool/Program.cs | 6 +- Duplicati/CommandLine/ServerUtil/Program.cs | 2 + .../GUI/Duplicati.GUI.TrayIcon/Program.cs | 55 ++++++++----------- Duplicati/Library/Snapshots/Program.cs | 43 ++++++++------- .../Utility/Duplicati.Library.Utility.csproj | 1 + .../Library/Utility/HelpOptionExtensions.cs | 50 +++++++++++++++++ Duplicati/Server/Program.cs | 44 +++++++-------- Duplicati/WindowsService/Program.cs | 42 +++++++------- 12 files changed, 233 insertions(+), 176 deletions(-) create mode 100644 Duplicati/Library/Utility/HelpOptionExtensions.cs diff --git a/Duplicati/Agent/Program.cs b/Duplicati/Agent/Program.cs index 285784511..b5a9f7baa 100644 --- a/Duplicati/Agent/Program.cs +++ b/Duplicati/Agent/Program.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System.CommandLine; @@ -166,6 +166,7 @@ public static class Program context.ExitCode = 1; } }) + .UseAdditionalHelpAliases() .Build() .InvokeAsync(args); } diff --git a/Duplicati/CommandLine/BackendTester/Program.cs b/Duplicati/CommandLine/BackendTester/Program.cs index 9b8518521..0af39f96e 100644 --- a/Duplicati/CommandLine/BackendTester/Program.cs +++ b/Duplicati/CommandLine/BackendTester/Program.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -32,7 +32,7 @@ using Duplicati.StreamUtil; namespace Duplicati.CommandLine.BackendTester { public class Program - { + { class TempFile { public readonly string remotefilename; @@ -82,7 +82,8 @@ namespace Duplicati.CommandLine.BackendTester List args = new List(_args); Dictionary options = Library.Utility.CommandLineParser.ExtractOptions(args); - if (args.Count != 1 || String.Equals(args[0], "help", StringComparison.OrdinalIgnoreCase) || args[0] == "?") + if (args.Count != 1 || HelpOptionExtensions.AlternativeHelpStrings.Any(helpString => + args.Contains(helpString, StringComparer.OrdinalIgnoreCase))) { Console.WriteLine("Usage: ://:@"); Console.WriteLine("Example: ftp://user:pass@server/folder"); diff --git a/Duplicati/CommandLine/BackendTool/Program.cs b/Duplicati/CommandLine/BackendTool/Program.cs index 0e37d5541..eaed7fbf1 100644 --- a/Duplicati/CommandLine/BackendTool/Program.cs +++ b/Duplicati/CommandLine/BackendTool/Program.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Interface; @@ -74,7 +74,8 @@ namespace Duplicati.CommandLine.BackendTool } - if (args.Count < 2 || String.Equals(args[0], "help", StringComparison.OrdinalIgnoreCase) || args[0] == "?" || command == null) + if (args.Count < 2 || HelpOptionExtensions.AlternativeHelpStrings.Any(helpString => + args.Contains(helpString, StringComparer.OrdinalIgnoreCase)) || command == null) { if (command == null && args.Count >= 2) { diff --git a/Duplicati/CommandLine/CLI/Program.cs b/Duplicati/CommandLine/CLI/Program.cs index 66e7bb7a0..0fc00eaff 100644 --- a/Duplicati/CommandLine/CLI/Program.cs +++ b/Duplicati/CommandLine/CLI/Program.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -27,11 +27,13 @@ using System.IO; using Duplicati.Library.Interface; using Duplicati.Library.AutoUpdater; using Duplicati.Library.Crashlog; +using Duplicati.Library.Utility; namespace Duplicati.CommandLine { public class Program { + private static readonly string LOGTAG = Library.Logging.Log.LogTagFromType(); public static bool FROM_COMMANDLINE = false; @@ -163,7 +165,8 @@ namespace Duplicati.CommandLine } // Probe for "help" to avoid extra processing - if (cargs.Count == 0 || (string.Equals(cargs[0], "help", StringComparison.OrdinalIgnoreCase))) + if (cargs.Count == 0 || HelpOptionExtensions.AlternativeHelpStrings.Any(helpString => + cargs.Contains(helpString, StringComparer.OrdinalIgnoreCase))) { return Commands.Help(outwriter, setup, cargs, options, filter); } diff --git a/Duplicati/CommandLine/SecretTool/Program.cs b/Duplicati/CommandLine/SecretTool/Program.cs index 8148b2278..fb59dba26 100644 --- a/Duplicati/CommandLine/SecretTool/Program.cs +++ b/Duplicati/CommandLine/SecretTool/Program.cs @@ -25,6 +25,8 @@ using System.CommandLine.NamingConventionBinder; using System.CommandLine.Parsing; using Duplicati.Library.DynamicLoader; using Duplicati.Library.Interface; +using Duplicati.Library.Utility; +using Uri = System.Uri; namespace Duplicati.CommandLine.SecretTool; @@ -74,7 +76,9 @@ public static class Program Console.WriteLine("Exception: " + ex); context.ExitCode = 1; } - }).Build() + }) + .UseAdditionalHelpAliases() + .Build() .InvokeAsync(args); } diff --git a/Duplicati/CommandLine/ServerUtil/Program.cs b/Duplicati/CommandLine/ServerUtil/Program.cs index 8485fbc8d..2ba212806 100644 --- a/Duplicati/CommandLine/ServerUtil/Program.cs +++ b/Duplicati/CommandLine/ServerUtil/Program.cs @@ -22,6 +22,7 @@ using System.CommandLine; using System.CommandLine.Builder; using System.CommandLine.Parsing; using Duplicati.CommandLine.ServerUtil.Commands; +using Duplicati.Library.Utility; namespace Duplicati.CommandLine.ServerUtil; @@ -80,6 +81,7 @@ public static class Program await next(context); }) + .UseAdditionalHelpAliases() .Build() .InvokeAsync(args); } diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs index de14695d5..6215be08a 100644 --- a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs +++ b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -26,7 +26,9 @@ using System.Linq; using System.Net; using Duplicati.Library.Interface; using Duplicati.Library.RestAPI; +using Duplicati.Library.Utility; using Duplicati.Server; +using Uri = System.Uri; namespace Duplicati.GUI.TrayIcon { @@ -67,8 +69,6 @@ namespace Duplicati.GUI.TrayIcon private static bool disableTrayIconLogin = false; private static bool openui = false; private static Uri serverURL = new Uri(DEFAULT_HOSTURL); - - public static string BrowserCommand { get { return _browser_command; } } public static Server.Database.Connection databaseConnection = null; @@ -84,16 +84,9 @@ namespace Duplicati.GUI.TrayIcon if (OperatingSystem.IsWindows() && !Library.Utility.Utility.ParseBoolOption(options, DETACHED_PROCESS)) Library.Utility.Win32.AttachConsole(Library.Utility.Win32.ATTACH_PARENT_PROCESS); - - foreach (string s in args) - if ( - s.Equals("help", StringComparison.OrdinalIgnoreCase) || - s.Equals("/help", StringComparison.OrdinalIgnoreCase) || - s.Equals("usage", StringComparison.OrdinalIgnoreCase) || - s.Equals("/usage", StringComparison.OrdinalIgnoreCase)) - options["help"] = ""; - - if (options.ContainsKey("help")) + + if (HelpOptionExtensions.AlternativeHelpStrings.Any(helpString => + args.Contains(helpString, StringComparer.OrdinalIgnoreCase))) { Console.WriteLine("Supported commandline arguments:"); Console.WriteLine(); diff --git a/Duplicati/Library/Snapshots/Program.cs b/Duplicati/Library/Snapshots/Program.cs index 38f7f1fa1..b1fd61386 100644 --- a/Duplicati/Library/Snapshots/Program.cs +++ b/Duplicati/Library/Snapshots/Program.cs @@ -1,28 +1,30 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; +using System.Linq; using Duplicati.Library.AutoUpdater; using Duplicati.Library.Common; +using Duplicati.Library.Utility; namespace Duplicati.Library.Snapshots { @@ -72,7 +74,8 @@ namespace Duplicati.Library.Snapshots if (args.Count == 0) args = new List { System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location) }; - if (args.Count != 1) + if (args.Count != 1 || HelpOptionExtensions.AlternativeHelpStrings.Any(helpString => + args.Contains(helpString, StringComparer.OrdinalIgnoreCase))) { Console.WriteLine(@$"Usage: {PackageHelper.GetExecutableName(PackageHelper.NamedExecutable.Snapshots)} [test-folder] diff --git a/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj b/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj index cc7364776..8e8400f4d 100644 --- a/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj +++ b/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj @@ -10,6 +10,7 @@ + diff --git a/Duplicati/Library/Utility/HelpOptionExtensions.cs b/Duplicati/Library/Utility/HelpOptionExtensions.cs new file mode 100644 index 000000000..91f4e87eb --- /dev/null +++ b/Duplicati/Library/Utility/HelpOptionExtensions.cs @@ -0,0 +1,50 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using System.CommandLine.Builder; +using System.Linq; + +namespace Duplicati.Library.Utility; + +public static class HelpOptionExtensions +{ + /// + /// Aliases to support for help command + /// + public static readonly string[] AlternativeHelpStrings = ["help", "/help","--help", "usage", "/usage", "--usage", "/h", "-h"]; + + /// + /// Adds the following aliases: "help", "/help", "--usage", "/usage". + /// These aliases will work the same way as the default help options (-h, --help). + /// + /// The command line builder to extend. + /// The command line builder with additional help aliases. + public static CommandLineBuilder UseAdditionalHelpAliases(this CommandLineBuilder builder) + { + var rootHelpOption = builder.Command.Options.FirstOrDefault(o => o.GetType().Name == "HelpOption"); + if (rootHelpOption == null) return builder; + foreach (var alias in AlternativeHelpStrings) + if (!rootHelpOption.Aliases.Contains(alias)) + rootHelpOption.AddAlias(alias); + + return builder; + } +} \ No newline at end of file diff --git a/Duplicati/Server/Program.cs b/Duplicati/Server/Program.cs index 498f4c138..2a28b4bb6 100644 --- a/Duplicati/Server/Program.cs +++ b/Duplicati/Server/Program.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -44,11 +44,7 @@ namespace Duplicati.Server { public class Program { - - private static readonly string[] AlternativeHelpStrings = ["help", "/help", "usage", "/usage", "--help"]; - private static readonly string[] ParameterFileOptionStrings = ["parameters-file", "parameterfile"]; - private const string PING_PONG_KEEPALIVE_OPTION = "ping-pong-keepalive"; private const string WINDOWS_EVENTLOG_OPTION = "windows-eventlog"; private const string WINDOWS_EVENTLOG_LEVEL_OPTION = "windows-eventlog-level"; @@ -227,7 +223,7 @@ namespace Duplicati.Server var commandlineOptions = optionsWithFilter.Item1; var filter = optionsWithFilter.Item2; - if (_args.Select(s => s.ToLower()).Intersect(AlternativeHelpStrings.Select(x => x.ToLower())).Any()) + if (HelpOptionExtensions.AlternativeHelpStrings.Any(helpString => args.Contains(helpString, StringComparer.OrdinalIgnoreCase))) { return ShowHelp(writeToConsoleOnException); } diff --git a/Duplicati/WindowsService/Program.cs b/Duplicati/WindowsService/Program.cs index ed0762f8b..4c37b643e 100644 --- a/Duplicati/WindowsService/Program.cs +++ b/Duplicati/WindowsService/Program.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.IO; @@ -24,11 +24,13 @@ using System.Linq; using System.Reflection; using System.ServiceProcess; using Duplicati.Library.AutoUpdater; +using Duplicati.Library.Utility; namespace Duplicati.WindowsService { public class Program { + [STAThread] public static int Main(string[] args) { @@ -40,7 +42,7 @@ namespace Duplicati.WindowsService var uninstall = args != null && args.Any(x => string.Equals("uninstall", x, StringComparison.OrdinalIgnoreCase)); var install_agent = args != null && args.Any(x => string.Equals("install-agent", x, StringComparison.OrdinalIgnoreCase) || string.Equals("install-only-agent", x, StringComparison.OrdinalIgnoreCase)); var uninstall_agent = args != null && args.Any(x => string.Equals("uninstall-agent", x, StringComparison.OrdinalIgnoreCase)); - var help = args != null && args.Any(x => string.Equals("help", x, StringComparison.OrdinalIgnoreCase)); + var help = args != null && HelpOptionExtensions.AlternativeHelpStrings.Any(helpString => args.Contains(helpString, StringComparer.OrdinalIgnoreCase));; if (help) { From 04205d50847479c46351a05d6b97bd99d705e443 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sat, 18 Jan 2025 11:03:52 +0100 Subject: [PATCH 010/362] Added support for CORS --- Duplicati/Library/RestAPI/Strings.cs | 1 + Duplicati/Server/Program.cs | 4 +- Duplicati/Server/WebServerLoader.cs | 11 +++++- Duplicati/WebserverCore/DuplicatiWebserver.cs | 38 +++++++++++++++++-- .../Extensions/WebApplicationExtensions.cs | 9 +++-- 5 files changed, 54 insertions(+), 9 deletions(-) diff --git a/Duplicati/Library/RestAPI/Strings.cs b/Duplicati/Library/RestAPI/Strings.cs index b773359f1..115f56e7b 100644 --- a/Duplicati/Library/RestAPI/Strings.cs +++ b/Duplicati/Library/RestAPI/Strings.cs @@ -73,6 +73,7 @@ Error message: {0}", error); } public static string WebserverApiOnlyDescription { get { return LC.L(@"Disable the web interface and only allow API access"); } } public static string WebserverDisableSigninTokensDescription { get { return LC.L(@"Disable the use of signin tokens"); } } public static string WebserverSpaPathsDescription { get { return LC.L(@"The relative paths that should be served as single page applications, separated with semicolons."); } } + public static string WebserverCorsOriginsDescription { get { return LC.L(@"A list of CORS origins to allow, separated with semicolons. Each origin must be a valid URL."); } } public static string WebserverTimezoneDescription { get { return LC.L(@"The timezone to use for the webserver. The timezone must be a valid timezone identifier, such as ""America/New_York"" or ""UTC"". Common three-letter abbreviations like ""CET"" are supported, but ambiguous in some cases."); } } public static string DisabledbencryptionLong { get { return LC.L(@"Use this option to disable database encryption of sensitive fields"); } } public static string DisabledbencryptionShort { get { return LC.L(@"Disable database encryption"); } } diff --git a/Duplicati/Server/Program.cs b/Duplicati/Server/Program.cs index 2e69d3d23..3a42d2bdd 100644 --- a/Duplicati/Server/Program.cs +++ b/Duplicati/Server/Program.cs @@ -402,7 +402,8 @@ namespace Duplicati.Server parsedOptions.Servername, parsedOptions.AllowedHostnames, parsedOptions.DisableStaticFiles, - parsedOptions.SPAPaths + parsedOptions.SPAPaths, + parsedOptions.CorsOrigins ); var server = DuplicatiWebserver.CreateWebServer(mappedSettings, connection); @@ -949,6 +950,7 @@ namespace Duplicati.Server new Duplicati.Library.Interface.CommandLineArgument(WebServerLoader.OPTION_WEBSERVICE_DISABLE_SIGNIN_TOKENS, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.Program.WebserverDisableSigninTokensDescription, Strings.Program.WebserverDisableSigninTokensDescription), new Duplicati.Library.Interface.CommandLineArgument(WebServerLoader.OPTION_WEBSERVICE_SPAPATHS, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Path, Strings.Program.WebserverSpaPathsDescription, Strings.Program.WebserverSpaPathsDescription, WebServerLoader.DEFAULT_OPTION_SPAPATHS), new Duplicati.Library.Interface.CommandLineArgument(WebServerLoader.OPTION_WEBSERVICE_TIMEZONE, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.String, Strings.Program.WebserverTimezoneDescription, Strings.Program.WebserverTimezoneDescription, TimeZoneHelper.GetLocalTimeZone(), null, TimeZoneHelper.GetTimeZones().Select(x => x.Id).ToArray()), + new Duplicati.Library.Interface.CommandLineArgument(WebServerLoader.OPTION_WEBSERVICE_CORS_ORIGINS, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Path, Strings.Program.WebserverCorsOriginsDescription, Strings.Program.WebserverCorsOriginsDescription, WebServerLoader.DEFAULT_OPTION_SPAPATHS), new Duplicati.Library.Interface.CommandLineArgument(PING_PONG_KEEPALIVE_OPTION, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.Program.PingpongkeepaliveShort, Strings.Program.PingpongkeepaliveLong), new Duplicati.Library.Interface.CommandLineArgument("log-retention", Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Timespan, Strings.Program.LogretentionShort, Strings.Program.LogretentionLong, DEFAULT_LOG_RETENTION), new Duplicati.Library.Interface.CommandLineArgument("server-datafolder", Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Path, Strings.Program.ServerdatafolderShort, Strings.Program.ServerdatafolderLong(DATAFOLDER_ENV_NAME), System.IO.Path.Combine(System.Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), Library.AutoUpdater.AutoUpdateSettings.AppName)), diff --git a/Duplicati/Server/WebServerLoader.cs b/Duplicati/Server/WebServerLoader.cs index 7f14dd5d0..3b82cf767 100644 --- a/Duplicati/Server/WebServerLoader.cs +++ b/Duplicati/Server/WebServerLoader.cs @@ -99,6 +99,10 @@ public static class WebServerLoader /// public const string OPTION_WEBSERVICE_SPAPATHS = "webservice-spa-paths"; /// + /// The CORS origins to allow + /// + public const string OPTION_WEBSERVICE_CORS_ORIGINS = "webservice-cors-origins"; + /// /// Option for setting the webservice timezone /// public const string OPTION_WEBSERVICE_TIMEZONE = "webservice-timezone"; @@ -154,6 +158,7 @@ public static class WebServerLoader /// The allowed hostnames /// If static files should be disabled /// The paths to serve as SPAs + /// The origins to allow for CORS public record ParsedWebserverSettings( string WebRoot, int Port, @@ -162,7 +167,8 @@ public static class WebServerLoader string Servername, IEnumerable AllowedHostnames, bool DisableStaticFiles, - IEnumerable SPAPaths + IEnumerable SPAPaths, + IEnumerable CorsOrigins ); @@ -251,7 +257,8 @@ public static class WebServerLoader string.Format("{0} v{1}", Library.AutoUpdater.AutoUpdateSettings.AppName, Library.AutoUpdater.UpdaterManager.SelfVersion.Version), (connection.ApplicationSettings.AllowedHostnames ?? string.Empty).Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries), Duplicati.Library.Utility.Utility.ParseBoolOption(options, OPTION_WEBSERVICE_API_ONLY), - spaPathsString.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries) + spaPathsString.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries), + options.GetValueOrDefault(OPTION_WEBSERVICE_CORS_ORIGINS)?.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries) ?? Enumerable.Empty() ); // Materialize the list of ports, and move the last-used port to the front, so we try the last-known port first diff --git a/Duplicati/WebserverCore/DuplicatiWebserver.cs b/Duplicati/WebserverCore/DuplicatiWebserver.cs index 2010d70b4..9f5f0b4fc 100644 --- a/Duplicati/WebserverCore/DuplicatiWebserver.cs +++ b/Duplicati/WebserverCore/DuplicatiWebserver.cs @@ -48,6 +48,11 @@ public class DuplicatiWebserver /// The log tag for this class /// private static readonly string LOGTAG = Library.Logging.Log.LogTagFromType(); + /// + /// The name of the CORS policy + /// + public const string CorsPolicyName = "CustomCorsPolicy"; + /// /// The configuration for the server /// @@ -68,6 +73,11 @@ public class DuplicatiWebserver /// public required int Port { get; init; } + /// + /// The port the server is listening on + /// + public required bool CorsEnabled { get; init; } + /// /// The task that will be set when the server is terminated /// @@ -93,6 +103,7 @@ public class DuplicatiWebserver /// The allowed hostnames /// If static files should be disabled /// The paths to serve as SPAs + /// The origins to allow for CORS public record InitSettings( string WebRoot, int Port, @@ -101,7 +112,8 @@ public class DuplicatiWebserver string Servername, IEnumerable AllowedHostnames, bool DisableStaticFiles, - IEnumerable SPAPaths + IEnumerable SPAPaths, + IEnumerable CorsOrigins ); /// @@ -253,6 +265,22 @@ public class DuplicatiWebserver builder.Services.AddHttpClient(); + var useCors = settings.CorsOrigins != null && settings.CorsOrigins.Any(); + if (useCors) + { + builder.Services.AddCors(options => + { + options.AddPolicy( + name: CorsPolicyName, + policy => + { + policy.WithOrigins(settings.CorsOrigins!.ToArray()) + .AllowAnyHeader() + .AllowAnyMethod(); + }); + }); + } + var app = builder.Build(); HttpClientHelper.Configure(app.Services.GetRequiredService()); @@ -271,6 +299,9 @@ public class DuplicatiWebserver if (!settings.DisableStaticFiles) app.UseDefaultStaticFiles(settings.WebRoot, settings.SPAPaths); + if (useCors) + app.UseCors(CorsPolicyName); + app.UseExceptionHandler(app => { app.Run(async context => @@ -308,7 +339,8 @@ public class DuplicatiWebserver { Configuration = builder.Configuration, App = app, - Port = settings.Port + Port = settings.Port, + CorsEnabled = useCors }; } @@ -345,7 +377,7 @@ public class DuplicatiWebserver public Task Start() { App.MapHealthChecks("/health"); - App.AddEndpoints() + App.AddEndpoints(CorsEnabled) .UseNotifications("/notifications"); return TerminationTask = App.RunAsync(); diff --git a/Duplicati/WebserverCore/Extensions/WebApplicationExtensions.cs b/Duplicati/WebserverCore/Extensions/WebApplicationExtensions.cs index 00485cef9..254551ef9 100644 --- a/Duplicati/WebserverCore/Extensions/WebApplicationExtensions.cs +++ b/Duplicati/WebserverCore/Extensions/WebApplicationExtensions.cs @@ -26,12 +26,12 @@ namespace Duplicati.WebserverCore.Extensions; public static class WebApplicationExtensions { - public static WebApplication AddEndpoints(this WebApplication application) + public static WebApplication AddEndpoints(this WebApplication application, bool useCors) { - return AddV1(application); + return AddV1(application, useCors); } - private static WebApplication AddV1(WebApplication application) + private static WebApplication AddV1(WebApplication application, bool useCors) { var mapperInterfaceType = typeof(IEndpointV1); var endpoints = @@ -46,6 +46,9 @@ public static class WebApplicationExtensions if (!string.IsNullOrWhiteSpace(PreSharedKeyFilter.PreSharedKey)) group = group.AddEndpointFilter(); + if (useCors) + group.RequireCors(DuplicatiWebserver.CorsPolicyName); + foreach (var endpoint in endpoints) { var methodMap = endpoint.GetMethod(nameof(IEndpointV1.Map), BindingFlags.Static | BindingFlags.Public); From 85833a2ff5f0b4978708fdbc2639f7d8418de162 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 20 Jan 2025 07:58:21 +0100 Subject: [PATCH 011/362] Added a copy function to the rsync tool --- Tools/RemoteSynchronization/Program.cs | 40 +++++++++++++++++--------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index bd19630fe..baee7b5be 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using Duplicati.Library.Main; @@ -56,18 +56,7 @@ namespace RemoteSynchronization Console.WriteLine($"Found {files2.Count()} remote files in {l2}"); if (files1.Count() >= files2.Count()) { - foreach (var f1 in files1) - { - if (!files2.Any(f2 => f2.Name == f1.Name)) - { - var s = new MemoryStream(); - var s1 = b1s.GetAsync(f1.Name, s, CancellationToken.None); - s1.Wait(); - var s2 = b2s.PutAsync(f1.Name, s, CancellationToken.None); - s2.Wait(); - count++; - } - } + count = Copy(b1s, b2s, files1); } else { @@ -93,5 +82,30 @@ namespace RemoteSynchronization Console.WriteLine($"Restored {results.RestoredFiles} files to {options["restore-path"]}"); } } + + + // Forcefully synchronize the remote backends + private static long Copy(IStreamingBackend bsrc, IStreamingBackend bdst, IEnumerable files) + { + long successful_copies = 0; + foreach (var f in files) + { + try + { + var s = new MemoryStream(); + var s1 = bsrc.GetAsync(f.Name, s, CancellationToken.None); + s1.Wait(); + var s2 = bdst.PutAsync(f.Name, s, CancellationToken.None); + s2.Wait(); + successful_copies++; + } + catch (Exception e) + { + // TODO log the error + Console.WriteLine($"Error copying {f.Name}: {e.Message}"); + } + } + return successful_copies; + } } } From fcb41ebec928fc80048d95fb5fdb5b8a04ea0f32 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 20 Jan 2025 07:58:39 +0100 Subject: [PATCH 012/362] Added TODO comments in the rsync tool --- Tools/RemoteSynchronization/Program.cs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index baee7b5be..85d04445a 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using Duplicati.Library.Main; @@ -14,6 +14,7 @@ namespace RemoteSynchronization { static void Main(string[] args) { + // TODO testing start var home = Environment.GetEnvironmentVariable("HOME"); var base_path = $"{home}/git/duplicati-carl/rsync"; var l1 = $"{base_path}/l1"; @@ -40,6 +41,8 @@ namespace RemoteSynchronization Console.WriteLine($"Backed up {results.AddedFiles} files to {l1}"); } + // TODO testing end + // Sync the first level to the second level long count = 0; using (var b1 = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend($"file://{l1}", options)) @@ -66,6 +69,7 @@ namespace RemoteSynchronization } Console.WriteLine($"Synced {count} files from {l1} to {l2}"); + // TODO testing start // Try to restore the first level options["restore-path"] = l1r; using (var c = new Controller($"file://{l1}", options, null)) @@ -81,8 +85,16 @@ namespace RemoteSynchronization var results = c.Restore([]); Console.WriteLine($"Restored {results.RestoredFiles} files to {options["restore-path"]}"); } + // TODO testing end } + // TODO check whether a backend is empty + // TODO check whether the backend versions "match": src.version >= dst.version + // TODO check whether the backend files are consistent: src.files.count >= dst.files.count + // TODO handle files deleted from src, which should also be deleted in dst + // TODO maximum retention?: don't delete files, only rename old ones. + // TODO check database consistency. I.e. have both databases, check that the block, volume, files, etc match up. + // TODO introduce these checks as a post processing step? Especially the database consistency check, as that is often recreated from the index files. // Forcefully synchronize the remote backends private static long Copy(IStreamingBackend bsrc, IStreamingBackend bdst, IEnumerable files) From 7b64f0da926bdc0a5d2a91f54fe91320518a90be Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 21 Jan 2025 15:14:50 +0100 Subject: [PATCH 013/362] Added functions for verifying the file list --- Tools/RemoteSynchronization/Program.cs | 75 +++++++++++++++++++------- 1 file changed, 57 insertions(+), 18 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 85d04445a..6cc6a80a5 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using Duplicati.Library.Main; @@ -44,7 +44,6 @@ namespace RemoteSynchronization // TODO testing end // Sync the first level to the second level - long count = 0; using (var b1 = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend($"file://{l1}", options)) { var b1s = b1 as IStreamingBackend; @@ -53,21 +52,11 @@ namespace RemoteSynchronization { var b2s = b2 as IStreamingBackend; System.Diagnostics.Debug.Assert(b2s != null); - var files1 = b1s.List(); - Console.WriteLine($"Found {files1.Count()} remote files in {l1}"); - var files2 = b2s.List(); - Console.WriteLine($"Found {files2.Count()} remote files in {l2}"); - if (files1.Count() >= files2.Count()) - { - count = Copy(b1s, b2s, files1); - } - else - { - Console.WriteLine("Error: l1 has less files than l2 - something is wrong"); - } + var (to_copy, to_delete) = PrepareFileLists(b1s, b2s); + var copied = Copy(b1s, b2s, to_copy); + Console.WriteLine($"Copied {copied} files from {l1} to {l2}"); } } - Console.WriteLine($"Synced {count} files from {l1} to {l2}"); // TODO testing start // Try to restore the first level @@ -97,7 +86,7 @@ namespace RemoteSynchronization // TODO introduce these checks as a post processing step? Especially the database consistency check, as that is often recreated from the index files. // Forcefully synchronize the remote backends - private static long Copy(IStreamingBackend bsrc, IStreamingBackend bdst, IEnumerable files) + private static long Copy(IStreamingBackend b_src, IStreamingBackend b_dst, IEnumerable files) { long successful_copies = 0; foreach (var f in files) @@ -105,19 +94,69 @@ namespace RemoteSynchronization try { var s = new MemoryStream(); - var s1 = bsrc.GetAsync(f.Name, s, CancellationToken.None); + var s1 = b_src.GetAsync(f.Name, s, CancellationToken.None); s1.Wait(); - var s2 = bdst.PutAsync(f.Name, s, CancellationToken.None); + var s2 = b_dst.PutAsync(f.Name, s, CancellationToken.None); s2.Wait(); successful_copies++; } catch (Exception e) { // TODO log the error + // TODO return failed files count? Console.WriteLine($"Error copying {f.Name}: {e.Message}"); } } return successful_copies; } + + private static (IEnumerable,IEnumerable) PrepareFileLists(IStreamingBackend b_src, IStreamingBackend b_dst) + { + var files_src = b_src.List(); + var files_dst = b_dst.List(); + + // Shortcut for empty destination + if (!files_dst.Any()) + { + return (files_src, []); + } + + var lookup_src = files_src.ToDictionary(x => x.Name); + var lookup_dst = files_dst.ToDictionary(x => x.Name); + + var to_copy = new List(); + var to_delete = new HashSet(); + + // Find all of the files in src that are not in dst, have a different size or have a more recent modification date + foreach (var f_src in files_src) + { + if (lookup_dst.TryGetValue(f_src.Name, out var f_dst)) + { + if (f_src.Size != f_dst.Size || f_src.LastModification > f_dst.LastModification) + { + to_copy.Add(f_src); + to_delete.Add(f_dst.Name); + } + } + else + { + to_copy.Add(f_src); + } + } + + // Find all of the files in dst that are not in src + foreach (var f_dst in files_dst) + { + if (to_delete.Contains(f_dst.Name)) + continue; + + if (!lookup_src.ContainsKey(f_dst.Name)) + { + to_delete.Add(f_dst.Name); + } + } + + return (to_copy, to_delete.Select(x => lookup_dst[x])); + } } } From 3d93c4885be0db594e23a959530ec293757d0ea0 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 21 Jan 2025 15:16:42 +0100 Subject: [PATCH 014/362] Implement file deletion functionality in remote synchronization tool --- Tools/RemoteSynchronization/Program.cs | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 6cc6a80a5..603e5b67e 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -53,6 +53,8 @@ namespace RemoteSynchronization var b2s = b2 as IStreamingBackend; System.Diagnostics.Debug.Assert(b2s != null); var (to_copy, to_delete) = PrepareFileLists(b1s, b2s); + var deleted = Delete(b2s, to_delete); + Console.WriteLine($"Deleted {deleted} files from {l2}"); var copied = Copy(b1s, b2s, to_copy); Console.WriteLine($"Copied {copied} files from {l1} to {l2}"); } @@ -78,7 +80,6 @@ namespace RemoteSynchronization } // TODO check whether a backend is empty - // TODO check whether the backend versions "match": src.version >= dst.version // TODO check whether the backend files are consistent: src.files.count >= dst.files.count // TODO handle files deleted from src, which should also be deleted in dst // TODO maximum retention?: don't delete files, only rename old ones. @@ -110,6 +111,26 @@ namespace RemoteSynchronization return successful_copies; } + private static long Delete(IStreamingBackend b, IEnumerable files) + { + long successful_deletes = 0; + foreach (var f in files) + { + try + { + b.DeleteAsync(f.Name, CancellationToken.None).Wait(); + successful_deletes++; + } + catch (Exception e) + { + // TODO log the error + // TODO return failed files count? + Console.WriteLine($"Error deleting {f.Name}: {e.Message}"); + } + } + return successful_deletes; + } + private static (IEnumerable,IEnumerable) PrepareFileLists(IStreamingBackend b_src, IStreamingBackend b_dst) { var files_src = b_src.List(); From d97e9073f4dff92bcf4d590ab350085ddc9d7f8f Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 21 Jan 2025 15:16:59 +0100 Subject: [PATCH 015/362] Add rename functionality to remote synchronization tool --- Tools/RemoteSynchronization/Program.cs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 603e5b67e..38423445a 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -179,5 +179,29 @@ namespace RemoteSynchronization return (to_copy, to_delete.Select(x => lookup_dst[x])); } + + private static long Rename(IStreamingBackend b, IEnumerable files) + { + long successful_renames = 0; + string suffix = $"{System.DateTime.Now:yyyyMMddHHmmss}.old"; + foreach (var f in files) + { + try + { + var downloaded = new MemoryStream(); + b.GetAsync(f.Name, downloaded, CancellationToken.None).Wait(); + b.PutAsync($"{f.Name}.{suffix}", downloaded, CancellationToken.None).Wait(); + successful_renames++; + } + catch (Exception e) + { + // TODO log the error + // TODO return failed files count? + Console.WriteLine($"Error renaming {f.Name}: {e.Message}"); + } + } + return successful_renames; + } + } } From ce3583c388dcbffed492872f69fd481affa0a46b Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 21 Jan 2025 15:18:32 +0100 Subject: [PATCH 016/362] Add additional TODO comments for future enhancements --- Tools/RemoteSynchronization/Program.cs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 38423445a..8b923eb1e 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using Duplicati.Library.Main; @@ -83,8 +83,16 @@ namespace RemoteSynchronization // TODO check whether the backend files are consistent: src.files.count >= dst.files.count // TODO handle files deleted from src, which should also be deleted in dst // TODO maximum retention?: don't delete files, only rename old ones. + // TODO have concurrency parameters: uploaders, downloaders + // TODO low memory mode, where things aren't kept in memory. Maybe utilize SQLite? + // TODO Force parameter + // TODO Progress reporting + // TODO Dry run + // TODO Logging + // TODO check database consistency. I.e. have both databases, check that the block, volume, files, etc match up. // TODO introduce these checks as a post processing step? Especially the database consistency check, as that is often recreated from the index files. + // TODO This tool shouldn't handle it, but for convenience, it should support making the seperate call to the regular Duplicati on the destination backend, which alread carries this functionality. // Forcefully synchronize the remote backends private static long Copy(IStreamingBackend b_src, IStreamingBackend b_dst, IEnumerable files) @@ -103,8 +111,6 @@ namespace RemoteSynchronization } catch (Exception e) { - // TODO log the error - // TODO return failed files count? Console.WriteLine($"Error copying {f.Name}: {e.Message}"); } } @@ -123,8 +129,6 @@ namespace RemoteSynchronization } catch (Exception e) { - // TODO log the error - // TODO return failed files count? Console.WriteLine($"Error deleting {f.Name}: {e.Message}"); } } @@ -195,8 +199,6 @@ namespace RemoteSynchronization } catch (Exception e) { - // TODO log the error - // TODO return failed files count? Console.WriteLine($"Error renaming {f.Name}: {e.Message}"); } } From a2b13b1805cdb0896db3015e2a03b29909104e8d Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 21 Jan 2025 15:22:04 +0100 Subject: [PATCH 017/362] Add copyright notice and licensing to the remote synchronization tool --- Tools/RemoteSynchronization/Program.cs | 27 +++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 8b923eb1e..c96642a76 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -1,10 +1,31 @@ -using System; -using System.Collections.Generic; -using System.Linq; +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + using Duplicati.Library.Main; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; +using System; +using System.Collections.Generic; using System.IO; +using System.Linq; using System.Threading; From cee1b8428d94ed4cc402d3c4b95947abe62ae153 Mon Sep 17 00:00:00 2001 From: "Marcelo C." Date: Tue, 21 Jan 2025 12:45:53 -0300 Subject: [PATCH 018/362] Centralizing the check on Helper function. --- Duplicati/CommandLine/BackendTester/Program.cs | 3 +-- Duplicati/CommandLine/BackendTool/Program.cs | 3 +-- Duplicati/CommandLine/CLI/Program.cs | 3 +-- Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs | 3 +-- Duplicati/Library/Snapshots/Program.cs | 3 +-- Duplicati/Library/Utility/HelpOptionExtensions.cs | 11 ++++++++++- Duplicati/Server/Program.cs | 2 +- Duplicati/WindowsService/Program.cs | 2 +- 8 files changed, 17 insertions(+), 13 deletions(-) diff --git a/Duplicati/CommandLine/BackendTester/Program.cs b/Duplicati/CommandLine/BackendTester/Program.cs index 0af39f96e..9332386d8 100644 --- a/Duplicati/CommandLine/BackendTester/Program.cs +++ b/Duplicati/CommandLine/BackendTester/Program.cs @@ -82,8 +82,7 @@ namespace Duplicati.CommandLine.BackendTester List args = new List(_args); Dictionary options = Library.Utility.CommandLineParser.ExtractOptions(args); - if (args.Count != 1 || HelpOptionExtensions.AlternativeHelpStrings.Any(helpString => - args.Contains(helpString, StringComparer.OrdinalIgnoreCase))) + if (args.Count != 1 || HelpOptionExtensions.IsArgumentAnyHelpString(args)) { Console.WriteLine("Usage: ://:@"); Console.WriteLine("Example: ftp://user:pass@server/folder"); diff --git a/Duplicati/CommandLine/BackendTool/Program.cs b/Duplicati/CommandLine/BackendTool/Program.cs index eaed7fbf1..46782329d 100644 --- a/Duplicati/CommandLine/BackendTool/Program.cs +++ b/Duplicati/CommandLine/BackendTool/Program.cs @@ -74,8 +74,7 @@ namespace Duplicati.CommandLine.BackendTool } - if (args.Count < 2 || HelpOptionExtensions.AlternativeHelpStrings.Any(helpString => - args.Contains(helpString, StringComparer.OrdinalIgnoreCase)) || command == null) + if (args.Count < 2 || HelpOptionExtensions.IsArgumentAnyHelpString(args) || command == null) { if (command == null && args.Count >= 2) { diff --git a/Duplicati/CommandLine/CLI/Program.cs b/Duplicati/CommandLine/CLI/Program.cs index 0fc00eaff..adea1cc01 100644 --- a/Duplicati/CommandLine/CLI/Program.cs +++ b/Duplicati/CommandLine/CLI/Program.cs @@ -165,8 +165,7 @@ namespace Duplicati.CommandLine } // Probe for "help" to avoid extra processing - if (cargs.Count == 0 || HelpOptionExtensions.AlternativeHelpStrings.Any(helpString => - cargs.Contains(helpString, StringComparer.OrdinalIgnoreCase))) + if (cargs.Count == 0 || HelpOptionExtensions.IsArgumentAnyHelpString(cargs)) { return Commands.Help(outwriter, setup, cargs, options, filter); } diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs index 6215be08a..0e7ca2af6 100644 --- a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs +++ b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs @@ -85,8 +85,7 @@ namespace Duplicati.GUI.TrayIcon if (OperatingSystem.IsWindows() && !Library.Utility.Utility.ParseBoolOption(options, DETACHED_PROCESS)) Library.Utility.Win32.AttachConsole(Library.Utility.Win32.ATTACH_PARENT_PROCESS); - if (HelpOptionExtensions.AlternativeHelpStrings.Any(helpString => - args.Contains(helpString, StringComparer.OrdinalIgnoreCase))) + if (HelpOptionExtensions.IsArgumentAnyHelpString(args)) { Console.WriteLine("Supported commandline arguments:"); Console.WriteLine(); diff --git a/Duplicati/Library/Snapshots/Program.cs b/Duplicati/Library/Snapshots/Program.cs index b1fd61386..5f39417e9 100644 --- a/Duplicati/Library/Snapshots/Program.cs +++ b/Duplicati/Library/Snapshots/Program.cs @@ -74,8 +74,7 @@ namespace Duplicati.Library.Snapshots if (args.Count == 0) args = new List { System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location) }; - if (args.Count != 1 || HelpOptionExtensions.AlternativeHelpStrings.Any(helpString => - args.Contains(helpString, StringComparer.OrdinalIgnoreCase))) + if (args.Count != 1 || HelpOptionExtensions.IsArgumentAnyHelpString(args)) { Console.WriteLine(@$"Usage: {PackageHelper.GetExecutableName(PackageHelper.NamedExecutable.Snapshots)} [test-folder] diff --git a/Duplicati/Library/Utility/HelpOptionExtensions.cs b/Duplicati/Library/Utility/HelpOptionExtensions.cs index 91f4e87eb..209adfab3 100644 --- a/Duplicati/Library/Utility/HelpOptionExtensions.cs +++ b/Duplicati/Library/Utility/HelpOptionExtensions.cs @@ -19,6 +19,8 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +using System; +using System.Collections.Generic; using System.CommandLine.Builder; using System.Linq; @@ -29,7 +31,7 @@ public static class HelpOptionExtensions /// /// Aliases to support for help command /// - public static readonly string[] AlternativeHelpStrings = ["help", "/help","--help", "usage", "/usage", "--usage", "/h", "-h"]; + private static readonly string[] AlternativeHelpStrings = ["help", "/help","--help", "usage", "/usage", "--usage", "/h", "-h"]; /// /// Adds the following aliases: "help", "/help", "--usage", "/usage". @@ -47,4 +49,11 @@ public static class HelpOptionExtensions return builder; } + + /// + /// Returns true if any of the arguments are any of the help strings. + /// + /// Arguments + public static bool IsArgumentAnyHelpString(IEnumerable args) + => args != null && AlternativeHelpStrings.Any(x => args.Contains(x, StringComparer.OrdinalIgnoreCase)); } \ No newline at end of file diff --git a/Duplicati/Server/Program.cs b/Duplicati/Server/Program.cs index 2a28b4bb6..310bf89eb 100644 --- a/Duplicati/Server/Program.cs +++ b/Duplicati/Server/Program.cs @@ -223,7 +223,7 @@ namespace Duplicati.Server var commandlineOptions = optionsWithFilter.Item1; var filter = optionsWithFilter.Item2; - if (HelpOptionExtensions.AlternativeHelpStrings.Any(helpString => args.Contains(helpString, StringComparer.OrdinalIgnoreCase))) + if (HelpOptionExtensions.IsArgumentAnyHelpString(args)) { return ShowHelp(writeToConsoleOnException); } diff --git a/Duplicati/WindowsService/Program.cs b/Duplicati/WindowsService/Program.cs index 4c37b643e..21b71782a 100644 --- a/Duplicati/WindowsService/Program.cs +++ b/Duplicati/WindowsService/Program.cs @@ -42,7 +42,7 @@ namespace Duplicati.WindowsService var uninstall = args != null && args.Any(x => string.Equals("uninstall", x, StringComparison.OrdinalIgnoreCase)); var install_agent = args != null && args.Any(x => string.Equals("install-agent", x, StringComparison.OrdinalIgnoreCase) || string.Equals("install-only-agent", x, StringComparison.OrdinalIgnoreCase)); var uninstall_agent = args != null && args.Any(x => string.Equals("uninstall-agent", x, StringComparison.OrdinalIgnoreCase)); - var help = args != null && HelpOptionExtensions.AlternativeHelpStrings.Any(helpString => args.Contains(helpString, StringComparer.OrdinalIgnoreCase));; + var help = args != null && HelpOptionExtensions.IsArgumentAnyHelpString(args); if (help) { From af69436a1e70a2d311c96102206b43e577a2c69d Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 22 Jan 2025 06:10:43 +0100 Subject: [PATCH 019/362] Added preliminary unittest for the remote synchronization tool --- Duplicati/Tools/Duplicati.Tools.csproj | 9 ++-- Duplicati/UnitTest/ToolTests.cs | 65 ++++++++++++++++++++++++++ Tools/RemoteSynchronization/Program.cs | 54 +-------------------- 3 files changed, 73 insertions(+), 55 deletions(-) create mode 100644 Duplicati/UnitTest/ToolTests.cs diff --git a/Duplicati/Tools/Duplicati.Tools.csproj b/Duplicati/Tools/Duplicati.Tools.csproj index ecb48b4cc..28ca8eea6 100644 --- a/Duplicati/Tools/Duplicati.Tools.csproj +++ b/Duplicati/Tools/Duplicati.Tools.csproj @@ -5,7 +5,7 @@ Library Copyright © 2024 Team Duplicati, MIT license - + utility-scripts\DuplicatiVerify.ps1 @@ -24,12 +24,15 @@ utility-scripts\pause-resume.py - + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/Duplicati/UnitTest/ToolTests.cs b/Duplicati/UnitTest/ToolTests.cs new file mode 100644 index 000000000..a1e48084c --- /dev/null +++ b/Duplicati/UnitTest/ToolTests.cs @@ -0,0 +1,65 @@ +using Duplicati.Library.Common.IO; +using Duplicati.Library.Main; +using NUnit.Framework; +using System; +using System.Collections.Generic; +using System.IO; + +namespace Duplicati.UnitTest +{ + + public class ToolTests : BasicSetupHelper + { + + [Test] + [Category("Tools")] + public void TestRemoteSynchronization() + { + var home = Environment.GetEnvironmentVariable("HOME"); + var base_path = $"{home}/git/duplicati-carl/rsync"; + var l1 = $"{base_path}/l1"; + var l2 = $"{base_path}/l2"; + var l1r = $"{base_path}/l1_restore"; + var l2r = $"{base_path}/l2_restore"; + var backup_path = $"{home}/tmp/adaptivecpp"; + + Dictionary options = new () + { + ["passphrase"] = "1234" + }; + + // Create the directories if they do not exist + foreach (var p in new string[] { base_path, l1, l2, l1r, l2r }) + { + if (!SystemIO.IO_OS.DirectoryExists(p)) + SystemIO.IO_OS.DirectoryCreate(p); + } + + // Backup the first level + using (var c = new Controller($"file://{l1}", options, null)) + { + var results = c.Backup([backup_path]); + Console.WriteLine($"Backed up {results.AddedFiles} files to {l1}"); + } + + // TODO Call the remote synchronization tool + var exe = RemoteSynchronization.Program.Main; + + options["restore-path"] = l1r; + using (var c = new Controller($"file://{l1}", options, null)) + { + var results = c.Restore([Path.Combine(backup_path, "*")]); + Console.WriteLine($"Restored {results.RestoredFiles} files to {options["restore-path"]}"); + } + + // Try to restore the second level + options["restore-path"] = l2r; + using (var c = new Controller($"file://{l2}", options, null)) + { + var results = c.Restore([]); + Console.WriteLine($"Restored {results.RestoredFiles} files to {options["restore-path"]}"); + } + } + } + +} \ No newline at end of file diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index c96642a76..6ef22f714 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -19,7 +19,6 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using Duplicati.Library.Main; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using System; @@ -31,40 +30,12 @@ using System.Threading; namespace RemoteSynchronization { - class Program + public class Program { - static void Main(string[] args) + public static void Main(string[] args) { - // TODO testing start - var home = Environment.GetEnvironmentVariable("HOME"); - var base_path = $"{home}/git/duplicati-carl/rsync"; - var l1 = $"{base_path}/l1"; - var l2 = $"{base_path}/l2"; - var l1r = $"{base_path}/l1_restore"; - var l2r = $"{base_path}/l2_restore"; - var backup_path = $"{home}/tmp/adaptivecpp"; - Dictionary options = new (){ - ["passphrase"] = "1234" - }; - // Create the directories if they do not exist - foreach (var p in new string[] { base_path, l1, l2, l1r, l2r }) - { - if (!SystemIO.IO_OS.DirectoryExists(p)) - SystemIO.IO_OS.DirectoryCreate(p); - } - - // Backup the first level - using (var c = new Controller($"file://{l1}", options, null)) - { - var results = c.Backup([backup_path]); - Console.WriteLine($"Backed up {results.AddedFiles} files to {l1}"); - } - - // TODO testing end - - // Sync the first level to the second level using (var b1 = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend($"file://{l1}", options)) { var b1s = b1 as IStreamingBackend; @@ -80,29 +51,8 @@ namespace RemoteSynchronization Console.WriteLine($"Copied {copied} files from {l1} to {l2}"); } } - - // TODO testing start - // Try to restore the first level - options["restore-path"] = l1r; - using (var c = new Controller($"file://{l1}", options, null)) - { - var results = c.Restore([Path.Combine(backup_path, "*")]); - Console.WriteLine($"Restored {results.RestoredFiles} files to {options["restore-path"]}"); - } - - // Try to restore the second level - options["restore-path"] = l2r; - using (var c = new Controller($"file://{l2}", options, null)) - { - var results = c.Restore([]); - Console.WriteLine($"Restored {results.RestoredFiles} files to {options["restore-path"]}"); - } - // TODO testing end } - // TODO check whether a backend is empty - // TODO check whether the backend files are consistent: src.files.count >= dst.files.count - // TODO handle files deleted from src, which should also be deleted in dst // TODO maximum retention?: don't delete files, only rename old ones. // TODO have concurrency parameters: uploaders, downloaders // TODO low memory mode, where things aren't kept in memory. Maybe utilize SQLite? From f64d090fdafaeb98256a6083a80ee55b80204f5d Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 22 Jan 2025 09:37:28 +0100 Subject: [PATCH 020/362] Added preliminary commandline argument parsing to remote synchronization --- Tools/RemoteSynchronization/Program.cs | 44 +++++++++++++------ .../RemoteSynchronization.csproj | 1 + 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 6ef22f714..b0cba5528 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -23,34 +23,50 @@ using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using System; using System.Collections.Generic; +using System.CommandLine; using System.IO; using System.Linq; using System.Threading; +using System.Threading.Tasks; namespace RemoteSynchronization { public class Program { - public static void Main(string[] args) + public static async Task Main(string[] args) { + var src_arg = new Argument(name: "backend_src", description: "The source backend string"); + var dst_arg = new Argument(name: "backend_dst", description: "The destination backend string"); + var root_cmd = new RootCommand("Remote Synchronization Tool"); + root_cmd.AddArgument(src_arg); + root_cmd.AddArgument(dst_arg); - using (var b1 = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend($"file://{l1}", options)) + root_cmd.SetHandler(Run, src_arg, dst_arg); + + return await root_cmd.InvokeAsync(args); + } + + private static async Task Run(string src, string dst) + { + var options = new Dictionary(); + using (var b1 = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(src, options)) { - var b1s = b1 as IStreamingBackend; - System.Diagnostics.Debug.Assert(b1s != null); - using (var b2 = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend($"file://{l2}", options)) - { - var b2s = b2 as IStreamingBackend; - System.Diagnostics.Debug.Assert(b2s != null); - var (to_copy, to_delete) = PrepareFileLists(b1s, b2s); - var deleted = Delete(b2s, to_delete); - Console.WriteLine($"Deleted {deleted} files from {l2}"); - var copied = Copy(b1s, b2s, to_copy); - Console.WriteLine($"Copied {copied} files from {l1} to {l2}"); - } + var b1s = b1 as IStreamingBackend; + System.Diagnostics.Debug.Assert(b1s != null); + using (var b2 = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(dst, options)) + { + var b2s = b2 as IStreamingBackend; + System.Diagnostics.Debug.Assert(b2s != null); + var (to_copy, to_delete) = PrepareFileLists(b1s, b2s); + var deleted = Delete(b2s, to_delete); + Console.WriteLine($"Deleted {deleted} files from {dst}"); + var copied = Copy(b1s, b2s, to_copy); + Console.WriteLine($"Copied {copied} files from {src} to {dst}"); + } } + return 42; } // TODO maximum retention?: don't delete files, only rename old ones. diff --git a/Tools/RemoteSynchronization/RemoteSynchronization.csproj b/Tools/RemoteSynchronization/RemoteSynchronization.csproj index d1e587209..7324b2069 100644 --- a/Tools/RemoteSynchronization/RemoteSynchronization.csproj +++ b/Tools/RemoteSynchronization/RemoteSynchronization.csproj @@ -9,5 +9,6 @@ + From 71376b156fc68298298d8ce21f671c6f8c2238a6 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 27 Jan 2025 08:30:12 +0100 Subject: [PATCH 021/362] Changed to use async when available --- Tools/RemoteSynchronization/Program.cs | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index b0cba5528..5e1246c67 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -60,9 +60,9 @@ namespace RemoteSynchronization var b2s = b2 as IStreamingBackend; System.Diagnostics.Debug.Assert(b2s != null); var (to_copy, to_delete) = PrepareFileLists(b1s, b2s); - var deleted = Delete(b2s, to_delete); + var deleted = await DeleteAsync(b2s, to_delete); Console.WriteLine($"Deleted {deleted} files from {dst}"); - var copied = Copy(b1s, b2s, to_copy); + var copied = await CopyAsync(b1s, b2s, to_copy); Console.WriteLine($"Copied {copied} files from {src} to {dst}"); } } @@ -82,7 +82,7 @@ namespace RemoteSynchronization // TODO This tool shouldn't handle it, but for convenience, it should support making the seperate call to the regular Duplicati on the destination backend, which alread carries this functionality. // Forcefully synchronize the remote backends - private static long Copy(IStreamingBackend b_src, IStreamingBackend b_dst, IEnumerable files) + private static async Task CopyAsync(IStreamingBackend b_src, IStreamingBackend b_dst, IEnumerable files) { long successful_copies = 0; foreach (var f in files) @@ -90,10 +90,8 @@ namespace RemoteSynchronization try { var s = new MemoryStream(); - var s1 = b_src.GetAsync(f.Name, s, CancellationToken.None); - s1.Wait(); - var s2 = b_dst.PutAsync(f.Name, s, CancellationToken.None); - s2.Wait(); + await b_src.GetAsync(f.Name, s, CancellationToken.None); + await b_dst.PutAsync(f.Name, s, CancellationToken.None); successful_copies++; } catch (Exception e) @@ -104,14 +102,14 @@ namespace RemoteSynchronization return successful_copies; } - private static long Delete(IStreamingBackend b, IEnumerable files) + private static async Task DeleteAsync(IStreamingBackend b, IEnumerable files) { long successful_deletes = 0; foreach (var f in files) { try { - b.DeleteAsync(f.Name, CancellationToken.None).Wait(); + await b.DeleteAsync(f.Name, CancellationToken.None); successful_deletes++; } catch (Exception e) @@ -171,7 +169,7 @@ namespace RemoteSynchronization return (to_copy, to_delete.Select(x => lookup_dst[x])); } - private static long Rename(IStreamingBackend b, IEnumerable files) + private static async Task RenameAsync(IStreamingBackend b, IEnumerable files) { long successful_renames = 0; string suffix = $"{System.DateTime.Now:yyyyMMddHHmmss}.old"; @@ -180,8 +178,8 @@ namespace RemoteSynchronization try { var downloaded = new MemoryStream(); - b.GetAsync(f.Name, downloaded, CancellationToken.None).Wait(); - b.PutAsync($"{f.Name}.{suffix}", downloaded, CancellationToken.None).Wait(); + await b.GetAsync(f.Name, downloaded, CancellationToken.None); + await b.PutAsync($"{f.Name}.{suffix}", downloaded, CancellationToken.None); successful_renames++; } catch (Exception e) From 0e291bbeab42445b9d814d104362f955edba7068 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 27 Jan 2025 08:34:41 +0100 Subject: [PATCH 022/362] Reuse memorystream when applicable --- Tools/RemoteSynchronization/Program.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 5e1246c67..8052975fd 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2025, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a @@ -85,13 +85,14 @@ namespace RemoteSynchronization private static async Task CopyAsync(IStreamingBackend b_src, IStreamingBackend b_dst, IEnumerable files) { long successful_copies = 0; + using var s = new MemoryStream(); foreach (var f in files) { try { - var s = new MemoryStream(); await b_src.GetAsync(f.Name, s, CancellationToken.None); await b_dst.PutAsync(f.Name, s, CancellationToken.None); + s.SetLength(0); successful_copies++; } catch (Exception e) @@ -172,14 +173,14 @@ namespace RemoteSynchronization private static async Task RenameAsync(IStreamingBackend b, IEnumerable files) { long successful_renames = 0; - string suffix = $"{System.DateTime.Now:yyyyMMddHHmmss}.old"; + using var downloaded = new MemoryStream(); foreach (var f in files) { try { - var downloaded = new MemoryStream(); await b.GetAsync(f.Name, downloaded, CancellationToken.None); await b.PutAsync($"{f.Name}.{suffix}", downloaded, CancellationToken.None); + downloaded.SetLength(0); successful_renames++; } catch (Exception e) From 54736ff449df961b2862b5abe6c05660baae5413 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 27 Jan 2025 08:34:58 +0100 Subject: [PATCH 023/362] Remove redundant System. --- Tools/RemoteSynchronization/Program.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 8052975fd..a255c179c 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -173,6 +173,7 @@ namespace RemoteSynchronization private static async Task RenameAsync(IStreamingBackend b, IEnumerable files) { long successful_renames = 0; + string suffix = $"{DateTime.Now:yyyyMMddHHmmss}.old"; using var downloaded = new MemoryStream(); foreach (var f in files) { From 4e5282ccfd5b0f510ce7e29a2daa886c9b639f59 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 27 Jan 2025 08:35:48 +0100 Subject: [PATCH 024/362] After a rename, the original file should be deleted. --- Tools/RemoteSynchronization/Program.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index a255c179c..c539f0ad3 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2025, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a @@ -181,6 +181,7 @@ namespace RemoteSynchronization { await b.GetAsync(f.Name, downloaded, CancellationToken.None); await b.PutAsync($"{f.Name}.{suffix}", downloaded, CancellationToken.None); + await b.DeleteAsync(f.Name, CancellationToken.None); downloaded.SetLength(0); successful_renames++; } From 42d54a47f79582db24875e11462b16d960315c57 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 27 Jan 2025 08:37:32 +0100 Subject: [PATCH 025/362] Formatting --- Tools/RemoteSynchronization/Program.cs | 32 +++++++++++++------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index c539f0ad3..11f992565 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -51,21 +51,21 @@ namespace RemoteSynchronization private static async Task Run(string src, string dst) { var options = new Dictionary(); - using (var b1 = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(src, options)) - { - var b1s = b1 as IStreamingBackend; - System.Diagnostics.Debug.Assert(b1s != null); - using (var b2 = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(dst, options)) - { - var b2s = b2 as IStreamingBackend; - System.Diagnostics.Debug.Assert(b2s != null); - var (to_copy, to_delete) = PrepareFileLists(b1s, b2s); - var deleted = await DeleteAsync(b2s, to_delete); - Console.WriteLine($"Deleted {deleted} files from {dst}"); - var copied = await CopyAsync(b1s, b2s, to_copy); - Console.WriteLine($"Copied {copied} files from {src} to {dst}"); - } - } + + using var b1 = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(src, options); + var b1s = b1 as IStreamingBackend; + System.Diagnostics.Debug.Assert(b1s != null); + + using var b2 = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(dst, options); + var b2s = b2 as IStreamingBackend; + System.Diagnostics.Debug.Assert(b2s != null); + + var (to_copy, to_delete) = PrepareFileLists(b1s, b2s); + var deleted = await DeleteAsync(b2s, to_delete); + Console.WriteLine($"Deleted {deleted} files from {dst}"); + var copied = await CopyAsync(b1s, b2s, to_copy); + Console.WriteLine($"Copied {copied} files from {src} to {dst}"); + return 42; } @@ -121,7 +121,7 @@ namespace RemoteSynchronization return successful_deletes; } - private static (IEnumerable,IEnumerable) PrepareFileLists(IStreamingBackend b_src, IStreamingBackend b_dst) + private static (IEnumerable, IEnumerable) PrepareFileLists(IStreamingBackend b_src, IStreamingBackend b_dst) { var files_src = b_src.List(); var files_dst = b_dst.List(); From 3c62fb7e39e4134b386445dbfcfdcab6e7286450 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 27 Jan 2025 09:53:48 +0100 Subject: [PATCH 026/362] Preliminary rsync tool test implemented. --- Duplicati/UnitTest/ToolTests.cs | 93 ++++++++++++++++++++++++-- Tools/RemoteSynchronization/Program.cs | 2 +- 2 files changed, 88 insertions(+), 7 deletions(-) diff --git a/Duplicati/UnitTest/ToolTests.cs b/Duplicati/UnitTest/ToolTests.cs index a1e48084c..9591a3c55 100644 --- a/Duplicati/UnitTest/ToolTests.cs +++ b/Duplicati/UnitTest/ToolTests.cs @@ -1,16 +1,44 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + using Duplicati.Library.Common.IO; using Duplicati.Library.Main; using NUnit.Framework; using System; using System.Collections.Generic; using System.IO; +using System.Linq; namespace Duplicati.UnitTest { + /// + /// Tests the tools. + /// public class ToolTests : BasicSetupHelper { + /// + /// Tests the remote synchronization tool. + /// [Test] [Category("Tools")] public void TestRemoteSynchronization() @@ -21,9 +49,9 @@ namespace Duplicati.UnitTest var l2 = $"{base_path}/l2"; var l1r = $"{base_path}/l1_restore"; var l2r = $"{base_path}/l2_restore"; - var backup_path = $"{home}/tmp/adaptivecpp"; + var path_to_backup = $"{home}/tmp/adaptivecpp"; - Dictionary options = new () + Dictionary options = new() { ["passphrase"] = "1234" }; @@ -38,28 +66,81 @@ namespace Duplicati.UnitTest // Backup the first level using (var c = new Controller($"file://{l1}", options, null)) { - var results = c.Backup([backup_path]); + var results = c.Backup([path_to_backup]); + Assert.AreEqual(0, results.Errors.Count()); + Assert.AreEqual(0, results.Warnings.Count()); Console.WriteLine($"Backed up {results.AddedFiles} files to {l1}"); } - // TODO Call the remote synchronization tool + // Call the tool var exe = RemoteSynchronization.Program.Main; + var args = new string[] { $"file://{l1}", $"file://{l2}" }; + var async_call = exe(args); + async_call.Wait(); + Assert.AreEqual(0, async_call.Result, "Remote synchronization tool did not return 0."); + // Verify that the directories are equal + Assert.IsTrue(DirectoriesAndContentsAreEqual(l1, l2), "Synchronized directories are not equal"); + + // Try to restore the first level options["restore-path"] = l1r; using (var c = new Controller($"file://{l1}", options, null)) { - var results = c.Restore([Path.Combine(backup_path, "*")]); + var results = c.Restore([Path.Combine(path_to_backup, "*")]); + Assert.AreEqual(0, results.Errors.Count()); + Assert.AreEqual(0, results.Warnings.Count()); Console.WriteLine($"Restored {results.RestoredFiles} files to {options["restore-path"]}"); } + Assert.IsTrue(DirectoriesAndContentsAreEqual(path_to_backup, l1r), "Restored first level files is not equal to original files"); // Try to restore the second level options["restore-path"] = l2r; using (var c = new Controller($"file://{l2}", options, null)) { var results = c.Restore([]); + Assert.AreEqual(0, results.Errors.Count()); + Assert.AreEqual(0, results.Warnings.Count()); Console.WriteLine($"Restored {results.RestoredFiles} files to {options["restore-path"]}"); } + Assert.IsTrue(DirectoriesAndContentsAreEqual(path_to_backup, l2r), "Restored second level files is not equal to original files"); } - } + /// + /// Compares two directories and their contents. + /// + /// The first directory. + /// The second directory. + /// `true` if the two directories contain exactly the same files and if the contents of the files are equivalent. `false` otherwise. + public static bool DirectoriesAndContentsAreEqual(string d1, string d2) + { + // Recursively get the files in the two directories + var f1s = Directory.EnumerateFiles(d1, "*", SearchOption.AllDirectories).Select(x => x[(d1.Length + 1)..]).OrderDescending(); + var f2s = Directory.EnumerateFiles(d2, "*", SearchOption.AllDirectories).Select(x => x[(d2.Length + 1)..]).OrderDescending(); + + // If the two directories do not contain the same files, return false + if (!f1s.SequenceEqual(f2s)) + return false; + + // Check that each file pair exist and have the same content + foreach (var f1 in f1s) + { + var f1full = Path.Combine(d1, f1); + if (!File.Exists(f1full)) + return false; + + var f2full = Path.Combine(d2, f1); + if (!File.Exists(f2full)) + return false; + + var c1 = File.ReadAllText(f1full); + var c2 = File.ReadAllText(f2full); + + if (!c1.SequenceEqual(c2)) + return false; + } + + return true; + } + + } } \ No newline at end of file diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 11f992565..4972352ea 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -66,7 +66,7 @@ namespace RemoteSynchronization var copied = await CopyAsync(b1s, b2s, to_copy); Console.WriteLine($"Copied {copied} files from {src} to {dst}"); - return 42; + return 0; } // TODO maximum retention?: don't delete files, only rename old ones. From c0f4146ba740e0e5230bc3884372e8b37725d51e Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 27 Jan 2025 10:06:49 +0100 Subject: [PATCH 027/362] Changed the rsync unit test to use the test folders and generate test data --- Duplicati/UnitTest/ToolTests.cs | 61 ++++++++++++++++++++++++--------- 1 file changed, 45 insertions(+), 16 deletions(-) diff --git a/Duplicati/UnitTest/ToolTests.cs b/Duplicati/UnitTest/ToolTests.cs index 9591a3c55..6268e284f 100644 --- a/Duplicati/UnitTest/ToolTests.cs +++ b/Duplicati/UnitTest/ToolTests.cs @@ -43,21 +43,17 @@ namespace Duplicati.UnitTest [Category("Tools")] public void TestRemoteSynchronization() { - var home = Environment.GetEnvironmentVariable("HOME"); - var base_path = $"{home}/git/duplicati-carl/rsync"; - var l1 = $"{base_path}/l1"; - var l2 = $"{base_path}/l2"; - var l1r = $"{base_path}/l1_restore"; - var l2r = $"{base_path}/l2_restore"; - var path_to_backup = $"{home}/tmp/adaptivecpp"; + var l1 = $"{TARGETFOLDER}/l1"; + var l2 = $"{TARGETFOLDER}/l2"; + var l1r = $"{RESTOREFOLDER}/l1_restore"; + var l2r = $"{RESTOREFOLDER}/l2_restore"; - Dictionary options = new() - { - ["passphrase"] = "1234" - }; + var options = TestOptions; + + GenerateTestData(DATAFOLDER, 10, 3, 3, 1024 * 1024); // Create the directories if they do not exist - foreach (var p in new string[] { base_path, l1, l2, l1r, l2r }) + foreach (var p in new string[] { l1, l2, l1r, l2r }) { if (!SystemIO.IO_OS.DirectoryExists(p)) SystemIO.IO_OS.DirectoryCreate(p); @@ -66,7 +62,7 @@ namespace Duplicati.UnitTest // Backup the first level using (var c = new Controller($"file://{l1}", options, null)) { - var results = c.Backup([path_to_backup]); + var results = c.Backup([DATAFOLDER]); Assert.AreEqual(0, results.Errors.Count()); Assert.AreEqual(0, results.Warnings.Count()); Console.WriteLine($"Backed up {results.AddedFiles} files to {l1}"); @@ -86,12 +82,12 @@ namespace Duplicati.UnitTest options["restore-path"] = l1r; using (var c = new Controller($"file://{l1}", options, null)) { - var results = c.Restore([Path.Combine(path_to_backup, "*")]); + var results = c.Restore([Path.Combine(DATAFOLDER, "*")]); Assert.AreEqual(0, results.Errors.Count()); Assert.AreEqual(0, results.Warnings.Count()); Console.WriteLine($"Restored {results.RestoredFiles} files to {options["restore-path"]}"); } - Assert.IsTrue(DirectoriesAndContentsAreEqual(path_to_backup, l1r), "Restored first level files is not equal to original files"); + Assert.IsTrue(DirectoriesAndContentsAreEqual(DATAFOLDER, l1r), "Restored first level files is not equal to original files"); // Try to restore the second level options["restore-path"] = l2r; @@ -102,7 +98,7 @@ namespace Duplicati.UnitTest Assert.AreEqual(0, results.Warnings.Count()); Console.WriteLine($"Restored {results.RestoredFiles} files to {options["restore-path"]}"); } - Assert.IsTrue(DirectoriesAndContentsAreEqual(path_to_backup, l2r), "Restored second level files is not equal to original files"); + Assert.IsTrue(DirectoriesAndContentsAreEqual(DATAFOLDER, l2r), "Restored second level files is not equal to original files"); } /// @@ -142,5 +138,38 @@ namespace Duplicati.UnitTest return true; } + /// + /// Generates test data in the specified directory. + /// + /// The directory to fill with the generated data. + /// How many files the directory should have. + /// How many subdirectories the directory should have. + /// How deep the number of subdirectories within subdirectories should go. + /// The maximum size of the files to generate. + public static void GenerateTestData(string dir, int n_files, int n_dirs, int n_levels, int max_file_size) + { + if (!SystemIO.IO_OS.DirectoryExists(dir)) + SystemIO.IO_OS.DirectoryCreate(dir); + + var rnd = new Random(); + for (int i = 0; i < n_files; i++) + { + var file = Path.Combine(dir, $"file_{i}.txt"); + var size = rnd.Next(1, max_file_size); + var data = new byte[size]; + rnd.NextBytes(data); + File.WriteAllBytes(file, data); + } + + if (n_levels > 0) + { + for (int i = 0; i < n_dirs; i++) + { + var subdir = Path.Combine(dir, $"dir_{i}"); + GenerateTestData(subdir, n_files, n_dirs, n_levels - 1, max_file_size); + } + } + } + } } \ No newline at end of file From 97d62163c56a46dca39bbd9139e15e487df1566b Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 27 Jan 2025 11:46:24 +0100 Subject: [PATCH 028/362] Renamed methods for database path to better distinguish what they do --- Duplicati/CommandLine/CLI/Commands.cs | 46 ++++++++-------- Duplicati/Library/Main/Controller.cs | 40 +++++++------- Duplicati/Library/Main/DatabaseLocator.cs | 55 +++++++++++-------- Duplicati/Server/Program.cs | 2 +- Duplicati/WebserverCore/Dto/GetDbPathDto.cs | 2 +- .../WebserverCore/Endpoints/V1/Backups.cs | 2 +- .../Endpoints/V1/RemoteOperation.cs | 2 +- 7 files changed, 79 insertions(+), 70 deletions(-) diff --git a/Duplicati/CommandLine/CLI/Commands.cs b/Duplicati/CommandLine/CLI/Commands.cs index a1ae87553..7fa32f890 100644 --- a/Duplicati/CommandLine/CLI/Commands.cs +++ b/Duplicati/CommandLine/CLI/Commands.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -163,7 +163,7 @@ namespace Duplicati.CommandLine options.TryGetValue("dbpath", out dbpath); if (string.IsNullOrEmpty(dbpath)) { - dbpath = Library.Main.DatabaseLocator.GetDatabasePath(backend, new Duplicati.Library.Main.Options(options), false, true); + dbpath = Library.Main.DatabaseLocator.GetDatabasePathForCLI(backend, new Duplicati.Library.Main.Options(options), false, true); if (dbpath != null) options["dbpath"] = dbpath; } @@ -366,7 +366,7 @@ namespace Duplicati.CommandLine options.TryGetValue("dbpath", out dbpath); if (string.IsNullOrEmpty(dbpath)) { - dbpath = Library.Main.DatabaseLocator.GetDatabasePath(backend, new Duplicati.Library.Main.Options(options), false, true); + dbpath = Library.Main.DatabaseLocator.GetDatabasePathForCLI(backend, new Duplicati.Library.Main.Options(options), false, true); if (dbpath != null) options["dbpath"] = dbpath; } @@ -820,7 +820,7 @@ namespace Duplicati.CommandLine if (string.IsNullOrEmpty(dbpath)) { if (args.Count > 0) - dbpath = Library.Main.DatabaseLocator.GetDatabasePath(args[0], new Duplicati.Library.Main.Options(options), false, true); + dbpath = Library.Main.DatabaseLocator.GetDatabasePathForCLI(args[0], new Duplicati.Library.Main.Options(options), false, true); if (dbpath == null) { @@ -866,7 +866,7 @@ namespace Duplicati.CommandLine options.TryGetValue("dbpath", out dbpath); if (string.IsNullOrEmpty(dbpath)) { - dbpath = Library.Main.DatabaseLocator.GetDatabasePath(args[0], new Duplicati.Library.Main.Options(options), false, true); + dbpath = Library.Main.DatabaseLocator.GetDatabasePathForCLI(args[0], new Duplicati.Library.Main.Options(options), false, true); if (dbpath != null) options["dbpath"] = dbpath; } diff --git a/Duplicati/Library/Main/Controller.cs b/Duplicati/Library/Main/Controller.cs index eae1752ab..ad5b9cf8b 100644 --- a/Duplicati/Library/Main/Controller.cs +++ b/Duplicati/Library/Main/Controller.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -700,7 +700,7 @@ namespace Duplicati.Library.Main } if (string.IsNullOrEmpty(m_options.Dbpath)) - m_options.Dbpath = DatabaseLocator.GetDatabasePath(m_backend, m_options); + m_options.Dbpath = DatabaseLocator.GetDatabasePathForCLI(m_backend, m_options); ValidateOptions(); } diff --git a/Duplicati/Library/Main/DatabaseLocator.cs b/Duplicati/Library/Main/DatabaseLocator.cs index d4ddc57a0..60fad713b 100644 --- a/Duplicati/Library/Main/DatabaseLocator.cs +++ b/Duplicati/Library/Main/DatabaseLocator.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -71,7 +71,7 @@ namespace Duplicati.Library.Main /// The filename to look for /// The name of the application /// The default storage folder - public static string GetDefaultStorageFolder(string targetfilename, string appName = "Duplicati") + public static string GetDefaultStorageFolderDirect(string targetfilename, string appName = "Duplicati") { //Normal mode uses the systems "(Local) Application Data" folder // %LOCALAPPDATA% on Windows, ~/.config on Linux @@ -113,10 +113,19 @@ namespace Duplicati.Library.Main return folder; } - public static string GetDatabasePath(string backend, Options options, bool autoCreate = true, bool anyUsername = false) + /// + /// Gets the database path for a given backend url, using the JSON configuration file lookup. + /// If is true, a new database is created if none is found. + /// Otherwise, null is returned if no database is found. + /// + /// The backend url + /// The options to use + /// If true, a new database is created if none is found + /// If true, any username is accepted + /// The database path or null + public static string? GetDatabasePathForCLI(string backend, Options? options, bool autoCreate = true, bool anyUsername = false) { - if (options == null) - options = new Options(new Dictionary()); + options ??= new Options(new Dictionary()); if (!string.IsNullOrEmpty(options.Dbpath)) return options.Dbpath; diff --git a/Duplicati/Server/Program.cs b/Duplicati/Server/Program.cs index af8d41924..4a23e19f2 100644 --- a/Duplicati/Server/Program.cs +++ b/Duplicati/Server/Program.cs @@ -762,7 +762,7 @@ namespace Duplicati.Server { //Normal release mode uses the systems "(Local) Application Data" folder // %LOCALAPPDATA% on Windows, ~/.config on Linux, ~/Library/Application\ Support on MacOS - return DatabaseLocator.GetDefaultStorageFolder(SERVER_DATABASE_FILENAME, Library.AutoUpdater.AutoUpdateSettings.AppName); + return DatabaseLocator.GetDefaultStorageFolderDirect(SERVER_DATABASE_FILENAME, Library.AutoUpdater.AutoUpdateSettings.AppName); } } else diff --git a/Duplicati/WebserverCore/Dto/GetDbPathDto.cs b/Duplicati/WebserverCore/Dto/GetDbPathDto.cs index 81850193a..b6cc3b0f9 100644 --- a/Duplicati/WebserverCore/Dto/GetDbPathDto.cs +++ b/Duplicati/WebserverCore/Dto/GetDbPathDto.cs @@ -25,4 +25,4 @@ namespace Duplicati.WebserverCore.Dto; /// /// Whether the database path exists. /// The database path. -public sealed record GetDbPathDto(bool Exists, string Path); +public sealed record GetDbPathDto(bool Exists, string? Path); diff --git a/Duplicati/WebserverCore/Endpoints/V1/Backups.cs b/Duplicati/WebserverCore/Endpoints/V1/Backups.cs index 096061b6f..4a3576cd5 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/Backups.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/Backups.cs @@ -206,7 +206,7 @@ public class Backups : IEndpointV1 { if (existingDb) { - backup.DBPathSetter = Library.Main.DatabaseLocator.GetDatabasePath(data.Backup.TargetURL, null, false, false); + backup.DBPathSetter = Library.Main.DatabaseLocator.GetDatabasePathForCLI(data.Backup.TargetURL, null, false, false); if (string.IsNullOrWhiteSpace(data.Backup.DBPath)) throw new Exception("Unable to find remote db path?"); } diff --git a/Duplicati/WebserverCore/Endpoints/V1/RemoteOperation.cs b/Duplicati/WebserverCore/Endpoints/V1/RemoteOperation.cs index 938198c51..c90e67276 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/RemoteOperation.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/RemoteOperation.cs @@ -49,7 +49,7 @@ namespace Duplicati.WebserverCore.Endpoints.V1 private static Dto.GetDbPathDto ExecuteDbPath(string uri) { - var path = Library.Main.DatabaseLocator.GetDatabasePath(uri, null, false, false); + var path = Library.Main.DatabaseLocator.GetDatabasePathForCLI(uri, null, false, false); return new Dto.GetDbPathDto(!string.IsNullOrWhiteSpace(path), path); } From c05b00ae0bce8fb6607796a59872ed4e137c76f0 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 27 Jan 2025 11:49:17 +0100 Subject: [PATCH 029/362] Implemented support for avoiding storing data in the Windows folder --- Duplicati/Library/Main/DatabaseLocator.cs | 132 +++++++++++++++----- Duplicati/Library/RestAPI/SpecialFolders.cs | 45 +------ Duplicati/Library/Utility/SHGetFolder.cs | 104 +++++++++++++++ Duplicati/Server/Program.cs | 19 +++ 4 files changed, 224 insertions(+), 76 deletions(-) create mode 100644 Duplicati/Library/Utility/SHGetFolder.cs diff --git a/Duplicati/Library/Main/DatabaseLocator.cs b/Duplicati/Library/Main/DatabaseLocator.cs index 60fad713b..fc7e934f0 100644 --- a/Duplicati/Library/Main/DatabaseLocator.cs +++ b/Duplicati/Library/Main/DatabaseLocator.cs @@ -23,21 +23,54 @@ using System; using System.Linq; using System.Collections.Generic; +#nullable enable + namespace Duplicati.Library.Main { public static class DatabaseLocator { + /// + /// The log tag for this class + /// + public static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(DatabaseLocator)); + + /// + /// The entry for a backend configuration, used for serialization + /// public class BackendEntry { - public string Type; - public string Server; - public string Path; - public string Prefix; - public string Username; - //public string Passwordhash; - public int Port; - public string Databasepath; - public string ParameterFile; + /// + /// The type of the backend + /// + public required string Type; + /// + /// The server/hostname of the backend + /// + public required string Server; + /// + /// The path of the backend + /// + public required string Path; + /// + /// The prefix of the backup + /// + public required string Prefix; + /// + /// The username of the backend + /// + public required string? Username; + /// + /// The port of the backend + /// + public required int Port; + /// + /// The path to the database + /// + public required string Databasepath; + /// + /// The path to the parameter file + /// + public required string? ParameterFile; } /// @@ -49,7 +82,8 @@ namespace Duplicati.Library.Main /// Finds a default storage folder, using the operating system specific locations. /// The targetfilename is used to detect locations that are used in previous versions. /// If the targetfilename is found in an old location, but not the current, the old location is used. - /// If running with DEBUG defined, the storage folder is placed in the same folder as the executable + /// If running with DEBUG defined, the storage folder is placed in the same folder as the executable. + /// Note that the folder is not created, only the path is returned. /// /// The filename to look for /// The name of the application @@ -59,7 +93,7 @@ namespace Duplicati.Library.Main #if DEBUG return System.IO.Path.GetDirectoryName(typeof(DatabaseLocator).Assembly.Location) ?? string.Empty; #else - return GetDefaultStorageFolder(targetfilename, appName); + return GetDefaultStorageFolderDirect(targetfilename, appName); #endif } @@ -67,6 +101,7 @@ namespace Duplicati.Library.Main /// Finds a default storage folder, using the operating system specific locations. /// The targetfilename is used to detect locations that are used in previous versions. /// If the targetfilename is found in an old location, but not the current, the old location is used. + /// Note that the folder is not created, only the path is returned. /// /// The filename to look for /// The name of the application @@ -81,17 +116,38 @@ namespace Duplicati.Library.Main { // Special handling for Windows: // - Older versions use %APPDATA% - // - but new versions use %LOCALAPPDATA% + // - New versions use %LOCALAPPDATA% + // - And prevent using C:\Windows\ var newlocation = System.IO.Path.Combine(System.Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), appName); - var prevfile = System.IO.Path.Combine(folder, targetfilename); - var curfile = System.IO.Path.Combine(newlocation, targetfilename); + var folderOrder = new List() { + folder, + newlocation + }; - // If the new file exists, we use that - // If the new file does not exist, and the old file exists we use the old - // Otherwise we use the new location - if (System.IO.File.Exists(curfile) || !System.IO.File.Exists(prevfile)) - folder = newlocation; + // If %LOCALAPPDATA% is inside the Windows folder, prefer a LocalService folder instead + var windowsFolder = Environment.GetFolderPath(Environment.SpecialFolder.Windows); + if (newlocation.StartsWith(windowsFolder, StringComparison.OrdinalIgnoreCase)) + { + var userProfilesFolder = Library.Utility.SHGetFolder.UserProfilesFolder; + if (!string.IsNullOrWhiteSpace(userProfilesFolder)) + folderOrder.Add(System.IO.Path.Combine(userProfilesFolder, "LocalService", appName)); + } + + // Prefer the most recent location + var matches = folderOrder.AsEnumerable() + .Reverse() + .Select(x => System.IO.Path.Combine(x, targetfilename)) + .Where(System.IO.File.Exists) + .ToList(); + + // Use the most recent location found with content + // If none are found, use the most recent location + folder = matches.FirstOrDefault() ?? folderOrder.Last(); + + // Emit a warning if the database is stored in the Windows folder + if (folder.StartsWith(windowsFolder, StringComparison.OrdinalIgnoreCase)) + Logging.Log.WriteWarningMessage(LOGTAG, "DatabaseInWindowsFolder", null, "The database is stored in the Windows folder, this is not recommended as it will be deleted on Windows upgrades."); } if (OperatingSystem.IsMacOS()) @@ -137,22 +193,24 @@ namespace Duplicati.Library.Main if (!System.IO.File.Exists(file)) configs = new List(); else - configs = Newtonsoft.Json.JsonConvert.DeserializeObject>(System.IO.File.ReadAllText(file, System.Text.Encoding.UTF8)); + configs = Newtonsoft.Json.JsonConvert.DeserializeObject>(System.IO.File.ReadAllText(file, System.Text.Encoding.UTF8)) + ?? new List(); var uri = new Library.Utility.Uri(backend); - string server = uri.Host; - string path = uri.Path; - string type = uri.Scheme; - int port = uri.Port; - string username = uri.Username; - string prefix = options.Prefix; + var server = uri.Host; + var path = uri.Path; + var type = uri.Scheme; + var port = uri.Port; + var username = uri.Username; + var prefix = options.Prefix; if (username == null || uri.Password == null) { var sopts = DynamicLoader.BackendLoader.GetSupportedCommands(backend); - var ropts = new Dictionary(options.RawOptions); + var ropts = new Dictionary(options.RawOptions); foreach (var k in uri.QueryParameters.AllKeys) - ropts[k] = uri.QueryParameters[k]; + if (k != null) + ropts[k] = uri.QueryParameters[k]; if (sopts != null) { @@ -174,7 +232,6 @@ namespace Duplicati.Library.Main var matches = (from n in configs where n.Type == type && - //n.Passwordhash == password && n.Username == username && n.Port == port && n.Server == server && @@ -250,27 +307,38 @@ namespace Duplicati.Library.Main { return matches[0].Databasepath; } - } + /// + /// Generates a random name for a database + /// + /// A random name public static string GenerateRandomName() { var rnd = new Random(); - System.Text.StringBuilder backupName = new System.Text.StringBuilder(); + var backupName = new System.Text.StringBuilder(); for (var i = 0; i < 10; i++) backupName.Append((char)rnd.Next('A', 'Z' + 1)); return backupName.ToString(); } + /// + /// Checks if a database path is in use by any backup. + /// If the file does not exist, it is assumed to be free. + /// If the file exists, it is checked if it is in use by any backup. + /// + /// The path to check + /// true if the path is in use, false otherwise public static bool IsDatabasePathInUse(string path) { var file = System.IO.Path.Combine(GetDefaultStorageFolderWithDebugSupport(CONFIG_FILE), CONFIG_FILE); if (!System.IO.File.Exists(file)) return false; - return Newtonsoft.Json.JsonConvert.DeserializeObject>(System.IO.File.ReadAllText(file, System.Text.Encoding.UTF8)).Any(x => string.Equals(path, x.Databasepath, Library.Utility.Utility.ClientFilenameStringComparison)); + return (Newtonsoft.Json.JsonConvert.DeserializeObject>(System.IO.File.ReadAllText(file, System.Text.Encoding.UTF8)) ?? []) + .Any(x => string.Equals(path, x.Databasepath, Library.Utility.Utility.ClientFilenameStringComparison)); } } } diff --git a/Duplicati/Library/RestAPI/SpecialFolders.cs b/Duplicati/Library/RestAPI/SpecialFolders.cs index e60b2a05b..180810e77 100644 --- a/Duplicati/Library/RestAPI/SpecialFolders.cs +++ b/Duplicati/Library/RestAPI/SpecialFolders.cs @@ -133,7 +133,7 @@ namespace Duplicati.Server try { // PInvoke to get the download folder - TryAdd(lst, SHGetFolder.DownloadFolder, "%MY_DOWNLOADS%", "Downloads"); + TryAdd(lst, Library.Utility.SHGetFolder.DownloadFolder, "%MY_DOWNLOADS%", "Downloads"); } catch { @@ -208,49 +208,6 @@ namespace Duplicati.Server return result; } - - /// - /// Wrapper for SHGetKnownFolderPath to get the download folder on Windows - /// - [SupportedOSPlatform("windows")] - private static class SHGetFolder - { - /// - /// Gets the download folder path - /// - public static string DownloadFolder => GetKnownFolderPath(new Guid("374DE290-123F-4565-9164-39C4925E467B")); - - /// - /// SHGetKnownFolderPath function to get the folder path - /// - /// The folder GUID - /// Get folder flags - /// The access token - /// The folder path - /// The HRESULT error code - [DllImport("Shell32.dll", CharSet = CharSet.Unicode, SetLastError = true)] - private static extern int SHGetKnownFolderPath( - [MarshalAs(UnmanagedType.LPStruct)] Guid rfid, - uint dwFlags, - IntPtr hToken, - out IntPtr ppszPath); - - /// - /// Gets the folder path using SHGetKnownFolderPath - /// - /// The folder GUID - /// The folder path - private static string GetKnownFolderPath(Guid folderGuid) - { - var result = SHGetKnownFolderPath(folderGuid, 0, IntPtr.Zero, out var outPath); - if (result != 0) - Marshal.ThrowExceptionForHR(result); // Throws an exception for the HRESULT error code - - var path = Marshal.PtrToStringUni(outPath); - Marshal.FreeCoTaskMem(outPath); // Free the memory allocated by SHGetKnownFolderPath - return path; - } - } } } diff --git a/Duplicati/Library/Utility/SHGetFolder.cs b/Duplicati/Library/Utility/SHGetFolder.cs new file mode 100644 index 000000000..3b42fae77 --- /dev/null +++ b/Duplicati/Library/Utility/SHGetFolder.cs @@ -0,0 +1,104 @@ +using System; +using System.IO; +using System.Runtime.InteropServices; +using System.Runtime.Versioning; + +#nullable enable + +namespace Duplicati.Library.Utility; +/// +/// Wrapper for SHGetKnownFolderPath to get the download folder on Windows +/// +/// List of GUIDs: https://learn.microsoft.com/en-us/windows/win32/shell/knownfolderid +[SupportedOSPlatform("windows")] +public static class SHGetFolder +{ + /// + /// Gets the download folder path + /// + public static string? DownloadFolder => GetKnownFolderPath(new Guid("374DE290-123F-4565-9164-39C4925E467B")); + + /// + /// Gets the user profiles folder path + /// + public static string? UserProfilesFolder + { + get + { + try + { + // Prefered way, works on Windows Vista and later + var shPath = GetKnownFolderPath(new Guid("0762D272-C50A-4BB0-A382-697DCD729B80")); + if (!string.IsNullOrWhiteSpace(shPath) && Path.IsPathRooted(shPath) && Directory.Exists(shPath)) + return shPath; + } + catch + { + } + + try + { + // Fallback to "{systemDrive}\\Users" + var envPath = Environment.ExpandEnvironmentVariables("%SystemDrive%\\Users"); + if (!string.IsNullOrWhiteSpace(envPath) && Path.IsPathRooted(envPath) && Directory.Exists(envPath)) + return envPath; + } + catch + { + } + + try + { + // Fallback to "{systemDrive}\\Users", method 2 + var sysPath = Environment.GetFolderPath(Environment.SpecialFolder.System); + if (!string.IsNullOrWhiteSpace(sysPath) && Path.IsPathRooted(sysPath) && Directory.Exists(sysPath)) + { + var sysDir = Path.GetPathRoot(sysPath); + if (!string.IsNullOrWhiteSpace(sysDir)) + return Path.Combine(sysDir, "Users"); + } + } + catch + { + } + + // Most likely, this is the correct path + var path = "C:\\Users"; + if (Directory.Exists(path)) + return path; + + return null; + } + } + + /// + /// SHGetKnownFolderPath function to get the folder path + /// + /// The folder GUID + /// Get folder flags + /// The access token + /// The folder path + /// The HRESULT error code + [DllImport("Shell32.dll", CharSet = CharSet.Unicode, SetLastError = true)] + private static extern int SHGetKnownFolderPath( + [MarshalAs(UnmanagedType.LPStruct)] Guid rfid, + uint dwFlags, + IntPtr hToken, + out IntPtr ppszPath); + + /// + /// Gets the folder path using SHGetKnownFolderPath + /// + /// The folder GUID + /// The folder path + private static string? GetKnownFolderPath(Guid folderGuid) + { + var result = SHGetKnownFolderPath(folderGuid, 0, IntPtr.Zero, out var outPath); + if (result != 0) + Marshal.ThrowExceptionForHR(result); // Throws an exception for the HRESULT error code + + var path = Marshal.PtrToStringUni(outPath); + Marshal.FreeCoTaskMem(outPath); // Free the memory allocated by SHGetKnownFolderPath + return path; + } +} diff --git a/Duplicati/Server/Program.cs b/Duplicati/Server/Program.cs index 4a23e19f2..8565ce220 100644 --- a/Duplicati/Server/Program.cs +++ b/Duplicati/Server/Program.cs @@ -603,6 +603,25 @@ namespace Duplicati.Server ); } + if (OperatingSystem.IsWindows() && DataFolder.StartsWith(Environment.GetFolderPath(Environment.SpecialFolder.Windows), StringComparison.OrdinalIgnoreCase)) + { + DataConnection.RegisterNotification( + Serialization.NotificationType.Warning, + "Incorrect storage folder", + "The configuration is stored in the Windows folder, which will be deleted on Windows updates.", + null, + null, + "config:issue:windows-folder-used", + null, + "UnencryptedDatabase", + null, + (self, all) => + { + return all.FirstOrDefault(x => x.Action == "config:issue:windows-folder-used") ?? self; + } + ); + } + DataConnection.ApplicationSettings.LastConfigIssueCheckVersion = UpdaterManager.SelfVersion.Version; } } From 805d3b1c31d1643c57e7464a68c768a19ffe12de Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 27 Jan 2025 11:51:03 +0100 Subject: [PATCH 030/362] Added special handling of Linux to avoid storing data in the root of the filesystem --- Duplicati/Library/Main/DatabaseLocator.cs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Duplicati/Library/Main/DatabaseLocator.cs b/Duplicati/Library/Main/DatabaseLocator.cs index fc7e934f0..4128bee94 100644 --- a/Duplicati/Library/Main/DatabaseLocator.cs +++ b/Duplicati/Library/Main/DatabaseLocator.cs @@ -166,6 +166,22 @@ namespace Duplicati.Library.Main folder = configfolder; } + if (OperatingSystem.IsLinux() && (folder == $"/{appName}" || folder == appName)) + { + // Special handling for Linux with no home folder: + // - Older versions use / + // - but new versions use /var/lib/ + var libfolder = System.IO.Path.Combine("var", "lib", appName); + + var curfile = System.IO.Path.Combine(libfolder, targetfilename); + var prevfile = System.IO.Path.Combine(folder, targetfilename); + + // If the old file exists, and not the new file, we use the old + // Otherwise we use the new location + if (System.IO.File.Exists(curfile) || !System.IO.File.Exists(prevfile)) + folder = libfolder; + } + return folder; } From db3d74e34c7cd5595f3e93645981ea47da3bb5a0 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 27 Jan 2025 12:59:51 +0100 Subject: [PATCH 031/362] Moved warning emit call as the server may not be fully configured yet --- Duplicati/Server/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Server/Program.cs b/Duplicati/Server/Program.cs index 8565ce220..9e482222c 100644 --- a/Duplicati/Server/Program.cs +++ b/Duplicati/Server/Program.cs @@ -270,7 +270,6 @@ namespace Duplicati.Server StartOrStopUsageReporter(); AdjustApplicationSettings(commandlineOptions); - EmitWarningsForConfigurationIssues(commandlineOptions); ApplicationExitEvent = new System.Threading.ManualResetEvent(false); @@ -297,6 +296,7 @@ namespace Duplicati.Server DataConnection.ReWriteAllFieldsIfEncryptionChanged(); DataConnection.SetPreloadSettingsIfChanged(preloadDbSettings); + EmitWarningsForConfigurationIssues(commandlineOptions); Library.Logging.Log.WriteInformationMessage(LOGTAG, "ServerStarted", Strings.Program.ServerStarted(DuplicatiWebserver.Port)); logMessageToConsole(Strings.Program.ServerStarted(DuplicatiWebserver.Port)); From c1d89e32e963656d2cc9de598e344a49a259b444 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 27 Jan 2025 13:09:35 +0100 Subject: [PATCH 032/362] Guard against name clashes for Windows folder --- Duplicati/Library/Main/DatabaseLocator.cs | 722 +++++++++++----------- 1 file changed, 361 insertions(+), 361 deletions(-) diff --git a/Duplicati/Library/Main/DatabaseLocator.cs b/Duplicati/Library/Main/DatabaseLocator.cs index 4128bee94..73065132f 100644 --- a/Duplicati/Library/Main/DatabaseLocator.cs +++ b/Duplicati/Library/Main/DatabaseLocator.cs @@ -1,361 +1,361 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - -using System; -using System.Linq; -using System.Collections.Generic; - -#nullable enable - -namespace Duplicati.Library.Main -{ - public static class DatabaseLocator - { - /// - /// The log tag for this class - /// - public static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(DatabaseLocator)); - - /// - /// The entry for a backend configuration, used for serialization - /// - public class BackendEntry - { - /// - /// The type of the backend - /// - public required string Type; - /// - /// The server/hostname of the backend - /// - public required string Server; - /// - /// The path of the backend - /// - public required string Path; - /// - /// The prefix of the backup - /// - public required string Prefix; - /// - /// The username of the backend - /// - public required string? Username; - /// - /// The port of the backend - /// - public required int Port; - /// - /// The path to the database - /// - public required string Databasepath; - /// - /// The path to the parameter file - /// - public required string? ParameterFile; - } - - /// - /// The filename of the file with database configurations - /// - private const string CONFIG_FILE = "dbconfig.json"; - - /// - /// Finds a default storage folder, using the operating system specific locations. - /// The targetfilename is used to detect locations that are used in previous versions. - /// If the targetfilename is found in an old location, but not the current, the old location is used. - /// If running with DEBUG defined, the storage folder is placed in the same folder as the executable. - /// Note that the folder is not created, only the path is returned. - /// - /// The filename to look for - /// The name of the application - /// The default storage folder - public static string GetDefaultStorageFolderWithDebugSupport(string targetfilename, string appName = "Duplicati") - { -#if DEBUG - return System.IO.Path.GetDirectoryName(typeof(DatabaseLocator).Assembly.Location) ?? string.Empty; -#else - return GetDefaultStorageFolderDirect(targetfilename, appName); -#endif - } - - /// - /// Finds a default storage folder, using the operating system specific locations. - /// The targetfilename is used to detect locations that are used in previous versions. - /// If the targetfilename is found in an old location, but not the current, the old location is used. - /// Note that the folder is not created, only the path is returned. - /// - /// The filename to look for - /// The name of the application - /// The default storage folder - public static string GetDefaultStorageFolderDirect(string targetfilename, string appName = "Duplicati") - { - //Normal mode uses the systems "(Local) Application Data" folder - // %LOCALAPPDATA% on Windows, ~/.config on Linux - var folder = System.IO.Path.Combine(System.Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), appName); - - if (OperatingSystem.IsWindows()) - { - // Special handling for Windows: - // - Older versions use %APPDATA% - // - New versions use %LOCALAPPDATA% - // - And prevent using C:\Windows\ - var newlocation = System.IO.Path.Combine(System.Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), appName); - - var folderOrder = new List() { - folder, - newlocation - }; - - // If %LOCALAPPDATA% is inside the Windows folder, prefer a LocalService folder instead - var windowsFolder = Environment.GetFolderPath(Environment.SpecialFolder.Windows); - if (newlocation.StartsWith(windowsFolder, StringComparison.OrdinalIgnoreCase)) - { - var userProfilesFolder = Library.Utility.SHGetFolder.UserProfilesFolder; - if (!string.IsNullOrWhiteSpace(userProfilesFolder)) - folderOrder.Add(System.IO.Path.Combine(userProfilesFolder, "LocalService", appName)); - } - - // Prefer the most recent location - var matches = folderOrder.AsEnumerable() - .Reverse() - .Select(x => System.IO.Path.Combine(x, targetfilename)) - .Where(System.IO.File.Exists) - .ToList(); - - // Use the most recent location found with content - // If none are found, use the most recent location - folder = matches.FirstOrDefault() ?? folderOrder.Last(); - - // Emit a warning if the database is stored in the Windows folder - if (folder.StartsWith(windowsFolder, StringComparison.OrdinalIgnoreCase)) - Logging.Log.WriteWarningMessage(LOGTAG, "DatabaseInWindowsFolder", null, "The database is stored in the Windows folder, this is not recommended as it will be deleted on Windows upgrades."); - } - - if (OperatingSystem.IsMacOS()) - { - // Special handling for MacOS: - // - Older versions use ~/.config/ - // - but new versions use ~/Library/Application\ Support/ - var configfolder = System.IO.Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".config", appName); - - var prevfile = System.IO.Path.Combine(configfolder, targetfilename); - var curfile = System.IO.Path.Combine(folder, targetfilename); - - // If the old file exists, and not the new file, we use the old - // Otherwise we use the new location - if (!System.IO.File.Exists(curfile) && System.IO.File.Exists(prevfile)) - folder = configfolder; - } - - if (OperatingSystem.IsLinux() && (folder == $"/{appName}" || folder == appName)) - { - // Special handling for Linux with no home folder: - // - Older versions use / - // - but new versions use /var/lib/ - var libfolder = System.IO.Path.Combine("var", "lib", appName); - - var curfile = System.IO.Path.Combine(libfolder, targetfilename); - var prevfile = System.IO.Path.Combine(folder, targetfilename); - - // If the old file exists, and not the new file, we use the old - // Otherwise we use the new location - if (System.IO.File.Exists(curfile) || !System.IO.File.Exists(prevfile)) - folder = libfolder; - } - - return folder; - } - - /// - /// Gets the database path for a given backend url, using the JSON configuration file lookup. - /// If is true, a new database is created if none is found. - /// Otherwise, null is returned if no database is found. - /// - /// The backend url - /// The options to use - /// If true, a new database is created if none is found - /// If true, any username is accepted - /// The database path or null - public static string? GetDatabasePathForCLI(string backend, Options? options, bool autoCreate = true, bool anyUsername = false) - { - options ??= new Options(new Dictionary()); - - if (!string.IsNullOrEmpty(options.Dbpath)) - return options.Dbpath; - - var folder = GetDefaultStorageFolderWithDebugSupport(CONFIG_FILE); - - var file = System.IO.Path.Combine(folder, CONFIG_FILE); - List configs; - if (!System.IO.File.Exists(file)) - configs = new List(); - else - configs = Newtonsoft.Json.JsonConvert.DeserializeObject>(System.IO.File.ReadAllText(file, System.Text.Encoding.UTF8)) - ?? new List(); - - var uri = new Library.Utility.Uri(backend); - var server = uri.Host; - var path = uri.Path; - var type = uri.Scheme; - var port = uri.Port; - var username = uri.Username; - var prefix = options.Prefix; - - if (username == null || uri.Password == null) - { - var sopts = DynamicLoader.BackendLoader.GetSupportedCommands(backend); - var ropts = new Dictionary(options.RawOptions); - foreach (var k in uri.QueryParameters.AllKeys) - if (k != null) - ropts[k] = uri.QueryParameters[k]; - - if (sopts != null) - { - foreach (var o in sopts) - { - if (username == null && o.Aliases != null && o.Aliases.Contains("auth-username", StringComparer.OrdinalIgnoreCase) && ropts.ContainsKey(o.Name)) - username = ropts[o.Name]; - } - - foreach (var o in sopts) - { - if (username == null && o.Name.Equals("auth-username", StringComparison.OrdinalIgnoreCase) && ropts.ContainsKey("auth-username")) - username = ropts["auth-username"]; - } - } - } - - //Now find the one that matches :) - var matches = (from n in configs - where - n.Type == type && - n.Username == username && - n.Port == port && - n.Server == server && - n.Path == path && - n.Prefix == prefix - select n).ToList(); - - if (matches.Count > 1) - throw new Duplicati.Library.Interface.UserInformationException(string.Format("Multiple sources found for: {0}", backend), "MultipleLocalDatabaseSourcesFound"); - - // Re-select - if (matches.Count == 0 && anyUsername && string.IsNullOrEmpty(username)) - { - matches = (from n in configs - where - n.Type == type && - n.Port == port && - n.Server == server && - n.Path == path && - n.Prefix == prefix - select n).ToList(); - - if (matches.Count > 1) - throw new Duplicati.Library.Interface.UserInformationException(String.Format("Multiple sources found for \"{0}\", try supplying --{1}", backend, "auth-username"), "MultipleLocalDatabaseSourcesFound"); - } - - if (matches.Count == 0 && !autoCreate) - return null; - - if (matches.Count == 0) - { - var backupname = options.BackupName; - if (string.IsNullOrEmpty(backupname) || backupname == Options.DefaultBackupName) - backupname = GenerateRandomName(); - else - { - foreach (var c in System.IO.Path.GetInvalidFileNameChars()) - backupname = backupname.Replace(c.ToString(), ""); - } - - var newpath = System.IO.Path.Combine(folder, backupname + ".sqlite"); - int max_tries = 100; - while (System.IO.File.Exists(newpath) && max_tries-- > 0) - newpath = System.IO.Path.Combine(folder, GenerateRandomName()); - - if (System.IO.File.Exists(newpath)) - throw new Duplicati.Library.Interface.UserInformationException("Unable to find a unique name for the database, please use --dbpath", "CannotCreateRandomName"); - - //Create a new one, add it to the list, and save it - configs.Add(new BackendEntry() - { - Type = type, - Server = server, - Path = path, - Prefix = prefix, - Username = username, - //Passwordhash = password, - Port = port, - Databasepath = newpath, - ParameterFile = null - }); - - if (!System.IO.Directory.Exists(folder)) - System.IO.Directory.CreateDirectory(folder); - - var settings = new Newtonsoft.Json.JsonSerializerSettings(); - settings.Formatting = Newtonsoft.Json.Formatting.Indented; - System.IO.File.WriteAllText(file, Newtonsoft.Json.JsonConvert.SerializeObject(configs, settings), System.Text.Encoding.UTF8); - - return newpath; - } - else - { - return matches[0].Databasepath; - } - } - - /// - /// Generates a random name for a database - /// - /// A random name - public static string GenerateRandomName() - { - var rnd = new Random(); - - var backupName = new System.Text.StringBuilder(); - for (var i = 0; i < 10; i++) - backupName.Append((char)rnd.Next('A', 'Z' + 1)); - - return backupName.ToString(); - } - - /// - /// Checks if a database path is in use by any backup. - /// If the file does not exist, it is assumed to be free. - /// If the file exists, it is checked if it is in use by any backup. - /// - /// The path to check - /// true if the path is in use, false otherwise - public static bool IsDatabasePathInUse(string path) - { - var file = System.IO.Path.Combine(GetDefaultStorageFolderWithDebugSupport(CONFIG_FILE), CONFIG_FILE); - if (!System.IO.File.Exists(file)) - return false; - - return (Newtonsoft.Json.JsonConvert.DeserializeObject>(System.IO.File.ReadAllText(file, System.Text.Encoding.UTF8)) ?? []) - .Any(x => string.Equals(path, x.Databasepath, Library.Utility.Utility.ClientFilenameStringComparison)); - } - } -} - +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using System; +using System.Linq; +using System.Collections.Generic; + +#nullable enable + +namespace Duplicati.Library.Main +{ + public static class DatabaseLocator + { + /// + /// The log tag for this class + /// + public static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(DatabaseLocator)); + + /// + /// The entry for a backend configuration, used for serialization + /// + public class BackendEntry + { + /// + /// The type of the backend + /// + public required string Type; + /// + /// The server/hostname of the backend + /// + public required string Server; + /// + /// The path of the backend + /// + public required string Path; + /// + /// The prefix of the backup + /// + public required string Prefix; + /// + /// The username of the backend + /// + public required string? Username; + /// + /// The port of the backend + /// + public required int Port; + /// + /// The path to the database + /// + public required string Databasepath; + /// + /// The path to the parameter file + /// + public required string? ParameterFile; + } + + /// + /// The filename of the file with database configurations + /// + private const string CONFIG_FILE = "dbconfig.json"; + + /// + /// Finds a default storage folder, using the operating system specific locations. + /// The targetfilename is used to detect locations that are used in previous versions. + /// If the targetfilename is found in an old location, but not the current, the old location is used. + /// If running with DEBUG defined, the storage folder is placed in the same folder as the executable. + /// Note that the folder is not created, only the path is returned. + /// + /// The filename to look for + /// The name of the application + /// The default storage folder + public static string GetDefaultStorageFolderWithDebugSupport(string targetfilename, string appName = "Duplicati") + { +#if DEBUG + return System.IO.Path.GetDirectoryName(typeof(DatabaseLocator).Assembly.Location) ?? string.Empty; +#else + return GetDefaultStorageFolderDirect(targetfilename, appName); +#endif + } + + /// + /// Finds a default storage folder, using the operating system specific locations. + /// The targetfilename is used to detect locations that are used in previous versions. + /// If the targetfilename is found in an old location, but not the current, the old location is used. + /// Note that the folder is not created, only the path is returned. + /// + /// The filename to look for + /// The name of the application + /// The default storage folder + public static string GetDefaultStorageFolderDirect(string targetfilename, string appName = "Duplicati") + { + //Normal mode uses the systems "(Local) Application Data" folder + // %LOCALAPPDATA% on Windows, ~/.config on Linux + var folder = System.IO.Path.Combine(System.Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), appName); + + if (OperatingSystem.IsWindows()) + { + // Special handling for Windows: + // - Older versions use %APPDATA% + // - New versions use %LOCALAPPDATA% + // - And prevent using C:\Windows\ + var newlocation = System.IO.Path.Combine(System.Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), appName); + + var folderOrder = new List() { + folder, + newlocation + }; + + // If %LOCALAPPDATA% is inside the Windows folder, prefer a LocalService folder instead + var windowsFolder = Library.Common.IO.Util.AppendDirSeparator(Environment.GetFolderPath(Environment.SpecialFolder.Windows)); + if (newlocation.StartsWith(windowsFolder, StringComparison.OrdinalIgnoreCase)) + { + var userProfilesFolder = Library.Utility.SHGetFolder.UserProfilesFolder; + if (!string.IsNullOrWhiteSpace(userProfilesFolder)) + folderOrder.Add(System.IO.Path.Combine(userProfilesFolder, "LocalService", appName)); + } + + // Prefer the most recent location + var matches = folderOrder.AsEnumerable() + .Reverse() + .Select(x => System.IO.Path.Combine(x, targetfilename)) + .Where(System.IO.File.Exists) + .ToList(); + + // Use the most recent location found with content + // If none are found, use the most recent location + folder = matches.FirstOrDefault() ?? folderOrder.Last(); + + // Emit a warning if the database is stored in the Windows folder + if (folder.StartsWith(windowsFolder, StringComparison.OrdinalIgnoreCase)) + Logging.Log.WriteWarningMessage(LOGTAG, "DatabaseInWindowsFolder", null, "The database is stored in the Windows folder, this is not recommended as it will be deleted on Windows upgrades."); + } + + if (OperatingSystem.IsMacOS()) + { + // Special handling for MacOS: + // - Older versions use ~/.config/ + // - but new versions use ~/Library/Application\ Support/ + var configfolder = System.IO.Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".config", appName); + + var prevfile = System.IO.Path.Combine(configfolder, targetfilename); + var curfile = System.IO.Path.Combine(folder, targetfilename); + + // If the old file exists, and not the new file, we use the old + // Otherwise we use the new location + if (!System.IO.File.Exists(curfile) && System.IO.File.Exists(prevfile)) + folder = configfolder; + } + + if (OperatingSystem.IsLinux() && (folder == $"/{appName}" || folder == appName)) + { + // Special handling for Linux with no home folder: + // - Older versions use / + // - but new versions use /var/lib/ + var libfolder = System.IO.Path.Combine("var", "lib", appName); + + var curfile = System.IO.Path.Combine(libfolder, targetfilename); + var prevfile = System.IO.Path.Combine(folder, targetfilename); + + // If the old file exists, and not the new file, we use the old + // Otherwise we use the new location + if (System.IO.File.Exists(curfile) || !System.IO.File.Exists(prevfile)) + folder = libfolder; + } + + return folder; + } + + /// + /// Gets the database path for a given backend url, using the JSON configuration file lookup. + /// If is true, a new database is created if none is found. + /// Otherwise, null is returned if no database is found. + /// + /// The backend url + /// The options to use + /// If true, a new database is created if none is found + /// If true, any username is accepted + /// The database path or null + public static string? GetDatabasePathForCLI(string backend, Options? options, bool autoCreate = true, bool anyUsername = false) + { + options ??= new Options(new Dictionary()); + + if (!string.IsNullOrEmpty(options.Dbpath)) + return options.Dbpath; + + var folder = GetDefaultStorageFolderWithDebugSupport(CONFIG_FILE); + + var file = System.IO.Path.Combine(folder, CONFIG_FILE); + List configs; + if (!System.IO.File.Exists(file)) + configs = new List(); + else + configs = Newtonsoft.Json.JsonConvert.DeserializeObject>(System.IO.File.ReadAllText(file, System.Text.Encoding.UTF8)) + ?? new List(); + + var uri = new Library.Utility.Uri(backend); + var server = uri.Host; + var path = uri.Path; + var type = uri.Scheme; + var port = uri.Port; + var username = uri.Username; + var prefix = options.Prefix; + + if (username == null || uri.Password == null) + { + var sopts = DynamicLoader.BackendLoader.GetSupportedCommands(backend); + var ropts = new Dictionary(options.RawOptions); + foreach (var k in uri.QueryParameters.AllKeys) + if (k != null) + ropts[k] = uri.QueryParameters[k]; + + if (sopts != null) + { + foreach (var o in sopts) + { + if (username == null && o.Aliases != null && o.Aliases.Contains("auth-username", StringComparer.OrdinalIgnoreCase) && ropts.ContainsKey(o.Name)) + username = ropts[o.Name]; + } + + foreach (var o in sopts) + { + if (username == null && o.Name.Equals("auth-username", StringComparison.OrdinalIgnoreCase) && ropts.ContainsKey("auth-username")) + username = ropts["auth-username"]; + } + } + } + + //Now find the one that matches :) + var matches = (from n in configs + where + n.Type == type && + n.Username == username && + n.Port == port && + n.Server == server && + n.Path == path && + n.Prefix == prefix + select n).ToList(); + + if (matches.Count > 1) + throw new Duplicati.Library.Interface.UserInformationException(string.Format("Multiple sources found for: {0}", backend), "MultipleLocalDatabaseSourcesFound"); + + // Re-select + if (matches.Count == 0 && anyUsername && string.IsNullOrEmpty(username)) + { + matches = (from n in configs + where + n.Type == type && + n.Port == port && + n.Server == server && + n.Path == path && + n.Prefix == prefix + select n).ToList(); + + if (matches.Count > 1) + throw new Duplicati.Library.Interface.UserInformationException(String.Format("Multiple sources found for \"{0}\", try supplying --{1}", backend, "auth-username"), "MultipleLocalDatabaseSourcesFound"); + } + + if (matches.Count == 0 && !autoCreate) + return null; + + if (matches.Count == 0) + { + var backupname = options.BackupName; + if (string.IsNullOrEmpty(backupname) || backupname == Options.DefaultBackupName) + backupname = GenerateRandomName(); + else + { + foreach (var c in System.IO.Path.GetInvalidFileNameChars()) + backupname = backupname.Replace(c.ToString(), ""); + } + + var newpath = System.IO.Path.Combine(folder, backupname + ".sqlite"); + int max_tries = 100; + while (System.IO.File.Exists(newpath) && max_tries-- > 0) + newpath = System.IO.Path.Combine(folder, GenerateRandomName()); + + if (System.IO.File.Exists(newpath)) + throw new Duplicati.Library.Interface.UserInformationException("Unable to find a unique name for the database, please use --dbpath", "CannotCreateRandomName"); + + //Create a new one, add it to the list, and save it + configs.Add(new BackendEntry() + { + Type = type, + Server = server, + Path = path, + Prefix = prefix, + Username = username, + //Passwordhash = password, + Port = port, + Databasepath = newpath, + ParameterFile = null + }); + + if (!System.IO.Directory.Exists(folder)) + System.IO.Directory.CreateDirectory(folder); + + var settings = new Newtonsoft.Json.JsonSerializerSettings(); + settings.Formatting = Newtonsoft.Json.Formatting.Indented; + System.IO.File.WriteAllText(file, Newtonsoft.Json.JsonConvert.SerializeObject(configs, settings), System.Text.Encoding.UTF8); + + return newpath; + } + else + { + return matches[0].Databasepath; + } + } + + /// + /// Generates a random name for a database + /// + /// A random name + public static string GenerateRandomName() + { + var rnd = new Random(); + + var backupName = new System.Text.StringBuilder(); + for (var i = 0; i < 10; i++) + backupName.Append((char)rnd.Next('A', 'Z' + 1)); + + return backupName.ToString(); + } + + /// + /// Checks if a database path is in use by any backup. + /// If the file does not exist, it is assumed to be free. + /// If the file exists, it is checked if it is in use by any backup. + /// + /// The path to check + /// true if the path is in use, false otherwise + public static bool IsDatabasePathInUse(string path) + { + var file = System.IO.Path.Combine(GetDefaultStorageFolderWithDebugSupport(CONFIG_FILE), CONFIG_FILE); + if (!System.IO.File.Exists(file)) + return false; + + return (Newtonsoft.Json.JsonConvert.DeserializeObject>(System.IO.File.ReadAllText(file, System.Text.Encoding.UTF8)) ?? []) + .Any(x => string.Equals(path, x.Databasepath, Library.Utility.Utility.ClientFilenameStringComparison)); + } + } +} + From b025c9af015fbe58fbe7711425f86160ef8ba4a3 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 27 Jan 2025 21:25:18 +0100 Subject: [PATCH 033/362] Added passing all of the commandline options to the inner run method --- Tools/RemoteSynchronization/Program.cs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 4972352ea..7e4d6c8dc 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2025, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a @@ -24,6 +24,7 @@ using Duplicati.Library.Interface; using System; using System.Collections.Generic; using System.CommandLine; +using System.CommandLine.Invocation; using System.IO; using System.Linq; using System.Threading; @@ -43,12 +44,19 @@ namespace RemoteSynchronization root_cmd.AddArgument(src_arg); root_cmd.AddArgument(dst_arg); - root_cmd.SetHandler(Run, src_arg, dst_arg); + root_cmd.SetHandler((InvocationContext ctx) => + { + var parsed = ctx.ParseResult; + + Dictionary options = parsed.CommandResult.Command.Options.ToDictionary(x => x.Name, x => parsed.GetValueForOption(x)); + + Run(parsed.GetValueForArgument(src_arg), parsed.GetValueForArgument(dst_arg), options).Wait(); + }); return await root_cmd.InvokeAsync(args); } - private static async Task Run(string src, string dst) + private static async Task Run(string src, string dst, Dictionary options) { var options = new Dictionary(); From 6993969dde1147d70bdde66dc71f01028c3411d2 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 27 Jan 2025 21:26:06 +0100 Subject: [PATCH 034/362] Implement dry run option in the Remote Synchronization Tool --- Tools/RemoteSynchronization/Program.cs | 58 +++++++++++++++++++------- 1 file changed, 42 insertions(+), 16 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 7e4d6c8dc..471d2fac7 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2025, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a @@ -39,10 +39,12 @@ namespace RemoteSynchronization { var src_arg = new Argument(name: "backend_src", description: "The source backend string"); var dst_arg = new Argument(name: "backend_dst", description: "The destination backend string"); + var dry_run_opt = new Option(aliases: ["--dry-run", "-d"], description: "Do not actually write or delete files"); var root_cmd = new RootCommand("Remote Synchronization Tool"); root_cmd.AddArgument(src_arg); root_cmd.AddArgument(dst_arg); + root_cmd.AddOption(dry_run_opt); root_cmd.SetHandler((InvocationContext ctx) => { @@ -58,20 +60,24 @@ namespace RemoteSynchronization private static async Task Run(string src, string dst, Dictionary options) { - var options = new Dictionary(); + var dry_run = options["dry-run"] as bool? ?? false; + var duplicati_options = new Dictionary() + { + ["dry-run"] = dry_run.ToString() + }; - using var b1 = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(src, options); + using var b1 = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(src, duplicati_options); var b1s = b1 as IStreamingBackend; System.Diagnostics.Debug.Assert(b1s != null); - using var b2 = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(dst, options); + using var b2 = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(dst, duplicati_options); var b2s = b2 as IStreamingBackend; System.Diagnostics.Debug.Assert(b2s != null); var (to_copy, to_delete) = PrepareFileLists(b1s, b2s); - var deleted = await DeleteAsync(b2s, to_delete); + var deleted = await DeleteAsync(b2s, to_delete, dry_run); Console.WriteLine($"Deleted {deleted} files from {dst}"); - var copied = await CopyAsync(b1s, b2s, to_copy); + var copied = await CopyAsync(b1s, b2s, to_copy, dry_run); Console.WriteLine($"Copied {copied} files from {src} to {dst}"); return 0; @@ -82,15 +88,14 @@ namespace RemoteSynchronization // TODO low memory mode, where things aren't kept in memory. Maybe utilize SQLite? // TODO Force parameter // TODO Progress reporting - // TODO Dry run // TODO Logging - // TODO check database consistency. I.e. have both databases, check that the block, volume, files, etc match up. - // TODO introduce these checks as a post processing step? Especially the database consistency check, as that is often recreated from the index files. + // check database consistency. I.e. have both databases, check that the block, volume, files, etc match up. + // introduce these checks as a post processing step? Especially the database consistency check, as that is often recreated from the index files. // TODO This tool shouldn't handle it, but for convenience, it should support making the seperate call to the regular Duplicati on the destination backend, which alread carries this functionality. // Forcefully synchronize the remote backends - private static async Task CopyAsync(IStreamingBackend b_src, IStreamingBackend b_dst, IEnumerable files) + private static async Task CopyAsync(IStreamingBackend b_src, IStreamingBackend b_dst, IEnumerable files, bool dry_run) { long successful_copies = 0; using var s = new MemoryStream(); @@ -99,7 +104,14 @@ namespace RemoteSynchronization try { await b_src.GetAsync(f.Name, s, CancellationToken.None); - await b_dst.PutAsync(f.Name, s, CancellationToken.None); + if (dry_run) + { + Console.WriteLine($"Would write {s.Length} bytes of {f.Name} to {b_dst.DisplayName}"); + } + else + { + await b_dst.PutAsync(f.Name, s, CancellationToken.None); + } s.SetLength(0); successful_copies++; } @@ -111,14 +123,21 @@ namespace RemoteSynchronization return successful_copies; } - private static async Task DeleteAsync(IStreamingBackend b, IEnumerable files) + private static async Task DeleteAsync(IStreamingBackend b, IEnumerable files, bool dry_run) { long successful_deletes = 0; foreach (var f in files) { try { - await b.DeleteAsync(f.Name, CancellationToken.None); + if (dry_run) + { + Console.WriteLine($"Would delete {f.Name} from {b.DisplayName}"); + } + else + { + await b.DeleteAsync(f.Name, CancellationToken.None); + } successful_deletes++; } catch (Exception e) @@ -178,7 +197,7 @@ namespace RemoteSynchronization return (to_copy, to_delete.Select(x => lookup_dst[x])); } - private static async Task RenameAsync(IStreamingBackend b, IEnumerable files) + private static async Task RenameAsync(IStreamingBackend b, IEnumerable files, bool dry_run) { long successful_renames = 0; string suffix = $"{DateTime.Now:yyyyMMddHHmmss}.old"; @@ -188,8 +207,15 @@ namespace RemoteSynchronization try { await b.GetAsync(f.Name, downloaded, CancellationToken.None); - await b.PutAsync($"{f.Name}.{suffix}", downloaded, CancellationToken.None); - await b.DeleteAsync(f.Name, CancellationToken.None); + if (dry_run) + { + Console.WriteLine($"Would rename {f.Name} to {f.Name}.{suffix} by deleting and re-uploading {downloaded.Length} bytes to {b.DisplayName}"); + } + else + { + await b.PutAsync($"{f.Name}.{suffix}", downloaded, CancellationToken.None); + await b.DeleteAsync(f.Name, CancellationToken.None); + } downloaded.SetLength(0); successful_renames++; } From a819f34b37a0fc13b88753f47d4a181b39f413fa Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 27 Jan 2025 22:42:40 +0100 Subject: [PATCH 035/362] Improved logging for WindowsServer --- Duplicati/Server/Program.cs | 4 ++- Duplicati/Service/Runner.cs | 51 ++++++++++++++++++++----------------- 2 files changed, 31 insertions(+), 24 deletions(-) diff --git a/Duplicati/Server/Program.cs b/Duplicati/Server/Program.cs index 9e482222c..ef78784d4 100644 --- a/Duplicati/Server/Program.cs +++ b/Duplicati/Server/Program.cs @@ -330,6 +330,7 @@ namespace Duplicati.Server catch (SingleInstance.MultipleInstanceException mex) { crashed = true; + Log.WriteErrorMessage(LOGTAG, "MultipleInstanceError", mex, Strings.Program.ServerCrashed(mex.Message)); System.Diagnostics.Trace.WriteLine(Strings.Program.SeriousError(mex.ToString())); if (!writeToConsoleOnException) throw; @@ -339,6 +340,7 @@ namespace Duplicati.Server catch (Exception ex) { crashed = true; + Log.WriteErrorMessage(LOGTAG, "ServerCrashed", ex, Strings.Program.ServerCrashed(ex.Message)); System.Diagnostics.Trace.WriteLine(Strings.Program.SeriousError(ex.ToString())); if (writeToConsoleOnException) { @@ -350,7 +352,7 @@ namespace Duplicati.Server } finally { - Library.Logging.Log.WriteInformationMessage(LOGTAG, "ServerStopping", Strings.Program.ServerStopping); + Log.WriteInformationMessage(LOGTAG, "ServerStopping", Strings.Program.ServerStopping); var steps = new Action[] { () => StatusEventNotifyer.SignalNewEvent(), diff --git a/Duplicati/Service/Runner.cs b/Duplicati/Service/Runner.cs index c6327d822..a406fe1a7 100644 --- a/Duplicati/Service/Runner.cs +++ b/Duplicati/Service/Runner.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -117,6 +117,11 @@ namespace Duplicati.Service PingProcess(); } } + + if (m_process.ExitCode != 0) + m_reportMessage(string.Format("Process has exited with code {0}", m_process.ExitCode), true); + else if (!m_terminate) + m_reportMessage("Process has exited without an error code", true); } catch (Exception ex) { @@ -131,11 +136,11 @@ namespace Duplicati.Service m_terminate = true; } } - - // Throttle restarts - if (!m_terminate) - System.Threading.Thread.Sleep(TimeSpan.FromSeconds(10)); } + + // Throttle restarts + if (!m_terminate) + System.Threading.Thread.Sleep(TimeSpan.FromSeconds(10)); } } finally From 27f0bb4ea093a99594b712241adcaada6d2ace9a Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 27 Jan 2025 22:51:45 +0100 Subject: [PATCH 036/362] Unified code to find the data folder. The `machineid.txt` and `installation.txt` files are now default located in same folder where the database is default located. --- Duplicati/Agent/Settings.cs | 2 +- Duplicati/CommandLine/CLI/Commands.cs | 8 +- Duplicati/CommandLine/ServerUtil/Settings.cs | 2 +- .../Library/AutoUpdater/DatabaseLocator.cs | 144 ++++++++++++++++++ .../AutoUpdater/PreloadSettingsLoader.cs | 8 +- Duplicati/Library/AutoUpdater/Program.cs | 2 +- .../Library/AutoUpdater/UpdaterManager.cs | 107 ++++--------- ...tabaseLocator.cs => CLIDatabaseLocator.cs} | 115 +------------- Duplicati/Library/Main/Controller.cs | 2 +- .../Main/Operation/SystemInfoHandler.cs | 40 ++--- .../Library/RestAPI/Database/Connection.cs | 2 +- Duplicati/Library/Utility/SHGetFolder.cs | 2 +- Duplicati/Server/Program.cs | 9 +- .../Endpoints/V1/Backup/BackupGet.cs | 2 +- .../WebserverCore/Endpoints/V1/Backups.cs | 2 +- .../Endpoints/V1/RemoteOperation.cs | 2 +- 16 files changed, 219 insertions(+), 230 deletions(-) create mode 100644 Duplicati/Library/AutoUpdater/DatabaseLocator.cs rename Duplicati/Library/Main/{DatabaseLocator.cs => CLIDatabaseLocator.cs} (63%) diff --git a/Duplicati/Agent/Settings.cs b/Duplicati/Agent/Settings.cs index 759edd389..6785612a8 100644 --- a/Duplicati/Agent/Settings.cs +++ b/Duplicati/Agent/Settings.cs @@ -148,7 +148,7 @@ public sealed record Settings( /// The path to the settings file private static string GetSettingsPath() { - var folder = DatabaseLocator.GetDefaultStorageFolderWithDebugSupport(DefaultSettingsFilename); + var folder = Library.AutoUpdater.DatabaseLocator.GetDefaultStorageFolderWithDebugSupport(DefaultSettingsFilename); if (!Directory.Exists(folder)) Directory.CreateDirectory(folder); diff --git a/Duplicati/CommandLine/CLI/Commands.cs b/Duplicati/CommandLine/CLI/Commands.cs index 7fa32f890..b5834a95d 100644 --- a/Duplicati/CommandLine/CLI/Commands.cs +++ b/Duplicati/CommandLine/CLI/Commands.cs @@ -163,7 +163,7 @@ namespace Duplicati.CommandLine options.TryGetValue("dbpath", out dbpath); if (string.IsNullOrEmpty(dbpath)) { - dbpath = Library.Main.DatabaseLocator.GetDatabasePathForCLI(backend, new Duplicati.Library.Main.Options(options), false, true); + dbpath = Library.Main.CLIDatabaseLocator.GetDatabasePathForCLI(backend, new Library.Main.Options(options), false, true); if (dbpath != null) options["dbpath"] = dbpath; } @@ -366,7 +366,7 @@ namespace Duplicati.CommandLine options.TryGetValue("dbpath", out dbpath); if (string.IsNullOrEmpty(dbpath)) { - dbpath = Library.Main.DatabaseLocator.GetDatabasePathForCLI(backend, new Duplicati.Library.Main.Options(options), false, true); + dbpath = Library.Main.CLIDatabaseLocator.GetDatabasePathForCLI(backend, new Duplicati.Library.Main.Options(options), false, true); if (dbpath != null) options["dbpath"] = dbpath; } @@ -820,7 +820,7 @@ namespace Duplicati.CommandLine if (string.IsNullOrEmpty(dbpath)) { if (args.Count > 0) - dbpath = Library.Main.DatabaseLocator.GetDatabasePathForCLI(args[0], new Duplicati.Library.Main.Options(options), false, true); + dbpath = Library.Main.CLIDatabaseLocator.GetDatabasePathForCLI(args[0], new Duplicati.Library.Main.Options(options), false, true); if (dbpath == null) { @@ -866,7 +866,7 @@ namespace Duplicati.CommandLine options.TryGetValue("dbpath", out dbpath); if (string.IsNullOrEmpty(dbpath)) { - dbpath = Library.Main.DatabaseLocator.GetDatabasePathForCLI(args[0], new Duplicati.Library.Main.Options(options), false, true); + dbpath = Library.Main.CLIDatabaseLocator.GetDatabasePathForCLI(args[0], new Duplicati.Library.Main.Options(options), false, true); if (dbpath != null) options["dbpath"] = dbpath; } diff --git a/Duplicati/CommandLine/ServerUtil/Settings.cs b/Duplicati/CommandLine/ServerUtil/Settings.cs index 6859cad45..6caacc2bf 100644 --- a/Duplicati/CommandLine/ServerUtil/Settings.cs +++ b/Duplicati/CommandLine/ServerUtil/Settings.cs @@ -66,7 +66,7 @@ public sealed record Settings( ); private static string GetDefaultStorageFolder(string filename) { - var folder = DatabaseLocator.GetDefaultStorageFolderWithDebugSupport(filename); + var folder = Library.AutoUpdater.DatabaseLocator.GetDefaultStorageFolderWithDebugSupport(filename); if (!Directory.Exists(folder)) Directory.CreateDirectory(folder); diff --git a/Duplicati/Library/AutoUpdater/DatabaseLocator.cs b/Duplicati/Library/AutoUpdater/DatabaseLocator.cs new file mode 100644 index 000000000..727d91b3b --- /dev/null +++ b/Duplicati/Library/AutoUpdater/DatabaseLocator.cs @@ -0,0 +1,144 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using System; +using System.Linq; +using System.Collections.Generic; + +#nullable enable + +namespace Duplicati.Library.AutoUpdater; + +public static class DatabaseLocator +{ + /// + /// The log tag for this class + /// + public static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(DatabaseLocator)); + + /// + /// Finds a default storage folder, using the operating system specific locations. + /// The targetfilename is used to detect locations that are used in previous versions. + /// If the targetfilename is found in an old location, but not the current, the old location is used. + /// If running with DEBUG defined, the storage folder is placed in the same folder as the executable. + /// Note that the folder is not created, only the path is returned. + /// + /// The filename to look for + /// The name of the application + /// The default storage folder + public static string GetDefaultStorageFolderWithDebugSupport(string targetfilename, string appName = "Duplicati") + { +#if DEBUG + return System.IO.Path.GetDirectoryName(typeof(DatabaseLocator).Assembly.Location) ?? string.Empty; +#else + return GetDefaultStorageFolderDirect(targetfilename, appName); +#endif + } + + /// + /// Finds a default storage folder, using the operating system specific locations. + /// The targetfilename is used to detect locations that are used in previous versions. + /// If the targetfilename is found in an old location, but not the current, the old location is used. + /// Note that the folder is not created, only the path is returned. + /// + /// The filename to look for + /// The name of the application + /// The default storage folder + public static string GetDefaultStorageFolderDirect(string targetfilename, string appName = "Duplicati") + { + //Normal mode uses the systems "(Local) Application Data" folder + // %LOCALAPPDATA% on Windows, ~/.config on Linux + var folder = System.IO.Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), appName); + + if (OperatingSystem.IsWindows()) + { + // Special handling for Windows: + // - Older versions use %APPDATA% + // - New versions use %LOCALAPPDATA% + // - And prevent using C:\Windows\ + var newlocation = System.IO.Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), appName); + + var folderOrder = new List() { + folder, + newlocation + }; + + // If %LOCALAPPDATA% is inside the Windows folder, prefer a LocalService folder instead + var windowsFolder = Library.Common.IO.Util.AppendDirSeparator(Environment.GetFolderPath(Environment.SpecialFolder.Windows)); + if (newlocation.StartsWith(windowsFolder, StringComparison.OrdinalIgnoreCase)) + { + var userProfilesFolder = Library.Utility.SHGetFolder.UserProfilesFolder; + if (!string.IsNullOrWhiteSpace(userProfilesFolder)) + folderOrder.Add(System.IO.Path.Combine(userProfilesFolder, "LocalService", appName)); + } + + // Prefer the most recent location + var matches = folderOrder.AsEnumerable() + .Reverse() + .Where(x => System.IO.File.Exists(System.IO.Path.Combine(x, targetfilename))) + .ToList(); + + // Use the most recent location found with content + // If none are found, use the most recent location + folder = matches.FirstOrDefault() ?? folderOrder.Last(); + + // Emit a warning if the database is stored in the Windows folder + if (folder.StartsWith(windowsFolder, StringComparison.OrdinalIgnoreCase)) + Logging.Log.WriteWarningMessage(LOGTAG, "DatabaseInWindowsFolder", null, "The database is stored in the Windows folder, this is not recommended as it will be deleted on Windows upgrades."); + } + + if (OperatingSystem.IsMacOS()) + { + // Special handling for MacOS: + // - Older versions use ~/.config/ + // - but new versions use ~/Library/Application\ Support/ + var configfolder = System.IO.Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".config", appName); + + var prevfile = System.IO.Path.Combine(configfolder, targetfilename); + var curfile = System.IO.Path.Combine(folder, targetfilename); + + // If the old file exists, and not the new file, we use the old + // Otherwise we use the new location + if (!System.IO.File.Exists(curfile) && System.IO.File.Exists(prevfile)) + folder = configfolder; + } + + if (OperatingSystem.IsLinux() && (folder == $"/{appName}" || folder == appName)) + { + // Special handling for Linux with no home folder: + // - Older versions use / + // - but new versions use /var/lib/ + var libfolder = System.IO.Path.Combine("var", "lib", appName); + + var curfile = System.IO.Path.Combine(libfolder, targetfilename); + var prevfile = System.IO.Path.Combine(folder, targetfilename); + + // If the old file exists, and not the new file, we use the old + // Otherwise we use the new location + if (System.IO.File.Exists(curfile) || !System.IO.File.Exists(prevfile)) + folder = libfolder; + } + + return folder; + } +} + + diff --git a/Duplicati/Library/AutoUpdater/PreloadSettingsLoader.cs b/Duplicati/Library/AutoUpdater/PreloadSettingsLoader.cs index 0c311b89e..667a9dbce 100644 --- a/Duplicati/Library/AutoUpdater/PreloadSettingsLoader.cs +++ b/Duplicati/Library/AutoUpdater/PreloadSettingsLoader.cs @@ -74,7 +74,7 @@ public static class PreloadSettingsLoader // The path for preload settings with the install directory Path.Combine( Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location) ?? "", - "preload.json" + FILE_NAME ), // The path for preload settings specified with an environment variable @@ -109,11 +109,7 @@ public static class PreloadSettingsLoader // User-context path for preload settings, // the same default path as where other data is stored - Path.Combine( - Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), - FOLDER_NAME, - FILE_NAME - ) + Path.Combine(UpdaterManager.DATAFOLDER, FILE_NAME), } .Concat(PortablePreloadPaths) .Where(x => !string.IsNullOrEmpty(x)) diff --git a/Duplicati/Library/AutoUpdater/Program.cs b/Duplicati/Library/AutoUpdater/Program.cs index 064485578..dbdd82bb9 100644 --- a/Duplicati/Library/AutoUpdater/Program.cs +++ b/Duplicati/Library/AutoUpdater/Program.cs @@ -116,7 +116,7 @@ namespace Duplicati.Library.AutoUpdater Console.WriteLine("{0} - Choose different channel than the default {1}, valid settings: {2}", string.Format(AutoUpdateSettings.UPDATECHANNEL_ENVNAME_TEMPLATE, AutoUpdateSettings.AppName), AutoUpdater.AutoUpdateSettings.DefaultUpdateChannel, string.Join(",", Enum.GetNames(typeof(ReleaseType)).Where(x => x != ReleaseType.Unknown.ToString()))); Console.WriteLine(); Console.WriteLine("Updates are downloaded from: {0}", string.Join(";", AutoUpdateSettings.URLs)); - Console.WriteLine("Machine settings are installed in: {0}", UpdaterManager.UPDATEDIR); + Console.WriteLine("Machine settings are installed in: {0}", UpdaterManager.DATAFOLDER); Console.WriteLine("This version is \"{0}\" ({1}) and is installed in: {2}", UpdaterManager.SelfVersion.Displayname, UpdaterManager.SelfVersion.Version, UpdaterManager.INSTALLATIONDIR); Console.WriteLine(); } diff --git a/Duplicati/Library/AutoUpdater/UpdaterManager.cs b/Duplicati/Library/AutoUpdater/UpdaterManager.cs index ab336f399..4936db2e6 100644 --- a/Duplicati/Library/AutoUpdater/UpdaterManager.cs +++ b/Duplicati/Library/AutoUpdater/UpdaterManager.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2025, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a @@ -52,7 +52,7 @@ namespace Duplicati.Library.AutoUpdater /// /// The folder where the machine id is placed /// - public static readonly string UPDATEDIR; + public static readonly string DATAFOLDER; /// /// The directory where the program is running from /// @@ -92,12 +92,12 @@ namespace Duplicati.Library.AutoUpdater /// private const string PACKAGE_TYPE_FILE = "package_type_id.txt"; /// - /// The installation ID filename stored in + /// The installation ID filename stored in /// private const string INSTALL_FILE = "installation.txt"; /// - /// The machine ID filename stored in + /// The machine ID filename stored in /// private const string MACHINE_FILE = "machineid.txt"; @@ -122,60 +122,35 @@ namespace Duplicati.Library.AutoUpdater static UpdaterManager() { // Set the installation path - INSTALLATIONDIR = Path.GetDirectoryName(Duplicati.Library.Utility.Utility.getEntryAssembly().Location); + INSTALLATIONDIR = Path.GetDirectoryName(Utility.Utility.getEntryAssembly().Location); // Check for override - if (string.IsNullOrWhiteSpace(System.Environment.GetEnvironmentVariable(string.Format(UPDATEINSTALLDIR_ENVNAME_TEMPLATE, APPNAME)))) + if (string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable(string.Format(UPDATEINSTALLDIR_ENVNAME_TEMPLATE, APPNAME)))) { - // OS specific folders for probing - var candidates = new List(); - if (OperatingSystem.IsWindows()) - { - candidates.Add(System.IO.Path.Combine(System.Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), APPNAME)); - candidates.Add(System.IO.Path.Combine(System.Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), APPNAME)); - } - else - { - candidates.Add(System.IO.Path.Combine(System.Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), APPNAME)); - } - - if (OperatingSystem.IsMacOS()) - candidates.Add(System.IO.Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Personal), "Library", "Application Support", APPNAME)); - - // Find the first writeable directory in the list - UPDATEDIR = candidates.FirstOrDefault(p => !string.IsNullOrWhiteSpace(p) && System.IO.Directory.Exists(p) && TestDirectoryIsWriteable(p)); - - // Try to create a writeable folder, if none is found - if (string.IsNullOrWhiteSpace(UPDATEDIR)) - UPDATEDIR = candidates.Where(p => !string.IsNullOrWhiteSpace(p) && !System.IO.Directory.Exists(p)) - .Select(p => - { - try { System.IO.Directory.CreateDirectory(p); } - catch { } - return p; - }) - .Where(p => System.IO.Directory.Exists(p) && TestDirectoryIsWriteable(p)) - .FirstOrDefault(); + DATAFOLDER = DatabaseLocator.GetDefaultStorageFolderWithDebugSupport(INSTALL_FILE, APPNAME); } else { // Use override, no checks - UPDATEDIR = Environment.ExpandEnvironmentVariables(System.Environment.GetEnvironmentVariable(string.Format(UPDATEINSTALLDIR_ENVNAME_TEMPLATE, APPNAME))); + DATAFOLDER = Environment.ExpandEnvironmentVariables(Environment.GetEnvironmentVariable(string.Format(UPDATEINSTALLDIR_ENVNAME_TEMPLATE, APPNAME))); } - if (!string.IsNullOrWhiteSpace(UPDATEDIR)) + if (!string.IsNullOrWhiteSpace(DATAFOLDER)) { - if (!System.IO.File.Exists(System.IO.Path.Combine(UPDATEDIR, INSTALL_FILE))) + if (!Directory.Exists(DATAFOLDER)) + Directory.CreateDirectory(DATAFOLDER); + + if (!File.Exists(Path.Combine(DATAFOLDER, INSTALL_FILE))) { - // In case there was already a machine id file, copy it to the new location - if (System.IO.File.Exists(System.IO.Path.Combine(UPDATEDIR, "updates", INSTALL_FILE))) - System.IO.File.Copy(System.IO.Path.Combine(UPDATEDIR, "updates", INSTALL_FILE), System.IO.Path.Combine(UPDATEDIR, INSTALL_FILE), true); + // In case there was already a machine id file from 2.0.8.1 or older, copy it to the new location + if (File.Exists(Path.Combine(DATAFOLDER, "updates", INSTALL_FILE))) + File.Copy(Path.Combine(DATAFOLDER, "updates", INSTALL_FILE), Path.Combine(DATAFOLDER, INSTALL_FILE), true); else - System.IO.File.WriteAllText(System.IO.Path.Combine(UPDATEDIR, INSTALL_FILE), AutoUpdateSettings.UpdateInstallFileText); + File.WriteAllText(Path.Combine(DATAFOLDER, INSTALL_FILE), AutoUpdateSettings.UpdateInstallFileText); } - if (!System.IO.File.Exists(System.IO.Path.Combine(UPDATEDIR, MACHINE_FILE))) - System.IO.File.WriteAllText(System.IO.Path.Combine(UPDATEDIR, MACHINE_FILE), AutoUpdateSettings.UpdateMachineFileText(InstallID)); + if (!File.Exists(Path.Combine(DATAFOLDER, MACHINE_FILE))) + File.WriteAllText(Path.Combine(DATAFOLDER, MACHINE_FILE), AutoUpdateSettings.UpdateMachineFileText(InstallID)); } @@ -200,7 +175,7 @@ namespace Duplicati.Library.AutoUpdater UpdateSeverity: null, ChangeInfo: null, Packages: null, - Displayname: string.IsNullOrWhiteSpace(Duplicati.License.VersionNumbers.TAG) ? "Current" : Duplicati.License.VersionNumbers.TAG, + Displayname: string.IsNullOrWhiteSpace(License.VersionNumbers.TAG) ? "Current" : License.VersionNumbers.TAG, Version: System.Reflection.Assembly.GetExecutingAssembly().GetName().Version.ToString(), ReleaseTime: new DateTime(0), ReleaseType: @@ -222,28 +197,6 @@ namespace Duplicati.Library.AutoUpdater return new Version(0, 0); } - private static bool TestDirectoryIsWriteable(string path) - { - var p2 = System.IO.Path.Combine(path, "test-" + DateTime.UtcNow.ToString(DATETIME_FORMAT, System.Globalization.CultureInfo.InvariantCulture)); - var probe = System.IO.Directory.Exists(path) ? p2 : path; - - if (!System.IO.Directory.Exists(probe)) - { - try - { - System.IO.Directory.CreateDirectory(probe); - if (probe != path) - System.IO.Directory.Delete(probe); - return true; - } - catch - { - } - } - - return false; - } - /// /// The unique machine installation ID /// @@ -251,7 +204,7 @@ namespace Duplicati.Library.AutoUpdater { get { - try { return System.IO.File.ReadAllLines(System.IO.Path.Combine(UPDATEDIR, INSTALL_FILE)).FirstOrDefault(x => !string.IsNullOrWhiteSpace(x))?.Trim() ?? ""; } + try { return System.IO.File.ReadAllLines(Path.Combine(DATAFOLDER, INSTALL_FILE)).FirstOrDefault(x => !string.IsNullOrWhiteSpace(x))?.Trim() ?? ""; } catch { } return ""; @@ -266,7 +219,7 @@ namespace Duplicati.Library.AutoUpdater get { string machinedId = null; - try { machinedId = System.IO.File.ReadAllLines(System.IO.Path.Combine(UPDATEDIR, MACHINE_FILE)).FirstOrDefault(x => !string.IsNullOrWhiteSpace(x))?.Trim() ?? ""; } + try { machinedId = System.IO.File.ReadAllLines(Path.Combine(DATAFOLDER, MACHINE_FILE)).FirstOrDefault(x => !string.IsNullOrWhiteSpace(x))?.Trim() ?? ""; } catch { } return string.IsNullOrWhiteSpace(machinedId) @@ -278,7 +231,7 @@ namespace Duplicati.Library.AutoUpdater /// /// The machine name /// - public static string MachineName => System.Environment.MachineName; + public static string MachineName => Environment.MachineName; /// /// The package type ID @@ -287,7 +240,7 @@ namespace Duplicati.Library.AutoUpdater { get { - try { return System.IO.File.ReadAllLines(System.IO.Path.Combine(INSTALLATIONDIR, PACKAGE_TYPE_FILE)).FirstOrDefault(x => !string.IsNullOrWhiteSpace(x))?.Trim() ?? ""; } + try { return File.ReadAllLines(Path.Combine(INSTALLATIONDIR, PACKAGE_TYPE_FILE)).FirstOrDefault(x => !string.IsNullOrWhiteSpace(x))?.Trim() ?? ""; } catch { } #if DEBUG @@ -343,7 +296,7 @@ namespace Duplicati.Library.AutoUpdater timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS)); HttpClientHelper.DefaultClient.DownloadFile(request, tmpfile, null, timeoutToken.Token).ConfigureAwait(false).GetAwaiter().GetResult(); - using (var fs = System.IO.File.OpenRead(tmpfile)) + using (var fs = File.OpenRead(tmpfile)) { var verifyOps = SIGN_KEYS.Select(k => new JSONSignature.VerifyOperation( Algorithm: JSONSignature.RSA_SHA256, @@ -396,8 +349,8 @@ namespace Duplicati.Library.AutoUpdater private static UpdateInfo ReadInstalledManifest(string folder) { - var manifest = System.IO.Path.Combine(folder, UPDATE_MANIFEST_FILENAME); - if (System.IO.File.Exists(manifest)) + var manifest = Path.Combine(folder, UPDATE_MANIFEST_FILENAME); + if (File.Exists(manifest)) { try { @@ -406,7 +359,7 @@ namespace Duplicati.Library.AutoUpdater PublicKey: k.ToXmlString(false) )); - using (var fs = System.IO.File.OpenRead(manifest)) + using (var fs = File.OpenRead(manifest)) { if (!JSONSignature.VerifyAtLeastOne(fs, verifyOps)) throw new Exception("Installed manifest signature is invalid"); @@ -426,7 +379,7 @@ namespace Duplicati.Library.AutoUpdater public static bool DownloadUpdate(UpdateInfo version, PackageEntry package, string targetPath, Action progress = null) { - if (UPDATEDIR == null) + if (DATAFOLDER == null) return false; var updates = package.RemoteUrls.ToList(); @@ -455,7 +408,7 @@ namespace Duplicati.Library.AutoUpdater { try { - using (var tempfile = System.IO.File.Open(tempfilename, FileMode.Create, FileAccess.ReadWrite, FileShare.None)) + using (var tempfile = File.Open(tempfilename, FileMode.Create, FileAccess.ReadWrite, FileShare.None)) { Action cb = null; if (progress != null) diff --git a/Duplicati/Library/Main/DatabaseLocator.cs b/Duplicati/Library/Main/CLIDatabaseLocator.cs similarity index 63% rename from Duplicati/Library/Main/DatabaseLocator.cs rename to Duplicati/Library/Main/CLIDatabaseLocator.cs index 73065132f..7c5255e4c 100644 --- a/Duplicati/Library/Main/DatabaseLocator.cs +++ b/Duplicati/Library/Main/CLIDatabaseLocator.cs @@ -27,12 +27,12 @@ using System.Collections.Generic; namespace Duplicati.Library.Main { - public static class DatabaseLocator + public static class CLIDatabaseLocator { /// /// The log tag for this class /// - public static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(DatabaseLocator)); + public static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(CLIDatabaseLocator)); /// /// The entry for a backend configuration, used for serialization @@ -78,113 +78,6 @@ namespace Duplicati.Library.Main /// private const string CONFIG_FILE = "dbconfig.json"; - /// - /// Finds a default storage folder, using the operating system specific locations. - /// The targetfilename is used to detect locations that are used in previous versions. - /// If the targetfilename is found in an old location, but not the current, the old location is used. - /// If running with DEBUG defined, the storage folder is placed in the same folder as the executable. - /// Note that the folder is not created, only the path is returned. - /// - /// The filename to look for - /// The name of the application - /// The default storage folder - public static string GetDefaultStorageFolderWithDebugSupport(string targetfilename, string appName = "Duplicati") - { -#if DEBUG - return System.IO.Path.GetDirectoryName(typeof(DatabaseLocator).Assembly.Location) ?? string.Empty; -#else - return GetDefaultStorageFolderDirect(targetfilename, appName); -#endif - } - - /// - /// Finds a default storage folder, using the operating system specific locations. - /// The targetfilename is used to detect locations that are used in previous versions. - /// If the targetfilename is found in an old location, but not the current, the old location is used. - /// Note that the folder is not created, only the path is returned. - /// - /// The filename to look for - /// The name of the application - /// The default storage folder - public static string GetDefaultStorageFolderDirect(string targetfilename, string appName = "Duplicati") - { - //Normal mode uses the systems "(Local) Application Data" folder - // %LOCALAPPDATA% on Windows, ~/.config on Linux - var folder = System.IO.Path.Combine(System.Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), appName); - - if (OperatingSystem.IsWindows()) - { - // Special handling for Windows: - // - Older versions use %APPDATA% - // - New versions use %LOCALAPPDATA% - // - And prevent using C:\Windows\ - var newlocation = System.IO.Path.Combine(System.Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), appName); - - var folderOrder = new List() { - folder, - newlocation - }; - - // If %LOCALAPPDATA% is inside the Windows folder, prefer a LocalService folder instead - var windowsFolder = Library.Common.IO.Util.AppendDirSeparator(Environment.GetFolderPath(Environment.SpecialFolder.Windows)); - if (newlocation.StartsWith(windowsFolder, StringComparison.OrdinalIgnoreCase)) - { - var userProfilesFolder = Library.Utility.SHGetFolder.UserProfilesFolder; - if (!string.IsNullOrWhiteSpace(userProfilesFolder)) - folderOrder.Add(System.IO.Path.Combine(userProfilesFolder, "LocalService", appName)); - } - - // Prefer the most recent location - var matches = folderOrder.AsEnumerable() - .Reverse() - .Select(x => System.IO.Path.Combine(x, targetfilename)) - .Where(System.IO.File.Exists) - .ToList(); - - // Use the most recent location found with content - // If none are found, use the most recent location - folder = matches.FirstOrDefault() ?? folderOrder.Last(); - - // Emit a warning if the database is stored in the Windows folder - if (folder.StartsWith(windowsFolder, StringComparison.OrdinalIgnoreCase)) - Logging.Log.WriteWarningMessage(LOGTAG, "DatabaseInWindowsFolder", null, "The database is stored in the Windows folder, this is not recommended as it will be deleted on Windows upgrades."); - } - - if (OperatingSystem.IsMacOS()) - { - // Special handling for MacOS: - // - Older versions use ~/.config/ - // - but new versions use ~/Library/Application\ Support/ - var configfolder = System.IO.Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".config", appName); - - var prevfile = System.IO.Path.Combine(configfolder, targetfilename); - var curfile = System.IO.Path.Combine(folder, targetfilename); - - // If the old file exists, and not the new file, we use the old - // Otherwise we use the new location - if (!System.IO.File.Exists(curfile) && System.IO.File.Exists(prevfile)) - folder = configfolder; - } - - if (OperatingSystem.IsLinux() && (folder == $"/{appName}" || folder == appName)) - { - // Special handling for Linux with no home folder: - // - Older versions use / - // - but new versions use /var/lib/ - var libfolder = System.IO.Path.Combine("var", "lib", appName); - - var curfile = System.IO.Path.Combine(libfolder, targetfilename); - var prevfile = System.IO.Path.Combine(folder, targetfilename); - - // If the old file exists, and not the new file, we use the old - // Otherwise we use the new location - if (System.IO.File.Exists(curfile) || !System.IO.File.Exists(prevfile)) - folder = libfolder; - } - - return folder; - } - /// /// Gets the database path for a given backend url, using the JSON configuration file lookup. /// If is true, a new database is created if none is found. @@ -202,7 +95,7 @@ namespace Duplicati.Library.Main if (!string.IsNullOrEmpty(options.Dbpath)) return options.Dbpath; - var folder = GetDefaultStorageFolderWithDebugSupport(CONFIG_FILE); + var folder = AutoUpdater.DatabaseLocator.GetDefaultStorageFolderWithDebugSupport(CONFIG_FILE); var file = System.IO.Path.Combine(folder, CONFIG_FILE); List configs; @@ -349,7 +242,7 @@ namespace Duplicati.Library.Main /// true if the path is in use, false otherwise public static bool IsDatabasePathInUse(string path) { - var file = System.IO.Path.Combine(GetDefaultStorageFolderWithDebugSupport(CONFIG_FILE), CONFIG_FILE); + var file = System.IO.Path.Combine(AutoUpdater.DatabaseLocator.GetDefaultStorageFolderWithDebugSupport(CONFIG_FILE), CONFIG_FILE); if (!System.IO.File.Exists(file)) return false; diff --git a/Duplicati/Library/Main/Controller.cs b/Duplicati/Library/Main/Controller.cs index ad5b9cf8b..4f2336110 100644 --- a/Duplicati/Library/Main/Controller.cs +++ b/Duplicati/Library/Main/Controller.cs @@ -700,7 +700,7 @@ namespace Duplicati.Library.Main } if (string.IsNullOrEmpty(m_options.Dbpath)) - m_options.Dbpath = DatabaseLocator.GetDatabasePathForCLI(m_backend, m_options); + m_options.Dbpath = CLIDatabaseLocator.GetDatabasePathForCLI(m_backend, m_options); ValidateOptions(); } diff --git a/Duplicati/Library/Main/Operation/SystemInfoHandler.cs b/Duplicati/Library/Main/Operation/SystemInfoHandler.cs index 3c1f4579c..069f34e1f 100644 --- a/Duplicati/Library/Main/Operation/SystemInfoHandler.cs +++ b/Duplicati/Library/Main/Operation/SystemInfoHandler.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -38,7 +38,7 @@ namespace Duplicati.Library.Main.Operation yield return string.Format("Duplicati: {0} ({1})", Library.Utility.Utility.getEntryAssembly().FullName, System.Reflection.Assembly.GetExecutingAssembly().FullName); yield return string.Format("Autoupdate urls: {0}", string.Join(";", AutoUpdater.AutoUpdateSettings.URLs)); - yield return string.Format("Update folder: {0}", AutoUpdater.UpdaterManager.UPDATEDIR); + yield return string.Format("Default data folder: {0}", AutoUpdater.UpdaterManager.DATAFOLDER); yield return string.Format("Install folder: {0}", AutoUpdater.UpdaterManager.INSTALLATIONDIR); yield return string.Format("Version name: \"{0}\" ({1})", AutoUpdater.UpdaterManager.SelfVersion.Displayname, System.Reflection.Assembly.GetExecutingAssembly().GetName().Version); yield return string.Format("Current Version folder {0}", System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location)); diff --git a/Duplicati/Library/RestAPI/Database/Connection.cs b/Duplicati/Library/RestAPI/Database/Connection.cs index ac45e3edd..f9aadf1cd 100644 --- a/Duplicati/Library/RestAPI/Database/Connection.cs +++ b/Duplicati/Library/RestAPI/Database/Connection.cs @@ -605,7 +605,7 @@ namespace Duplicati.Server.Database for (var i = 0; i < 100; i++) { - var guess = System.IO.Path.Combine(folder, System.IO.Path.ChangeExtension(Duplicati.Library.Main.DatabaseLocator.GenerateRandomName(), ".sqlite")); + var guess = System.IO.Path.Combine(folder, System.IO.Path.ChangeExtension(CLIDatabaseLocator.GenerateRandomName(), ".sqlite")); if (!System.IO.File.Exists(guess)) { ((Backup)item).DBPath = guess; diff --git a/Duplicati/Library/Utility/SHGetFolder.cs b/Duplicati/Library/Utility/SHGetFolder.cs index 3b42fae77..fd1df513b 100644 --- a/Duplicati/Library/Utility/SHGetFolder.cs +++ b/Duplicati/Library/Utility/SHGetFolder.cs @@ -7,7 +7,7 @@ using System.Runtime.Versioning; namespace Duplicati.Library.Utility; /// -/// Wrapper for SHGetKnownFolderPath to get the download folder on Windows +/// Wrapper for SHGetKnownFolderPath to various system folders on Windows /// /// List of GUIDs: https://learn.microsoft.com/en-us/windows/win32/shell/knownfolderid [SupportedOSPlatform("windows")] diff --git a/Duplicati/Server/Program.cs b/Duplicati/Server/Program.cs index ef78784d4..14076140b 100644 --- a/Duplicati/Server/Program.cs +++ b/Duplicati/Server/Program.cs @@ -781,9 +781,12 @@ namespace Duplicati.Server } else { - //Normal release mode uses the systems "(Local) Application Data" folder - // %LOCALAPPDATA% on Windows, ~/.config on Linux, ~/Library/Application\ Support on MacOS - return DatabaseLocator.GetDefaultStorageFolderDirect(SERVER_DATABASE_FILENAME, Library.AutoUpdater.AutoUpdateSettings.AppName); + //Normal release mode uses + // %LOCALAPPDATA% on Windows, + // ~/.config on Linux, + // ~/Library/Application\ Support on MacOS + // with additional rules for special cases and probing for older locations + return DatabaseLocator.GetDefaultStorageFolderDirect(SERVER_DATABASE_FILENAME, AutoUpdateSettings.AppName); } } else diff --git a/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs b/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs index e845a99d5..166a494cd 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs @@ -292,7 +292,7 @@ public class BackupGet : IEndpointV1 } private static Dto.IsDbUsedElsewhereDto ExecuteGetIsdbUsedElsewhere(IBackup bk) - => new Dto.IsDbUsedElsewhereDto(Library.Main.DatabaseLocator.IsDatabasePathInUse(bk.DBPath)); + => new Dto.IsDbUsedElsewhereDto(Library.Main.CLIDatabaseLocator.IsDatabasePathInUse(bk.DBPath)); private static Dto.IsBackupActiveDto ExecuteGetIsActive(IWorkerThreadsManager workerThreadsManager, IBackup bk) { diff --git a/Duplicati/WebserverCore/Endpoints/V1/Backups.cs b/Duplicati/WebserverCore/Endpoints/V1/Backups.cs index 4a3576cd5..84783a1f3 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/Backups.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/Backups.cs @@ -206,7 +206,7 @@ public class Backups : IEndpointV1 { if (existingDb) { - backup.DBPathSetter = Library.Main.DatabaseLocator.GetDatabasePathForCLI(data.Backup.TargetURL, null, false, false); + backup.DBPathSetter = Library.Main.CLIDatabaseLocator.GetDatabasePathForCLI(data.Backup.TargetURL, null, false, false); if (string.IsNullOrWhiteSpace(data.Backup.DBPath)) throw new Exception("Unable to find remote db path?"); } diff --git a/Duplicati/WebserverCore/Endpoints/V1/RemoteOperation.cs b/Duplicati/WebserverCore/Endpoints/V1/RemoteOperation.cs index c90e67276..8771de22d 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/RemoteOperation.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/RemoteOperation.cs @@ -49,7 +49,7 @@ namespace Duplicati.WebserverCore.Endpoints.V1 private static Dto.GetDbPathDto ExecuteDbPath(string uri) { - var path = Library.Main.DatabaseLocator.GetDatabasePathForCLI(uri, null, false, false); + var path = CLIDatabaseLocator.GetDatabasePathForCLI(uri, null, false, false); return new Dto.GetDbPathDto(!string.IsNullOrWhiteSpace(path), path); } From 7a78f72f19bca1c007c785547bf98867add6f3cf Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 27 Jan 2025 23:17:23 +0100 Subject: [PATCH 037/362] Updated warning message based on review --- Duplicati/Server/Program.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Duplicati/Server/Program.cs b/Duplicati/Server/Program.cs index 14076140b..40bac2ee7 100644 --- a/Duplicati/Server/Program.cs +++ b/Duplicati/Server/Program.cs @@ -605,12 +605,12 @@ namespace Duplicati.Server ); } - if (OperatingSystem.IsWindows() && DataFolder.StartsWith(Environment.GetFolderPath(Environment.SpecialFolder.Windows), StringComparison.OrdinalIgnoreCase)) + if (OperatingSystem.IsWindows() && DataFolder.StartsWith(Util.AppendDirSeparator(Environment.GetFolderPath(Environment.SpecialFolder.Windows)), StringComparison.OrdinalIgnoreCase)) { DataConnection.RegisterNotification( Serialization.NotificationType.Warning, "Incorrect storage folder", - "The configuration is stored in the Windows folder, which will be deleted on Windows updates.", + "The server configuraion is stored inside the Windows folder. Please move the configuration to a different location, or it may be deleted on Windows version upgrades.", null, null, "config:issue:windows-folder-used", From 8c105c9e92587bbaaa96c33540fcecdd8532d41b Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 27 Jan 2025 23:17:43 +0100 Subject: [PATCH 038/362] Added help links based on review feedback --- Duplicati/Server/webroot/ngax/templates/notificationarea.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Duplicati/Server/webroot/ngax/templates/notificationarea.html b/Duplicati/Server/webroot/ngax/templates/notificationarea.html index d5904747d..60e4a9523 100644 --- a/Duplicati/Server/webroot/ngax/templates/notificationarea.html +++ b/Duplicati/Server/webroot/ngax/templates/notificationarea.html @@ -14,6 +14,9 @@ Repair Download + + Show help + Show help
From 2df0710d7eceacf2d483cdd00b126b0313b8ec33 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 28 Jan 2025 09:48:51 +0100 Subject: [PATCH 039/362] Added the capability to forward multiple tokens to an option --- Duplicati/UnitTest/ToolTests.cs | 7 ++++++- Tools/RemoteSynchronization/Program.cs | 27 ++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/Duplicati/UnitTest/ToolTests.cs b/Duplicati/UnitTest/ToolTests.cs index 6268e284f..b187bdfb9 100644 --- a/Duplicati/UnitTest/ToolTests.cs +++ b/Duplicati/UnitTest/ToolTests.cs @@ -70,7 +70,12 @@ namespace Duplicati.UnitTest // Call the tool var exe = RemoteSynchronization.Program.Main; - var args = new string[] { $"file://{l1}", $"file://{l2}" }; + var args = new string[] { + $"file://{l1}", $"file://{l2}", + // Pass along multi token options to test that the parser won't fail + "--src-options", "ssh-accept-any-fingerprints=true", "ssh-keyfile=/path/to/keyfile", + "--dst-options", "some-other-key=value", "another-key=value2" + }; var async_call = exe(args); async_call.Wait(); Assert.AreEqual(0, async_call.Result, "Remote synchronization tool did not return 0."); diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 471d2fac7..5766ecad1 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -40,11 +40,15 @@ namespace RemoteSynchronization var src_arg = new Argument(name: "backend_src", description: "The source backend string"); var dst_arg = new Argument(name: "backend_dst", description: "The destination backend string"); var dry_run_opt = new Option(aliases: ["--dry-run", "-d"], description: "Do not actually write or delete files"); + var src_opts = OptionWithMultipleTokens(aliases: ["--src-options"], description: "Options for the source backend"); + var dst_opts = OptionWithMultipleTokens(aliases: ["--dst-options"], description: "Options for the destination backend"); var root_cmd = new RootCommand("Remote Synchronization Tool"); root_cmd.AddArgument(src_arg); root_cmd.AddArgument(dst_arg); root_cmd.AddOption(dry_run_opt); + root_cmd.AddOption(src_opts); + root_cmd.AddOption(dst_opts); root_cmd.SetHandler((InvocationContext ctx) => { @@ -65,12 +69,22 @@ namespace RemoteSynchronization { ["dry-run"] = dry_run.ToString() }; + Dictionary src_opts = (options["src-options"] as List) + ?.Select(x => x.Split("=")) + .ToDictionary(x => x[0], x => x[1]) + ?? []; + Dictionary dst_opts = (options["dst-options"] as List) + ?.Select(x => x.Split("=")) + .ToDictionary(x => x[0], x => x[1]) + ?? []; + foreach (var x in duplicati_options) + src_opts[x.Key] = dst_opts[x.Key] = x.Value; - using var b1 = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(src, duplicati_options); + using var b1 = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(src, src_opts); var b1s = b1 as IStreamingBackend; System.Diagnostics.Debug.Assert(b1s != null); - using var b2 = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(dst, duplicati_options); + using var b2 = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(dst, dst_opts); var b2s = b2 as IStreamingBackend; System.Diagnostics.Debug.Assert(b2s != null); @@ -148,6 +162,15 @@ namespace RemoteSynchronization return successful_deletes; } + private static Option> OptionWithMultipleTokens(string[] aliases, string description) + { + return new Option>(aliases: aliases, description: description) + { + Arity = ArgumentArity.OneOrMore, + AllowMultipleArgumentsPerToken = true + }; + } + private static (IEnumerable, IEnumerable) PrepareFileLists(IStreamingBackend b_src, IStreamingBackend b_dst) { var files_src = b_src.List(); From 552f9747e20d81c2c9c42518e857cc7601eac15c Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 28 Jan 2025 09:49:22 +0100 Subject: [PATCH 040/362] Modified ToolTests.GenerateTestData to use async/await for improved performance --- Duplicati/UnitTest/ToolTests.cs | 46 +++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/Duplicati/UnitTest/ToolTests.cs b/Duplicati/UnitTest/ToolTests.cs index b187bdfb9..bd57e6506 100644 --- a/Duplicati/UnitTest/ToolTests.cs +++ b/Duplicati/UnitTest/ToolTests.cs @@ -23,9 +23,9 @@ using Duplicati.Library.Common.IO; using Duplicati.Library.Main; using NUnit.Framework; using System; -using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading.Tasks; namespace Duplicati.UnitTest { @@ -50,7 +50,9 @@ namespace Duplicati.UnitTest var options = TestOptions; - GenerateTestData(DATAFOLDER, 10, 3, 3, 1024 * 1024); + var now = DateTime.Now; + GenerateTestData(DATAFOLDER, 10, 3, 3, 1024 * 1024).Wait(); + Console.WriteLine($"Generated test data in {DATAFOLDER} in {DateTime.Now - now}"); // Create the directories if they do not exist foreach (var p in new string[] { l1, l2, l1r, l2r }) @@ -151,29 +153,33 @@ namespace Duplicati.UnitTest /// How many subdirectories the directory should have. /// How deep the number of subdirectories within subdirectories should go. /// The maximum size of the files to generate. - public static void GenerateTestData(string dir, int n_files, int n_dirs, int n_levels, int max_file_size) + public static async Task GenerateTestData(string dir, int n_files, int n_dirs, int n_levels, int max_file_size) { if (!SystemIO.IO_OS.DirectoryExists(dir)) SystemIO.IO_OS.DirectoryCreate(dir); - var rnd = new Random(); - for (int i = 0; i < n_files; i++) - { - var file = Path.Combine(dir, $"file_{i}.txt"); - var size = rnd.Next(1, max_file_size); - var data = new byte[size]; - rnd.NextBytes(data); - File.WriteAllBytes(file, data); - } + var fs = Enumerable.Range(0, n_files) + .Select(i => GenerateTestFile(dir, i, max_file_size)); + var ds = n_levels > 0 ? + Enumerable.Range(0, n_dirs) + .Select(i => + { + var subdir = Path.Combine(dir, $"dir_{i}"); + return GenerateTestData(subdir, n_files, n_dirs, n_levels - 1, max_file_size); + }) + : []; - if (n_levels > 0) - { - for (int i = 0; i < n_dirs; i++) - { - var subdir = Path.Combine(dir, $"dir_{i}"); - GenerateTestData(subdir, n_files, n_dirs, n_levels - 1, max_file_size); - } - } + await Task.WhenAll([.. fs, .. ds]); + } + + public static async Task GenerateTestFile(string dir, int i, int max_file_size) + { + var rnd = new Random(); + var file = Path.Combine(dir, $"file_{i}.txt"); + var size = rnd.Next(1, max_file_size); + var data = new byte[size]; + rnd.NextBytes(data); + await File.WriteAllBytesAsync(file, data); } } From f169e145f4695a190dcdc45a7aa9045d6e6cb7e7 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 28 Jan 2025 11:14:58 +0100 Subject: [PATCH 041/362] Add verify option to Remote Synchronization Tool for post-copy validation --- Tools/RemoteSynchronization/Program.cs | 105 +++++++++++++++++++++---- 1 file changed, 91 insertions(+), 14 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 5766ecad1..2b5dbb2d1 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2025, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a @@ -35,6 +35,8 @@ namespace RemoteSynchronization { public class Program { + private const bool DEFAULT_VERIFY = false; + public static async Task Main(string[] args) { var src_arg = new Argument(name: "backend_src", description: "The source backend string"); @@ -42,6 +44,7 @@ namespace RemoteSynchronization var dry_run_opt = new Option(aliases: ["--dry-run", "-d"], description: "Do not actually write or delete files"); var src_opts = OptionWithMultipleTokens(aliases: ["--src-options"], description: "Options for the source backend"); var dst_opts = OptionWithMultipleTokens(aliases: ["--dst-options"], description: "Options for the destination backend"); + var verify_opt = new Option(aliases: ["--verify"], description: "Verify the files after copying", getDefaultValue: () => DEFAULT_VERIFY); var root_cmd = new RootCommand("Remote Synchronization Tool"); root_cmd.AddArgument(src_arg); @@ -49,6 +52,7 @@ namespace RemoteSynchronization root_cmd.AddOption(dry_run_opt); root_cmd.AddOption(src_opts); root_cmd.AddOption(dst_opts); + root_cmd.AddOption(verify_opt); root_cmd.SetHandler((InvocationContext ctx) => { @@ -64,7 +68,7 @@ namespace RemoteSynchronization private static async Task Run(string src, string dst, Dictionary options) { - var dry_run = options["dry-run"] as bool? ?? false; + var verify = options["verify"] as bool? ?? DEFAULT_VERIFY; var duplicati_options = new Dictionary() { ["dry-run"] = dry_run.ToString() @@ -88,12 +92,27 @@ namespace RemoteSynchronization var b2s = b2 as IStreamingBackend; System.Diagnostics.Debug.Assert(b2s != null); - var (to_copy, to_delete) = PrepareFileLists(b1s, b2s); + var (to_copy, to_delete, to_verify) = PrepareFileLists(b1s, b2s); var deleted = await DeleteAsync(b2s, to_delete, dry_run); Console.WriteLine($"Deleted {deleted} files from {dst}"); - var copied = await CopyAsync(b1s, b2s, to_copy, dry_run); + var (copied, copy_errors) = await CopyAsync(b1s, b2s, to_copy, dry_run, verify); Console.WriteLine($"Copied {copied} files from {src} to {dst}"); + if (copy_errors.Any()) + { + Console.WriteLine($"Could not copy {copy_errors.Count()} files: {string.Join(", ", copy_errors)}"); + return copy_errors.Count(); + } + + if (verify) + { + var not_verified = await VerifyAsync(b1s, b2s, to_verify); + Console.WriteLine($"Could not verify {not_verified.Count()} files: {string.Join(", ", not_verified)}"); + return not_verified.Count(); + } + + Console.WriteLine("Done"); + return 0; } @@ -103,38 +122,52 @@ namespace RemoteSynchronization // TODO Force parameter // TODO Progress reporting // TODO Logging + // TODO Retry on errors // check database consistency. I.e. have both databases, check that the block, volume, files, etc match up. // introduce these checks as a post processing step? Especially the database consistency check, as that is often recreated from the index files. // TODO This tool shouldn't handle it, but for convenience, it should support making the seperate call to the regular Duplicati on the destination backend, which alread carries this functionality. // Forcefully synchronize the remote backends - private static async Task CopyAsync(IStreamingBackend b_src, IStreamingBackend b_dst, IEnumerable files, bool dry_run) + private static async Task<(long, IEnumerable)> CopyAsync(IStreamingBackend b_src, IStreamingBackend b_dst, IEnumerable files, bool dry_run, bool verify) { long successful_copies = 0; - using var s = new MemoryStream(); + List errors = []; + using var s_src = new MemoryStream(); + using var s_dst = new MemoryStream(); foreach (var f in files) { try { - await b_src.GetAsync(f.Name, s, CancellationToken.None); + await b_src.GetAsync(f.Name, s_src, CancellationToken.None); if (dry_run) { - Console.WriteLine($"Would write {s.Length} bytes of {f.Name} to {b_dst.DisplayName}"); + Console.WriteLine($"Would write {s_src.Length} bytes of {f.Name} to {b_dst.DisplayName}"); } else { - await b_dst.PutAsync(f.Name, s, CancellationToken.None); + await b_dst.PutAsync(f.Name, s_src, CancellationToken.None); + if (verify) + { + await b_dst.GetAsync(f.Name, s_dst, CancellationToken.None); + if (s_src.Length != s_dst.Length || !s_src.ToArray().SequenceEqual(s_dst.ToArray())) + { + Console.WriteLine($"Error verifying {f.Name}: The file was not copied correctly"); + errors.Add(f.Name); + } + s_dst.SetLength(0); + } } - s.SetLength(0); + s_src.SetLength(0); successful_copies++; } catch (Exception e) { Console.WriteLine($"Error copying {f.Name}: {e.Message}"); + errors.Add(f.Name); } } - return successful_copies; + return (successful_copies, errors); } private static async Task DeleteAsync(IStreamingBackend b, IEnumerable files, bool dry_run) @@ -171,7 +204,7 @@ namespace RemoteSynchronization }; } - private static (IEnumerable, IEnumerable) PrepareFileLists(IStreamingBackend b_src, IStreamingBackend b_dst) + private static (IEnumerable, IEnumerable, IEnumerable) PrepareFileLists(IStreamingBackend b_src, IStreamingBackend b_dst) { var files_src = b_src.List(); var files_dst = b_dst.List(); @@ -179,7 +212,7 @@ namespace RemoteSynchronization // Shortcut for empty destination if (!files_dst.Any()) { - return (files_src, []); + return (files_src, [], []); } var lookup_src = files_src.ToDictionary(x => x.Name); @@ -187,6 +220,7 @@ namespace RemoteSynchronization var to_copy = new List(); var to_delete = new HashSet(); + var to_verify = new List(); // Find all of the files in src that are not in dst, have a different size or have a more recent modification date foreach (var f_src in files_src) @@ -195,12 +229,19 @@ namespace RemoteSynchronization { if (f_src.Size != f_dst.Size || f_src.LastModification > f_dst.LastModification) { + // The file is different, so we need to copy it to_copy.Add(f_src); to_delete.Add(f_dst.Name); } + else + { + // The file seems to be the same, so we need to verify it if the user wants to + to_verify.Add(f_src); + } } else { + // The file is not in the destination, so we need to copy it to_copy.Add(f_src); } } @@ -217,7 +258,7 @@ namespace RemoteSynchronization } } - return (to_copy, to_delete.Select(x => lookup_dst[x])); + return (to_copy, to_delete.Select(x => lookup_dst[x]), to_verify); } private static async Task RenameAsync(IStreamingBackend b, IEnumerable files, bool dry_run) @@ -250,5 +291,41 @@ namespace RemoteSynchronization return successful_renames; } + // Post comparison + private static async Task> VerifyAsync(IStreamingBackend b_src, IStreamingBackend b_dst, IEnumerable files) + { + var errors = new List(); + using var s_src = new MemoryStream(); + using var s_dst = new MemoryStream(); + + foreach (var f in files) + { + try + { + // Get both files + var fs = b_src.GetAsync(f.Name, s_src, CancellationToken.None); + var ds = b_dst.GetAsync(f.Name, s_dst, CancellationToken.None); + await Task.WhenAll(fs, ds); + + // Compare the contents + if (s_src.Length != s_dst.Length || !s_src.ToArray().SequenceEqual(s_dst.ToArray())) + { + errors.Add(f.Name); + } + + // Reset the streams + s_src.SetLength(0); + s_dst.SetLength(0); + } + catch (Exception e) + { + errors.Add($"{f.Name}"); + Console.WriteLine($"Error verifying {f.Name}: {e.Message}"); + } + } + + return errors; + } + } } From 5939f033a642f61082e1273d6eff29a9019e5aff Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 28 Jan 2025 11:18:32 +0100 Subject: [PATCH 042/362] Set default values for dry run and verify options --- Tools/RemoteSynchronization/Program.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 2b5dbb2d1..cf2ac95e0 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -35,13 +35,14 @@ namespace RemoteSynchronization { public class Program { + private const bool DEFAULT_DRY_RUN = false; private const bool DEFAULT_VERIFY = false; public static async Task Main(string[] args) { var src_arg = new Argument(name: "backend_src", description: "The source backend string"); var dst_arg = new Argument(name: "backend_dst", description: "The destination backend string"); - var dry_run_opt = new Option(aliases: ["--dry-run", "-d"], description: "Do not actually write or delete files"); + var dry_run_opt = new Option(aliases: ["--dry-run", "-d"], description: "Do not actually write or delete files", getDefaultValue: () => DEFAULT_DRY_RUN); var src_opts = OptionWithMultipleTokens(aliases: ["--src-options"], description: "Options for the source backend"); var dst_opts = OptionWithMultipleTokens(aliases: ["--dst-options"], description: "Options for the destination backend"); var verify_opt = new Option(aliases: ["--verify"], description: "Verify the files after copying", getDefaultValue: () => DEFAULT_VERIFY); @@ -68,6 +69,7 @@ namespace RemoteSynchronization private static async Task Run(string src, string dst, Dictionary options) { + var dry_run = options["dry-run"] as bool? ?? DEFAULT_DRY_RUN; var verify = options["verify"] as bool? ?? DEFAULT_VERIFY; var duplicati_options = new Dictionary() { From bcdc9f0af8cee7cc5034361b9f1cf3e95f9946c5 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 28 Jan 2025 11:23:16 +0100 Subject: [PATCH 043/362] Add retry option to Remote Synchronization Tool --- Tools/RemoteSynchronization/Program.cs | 35 ++++++++++++++++++++------ 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index cf2ac95e0..313cc2dcd 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2025, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a @@ -37,6 +37,7 @@ namespace RemoteSynchronization { private const bool DEFAULT_DRY_RUN = false; private const bool DEFAULT_VERIFY = false; + private const int DEFAULT_RETRY = 3; public static async Task Main(string[] args) { @@ -46,6 +47,7 @@ namespace RemoteSynchronization var src_opts = OptionWithMultipleTokens(aliases: ["--src-options"], description: "Options for the source backend"); var dst_opts = OptionWithMultipleTokens(aliases: ["--dst-options"], description: "Options for the destination backend"); var verify_opt = new Option(aliases: ["--verify"], description: "Verify the files after copying", getDefaultValue: () => DEFAULT_VERIFY); + var retry_opt = new Option(aliases: ["--retry"], description: "Number of times to retry on errors", getDefaultValue: () => DEFAULT_RETRY) { Arity = ArgumentArity.ExactlyOne }; var root_cmd = new RootCommand("Remote Synchronization Tool"); root_cmd.AddArgument(src_arg); @@ -54,6 +56,7 @@ namespace RemoteSynchronization root_cmd.AddOption(src_opts); root_cmd.AddOption(dst_opts); root_cmd.AddOption(verify_opt); + root_cmd.AddOption(retry_opt); root_cmd.SetHandler((InvocationContext ctx) => { @@ -71,6 +74,7 @@ namespace RemoteSynchronization { var dry_run = options["dry-run"] as bool? ?? DEFAULT_DRY_RUN; var verify = options["verify"] as bool? ?? DEFAULT_VERIFY; + var retries = options["retry"] as int? ?? DEFAULT_RETRY; var duplicati_options = new Dictionary() { ["dry-run"] = dry_run.ToString() @@ -102,8 +106,24 @@ namespace RemoteSynchronization if (copy_errors.Any()) { - Console.WriteLine($"Could not copy {copy_errors.Count()} files: {string.Join(", ", copy_errors)}"); - return copy_errors.Count(); + if (retries > 0) + { + Console.WriteLine($"Retrying {retries} more times to copy the {copy_errors.Count()} files that failed"); + for (int i = 0; i < retries; i++) + { + Thread.Sleep(5000); // Wait 5 seconds before retrying + (copied, copy_errors) = await CopyAsync(b1s, b2s, copy_errors, dry_run, verify); + Console.WriteLine($"Copied {copied} files from {src} to {dst}"); + if (!copy_errors.Any()) + break; + } + } + + if (copy_errors.Any()) + { + Console.WriteLine($"Could not copy {copy_errors.Count()} files: {string.Join(", ", copy_errors)}"); + return copy_errors.Count(); + } } if (verify) @@ -124,17 +144,16 @@ namespace RemoteSynchronization // TODO Force parameter // TODO Progress reporting // TODO Logging - // TODO Retry on errors // check database consistency. I.e. have both databases, check that the block, volume, files, etc match up. // introduce these checks as a post processing step? Especially the database consistency check, as that is often recreated from the index files. // TODO This tool shouldn't handle it, but for convenience, it should support making the seperate call to the regular Duplicati on the destination backend, which alread carries this functionality. // Forcefully synchronize the remote backends - private static async Task<(long, IEnumerable)> CopyAsync(IStreamingBackend b_src, IStreamingBackend b_dst, IEnumerable files, bool dry_run, bool verify) + private static async Task<(long, IEnumerable)> CopyAsync(IStreamingBackend b_src, IStreamingBackend b_dst, IEnumerable files, bool dry_run, bool verify) { long successful_copies = 0; - List errors = []; + List errors = []; using var s_src = new MemoryStream(); using var s_dst = new MemoryStream(); foreach (var f in files) @@ -155,7 +174,7 @@ namespace RemoteSynchronization if (s_src.Length != s_dst.Length || !s_src.ToArray().SequenceEqual(s_dst.ToArray())) { Console.WriteLine($"Error verifying {f.Name}: The file was not copied correctly"); - errors.Add(f.Name); + errors.Add(f); } s_dst.SetLength(0); } @@ -166,7 +185,7 @@ namespace RemoteSynchronization catch (Exception e) { Console.WriteLine($"Error copying {f.Name}: {e.Message}"); - errors.Add(f.Name); + errors.Add(f); } } return (successful_copies, errors); From fac12ba77242a351dc3bdc131aa8e167a37b2e8e Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 28 Jan 2025 13:01:35 +0100 Subject: [PATCH 044/362] Further consolidated the data folder. Added support for `--portable-mode` and `--server-datafolder` for the CLI, ServerUtil, and Agent. Fixed a few places where file reads were not cached properly. Moved some responsibilities of UpdaterManager into DataFolderManager. This now supports storing `machineid.txt` and `installation.txt` inside the path pointed to by `--server-datafolder` or alternatively with `--portable-mode`. It is possible that some fringe backwards compatibility is lost with this update. Fixed an issue with AutoUpdater crashing on check/download. This fixes #5902 --- Duplicati/Agent/Program.cs | 41 ++- Duplicati/Agent/Settings.cs | 3 +- Duplicati/CommandLine/CLI/Help.cs | 40 +-- Duplicati/CommandLine/CLI/Program.cs | 66 ++-- Duplicati/CommandLine/CLI/Strings.cs | 44 ++- .../CommandLine/ServerUtil/Connection.cs | 12 +- Duplicati/CommandLine/ServerUtil/Settings.cs | 21 +- .../CommandLine/ServerUtil/SettingsBinder.cs | 9 +- .../GUI/Duplicati.GUI.TrayIcon/Program.cs | 41 +-- .../Library/AutoUpdater/AutoUpdateSettings.cs | 329 +++++++++++------- ...atabaseLocator.cs => DataFolderLocator.cs} | 38 +- .../Library/AutoUpdater/DataFolderManager.cs | 246 +++++++++++++ .../AutoUpdater/PreloadSettingsLoader.cs | 44 +-- Duplicati/Library/AutoUpdater/Program.cs | 3 +- .../Library/AutoUpdater/UpdaterManager.cs | 212 +++++------ Duplicati/Library/Common/IO/Util.cs | 51 ++- Duplicati/Library/Main/CLIDatabaseLocator.cs | 11 +- Duplicati/Library/Main/Controller.cs | 37 +- .../Main/Operation/SystemInfoHandler.cs | 4 +- Duplicati/Library/Main/Options.cs | 6 +- .../Library/Modules/Builtin/ReportHelper.cs | 6 +- .../RemoteControl/RegisterForRemote.cs | 9 +- Duplicati/Library/UsageReporter/ReportSet.cs | 40 +-- .../UsageReporter/ReportSetUploader.cs | 50 ++- .../Library/Utility/HttpClientExtensions.cs | 30 +- Duplicati/Library/Utility/HttpClientHelper.cs | 24 +- Duplicati/License/VersionNumbers.cs | 80 ++--- Duplicati/Server/Program.cs | 60 +--- .../Services/SystemInfoProvider.cs | 14 +- 29 files changed, 863 insertions(+), 708 deletions(-) rename Duplicati/Library/AutoUpdater/{DatabaseLocator.cs => DataFolderLocator.cs} (72%) create mode 100644 Duplicati/Library/AutoUpdater/DataFolderManager.cs diff --git a/Duplicati/Agent/Program.cs b/Duplicati/Agent/Program.cs index 285784511..ef4e80b16 100644 --- a/Duplicati/Agent/Program.cs +++ b/Duplicati/Agent/Program.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System.CommandLine; @@ -25,6 +25,7 @@ using System.CommandLine.NamingConventionBinder; using System.CommandLine.Parsing; using System.Reactive.Linq; using System.Security.Cryptography; +using Duplicati.Library.AutoUpdater; using Duplicati.Library.Encryption; using Duplicati.Library.Interface; using Duplicati.Library.Logging; @@ -123,6 +124,8 @@ public static class Program new Option("--secret-provider", description: "The secret provider to use", getDefaultValue: () => null), new Option("--secret-provider-cache", description: "The secret provider cache level", getDefaultValue: () => SecretProviderHelper.CachingLevel.None), new Option("--secret-provider-pattern", description: "The secret provider pattern", getDefaultValue: () => SecretProviderHelper.DEFAULT_PATTERN), + new Option($"--{DataFolderManager.PORTABLE_MODE_OPTION}", description: "Use portable mode for locating the database and storing configuration", getDefaultValue: () => DataFolderManager.PORTABLE_MODE), + new Option($"--{DataFolderManager.SERVER_DATAFOLDER_OPTION}", description: "The datafolder to use for locating the database and storing configuration", getDefaultValue: () => new DirectoryInfo(DataFolderManager.DATAFOLDER)), }; runcmd.Handler = CommandHandler.Create(RunAgent); diff --git a/Duplicati/Agent/Settings.cs b/Duplicati/Agent/Settings.cs index 6785612a8..3788eefa1 100644 --- a/Duplicati/Agent/Settings.cs +++ b/Duplicati/Agent/Settings.cs @@ -148,7 +148,8 @@ public sealed record Settings( /// The path to the settings file private static string GetSettingsPath() { - var folder = Library.AutoUpdater.DatabaseLocator.GetDefaultStorageFolderWithDebugSupport(DefaultSettingsFilename); + // Ideally, this should use DataFolderManager.DATAFOLDER, but we cannot due to backwards compatibility + var folder = Library.AutoUpdater.DataFolderLocator.GetDefaultStorageFolder(DefaultSettingsFilename); if (!Directory.Exists(folder)) Directory.CreateDirectory(folder); diff --git a/Duplicati/CommandLine/CLI/Help.cs b/Duplicati/CommandLine/CLI/Help.cs index cd431b0d4..2500a5495 100644 --- a/Duplicati/CommandLine/CLI/Help.cs +++ b/Duplicati/CommandLine/CLI/Help.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -116,7 +116,7 @@ namespace Duplicati.CommandLine Library.Main.Options opts = new Library.Main.Options(new Dictionary()); tp = tp.Replace("%CLI_EXE%", PackageHelper.GetExecutableName(PackageHelper.NamedExecutable.CommandLine)); - tp = tp.Replace("%VERSION%", License.VersionNumbers.Version); + tp = tp.Replace("%VERSION%", License.VersionNumbers.VERSION_NAME); tp = tp.Replace("%BACKENDS%", string.Join(", ", Library.DynamicLoader.BackendLoader.Keys)); tp = tp.Replace("%APP_PATH%", Path.Combine(UpdaterManager.INSTALLATIONDIR, PackageHelper.GetExecutableName(PackageHelper.NamedExecutable.CommandLine))); tp = tp.Replace("%PATH_SEPARATOR%", System.IO.Path.PathSeparator.ToString()); diff --git a/Duplicati/CommandLine/CLI/Program.cs b/Duplicati/CommandLine/CLI/Program.cs index 66e7bb7a0..d62bbf527 100644 --- a/Duplicati/CommandLine/CLI/Program.cs +++ b/Duplicati/CommandLine/CLI/Program.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -181,6 +181,10 @@ namespace Duplicati.CommandLine } } + // Don't pass these options into the controller + options.Remove(Library.AutoUpdater.DataFolderManager.SERVER_DATAFOLDER_OPTION); + options.Remove(Library.AutoUpdater.DataFolderManager.PORTABLE_MODE_OPTION); + if (!options.ContainsKey("passphrase")) if (!string.IsNullOrEmpty(System.Environment.GetEnvironmentVariable("PASSPHRASE"))) options["passphrase"] = System.Environment.GetEnvironmentVariable("PASSPHRASE"); @@ -283,20 +287,16 @@ namespace Duplicati.CommandLine } } - public static IList SupportedOptions - { - get - { - return new List(new Library.Interface.ICommandLineArgument[] { - new Library.Interface.CommandLineArgument("parameters-file", Library.Interface.CommandLineArgument.ArgumentType.Path, Strings.Program.ParametersFileOptionShort, Strings.Program.ParametersFileOptionLong2, "", new string[] {"parameter-file", "parameterfile"}), - new Library.Interface.CommandLineArgument("include", Library.Interface.CommandLineArgument.ArgumentType.String, Strings.Program.IncludeShort, Strings.Program.IncludeLong), - new Library.Interface.CommandLineArgument("exclude", Library.Interface.CommandLineArgument.ArgumentType.String, Strings.Program.ExcludeShort, Strings.Program.ExcludeLong), - new Library.Interface.CommandLineArgument("control-files", Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.Program.ControlFilesOptionShort, Strings.Program.ControlFilesOptionLong, "false"), - new Library.Interface.CommandLineArgument("quiet-console", Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.Program.QuietConsoleOptionShort, Strings.Program.QuietConsoleOptionLong, "false"), - new Library.Interface.CommandLineArgument("auto-update", Library.Interface.CommandLineArgument.ArgumentType.Boolean, LC.L("Toggle automatic updates"), LC.L("Set this option if you prefer to have the commandline version automatically update"), "false"), - }); - } - } + public static IList SupportedOptions => [ + new CommandLineArgument("parameters-file", CommandLineArgument.ArgumentType.Path, Strings.Program.ParametersFileOptionShort, Strings.Program.ParametersFileOptionLong2, "", new string[] {"parameter-file", "parameterfile"}), + new CommandLineArgument("include", CommandLineArgument.ArgumentType.String, Strings.Program.IncludeShort, Strings.Program.IncludeLong), + new CommandLineArgument("exclude", CommandLineArgument.ArgumentType.String, Strings.Program.ExcludeShort, Strings.Program.ExcludeLong), + new CommandLineArgument("control-files", CommandLineArgument.ArgumentType.Boolean, Strings.Program.ControlFilesOptionShort, Strings.Program.ControlFilesOptionLong, "false"), + new CommandLineArgument("quiet-console", CommandLineArgument.ArgumentType.Boolean, Strings.Program.QuietConsoleOptionShort, Strings.Program.QuietConsoleOptionLong, "false"), + new CommandLineArgument("auto-update", CommandLineArgument.ArgumentType.Boolean, Strings.Program.AutoUpdateOptionShort, Strings.Program.AutoUpdateOptionLong, "false"), + new CommandLineArgument(DataFolderManager.PORTABLE_MODE_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.Program.PortableModeOptionShort, Strings.Program.PortableModeOptionLong, DataFolderManager.PORTABLE_MODE.ToString().ToLowerInvariant()), + new CommandLineArgument(DataFolderManager.SERVER_DATAFOLDER_OPTION, CommandLineArgument.ArgumentType.Path, Strings.Program.DataFolderOptionShort, Strings.Program.DataFolderOptionLong, DataFolderManager.DATAFOLDER), + ]; private static bool ReadOptionsFromFile(TextWriter outwriter, string filename, ref Library.Utility.IFilter filter, List cargs, Dictionary options) { diff --git a/Duplicati/CommandLine/CLI/Strings.cs b/Duplicati/CommandLine/CLI/Strings.cs index d7a3a5127..c63e347aa 100644 --- a/Duplicati/CommandLine/CLI/Strings.cs +++ b/Duplicati/CommandLine/CLI/Strings.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Localization.Short; @@ -54,6 +54,12 @@ namespace Duplicati.CommandLine.Strings public static string QuietConsoleOptionLong { get { return LC.L(@"If this option is set, progress reports and other messages that would normally go to the console will be redirected to the log."); } } public static string QuietConsoleOptionShort { get { return LC.L(@"Disable console output"); } } public static string SkippingSourceArgumentsOnNonBackupOperation { get { return @"The --source argument was specified in the parameter file, but the current operation is not a backup operation, so the argument is ignored."; } } + public static string AutoUpdateOptionShort { get { return LC.L(@"Toggle automatic updates"); } } + public static string AutoUpdateOptionLong { get { return LC.L(@"Set this option if you prefer to have the commandline version automatically update"); } } + public static string PortableModeOptionShort { get { return LC.L(@"Use portable mode"); } } + public static string PortableModeOptionLong { get { return LC.L(@"If this option is set, the configuration files will be stored in a data subfolder of the Duplicati installation folder. This is useful for running from a USB stick or other portable media."); } } + public static string DataFolderOptionShort { get { return LC.L(@"Data folder"); } } + public static string DataFolderOptionLong { get { return LC.L(@"The folder where data is stored. This is the folder where the database and other files are stored."); } } // ReSharper disable once UnusedMember.Global // This is a placeholder message that is intended to be used with the code diff --git a/Duplicati/CommandLine/ServerUtil/Connection.cs b/Duplicati/CommandLine/ServerUtil/Connection.cs index 4e0d42e97..92d05aee6 100644 --- a/Duplicati/CommandLine/ServerUtil/Connection.cs +++ b/Duplicati/CommandLine/ServerUtil/Connection.cs @@ -21,6 +21,7 @@ using System.Net.Http.Json; using System.Net.Security; using System.Text.Json; +using Duplicati.Library.AutoUpdater; namespace Duplicati.CommandLine.ServerUtil; @@ -185,10 +186,9 @@ public class Connection try { var opts = new Dictionary(); - if (!string.IsNullOrWhiteSpace(settings.ServerDatafolder)) - opts.Add("server-datafolder", settings.ServerDatafolder); - - if (File.Exists(Path.Combine(Server.Program.GetDataFolderPath(opts), Server.Program.SERVER_DATABASE_FILENAME))) + if (settings.Key != null) + opts["settings-encryption-key"] = settings.Key.Key; + if (File.Exists(Path.Combine(DataFolderManager.DATAFOLDER, DataFolderManager.SERVER_DATABASE_FILENAME))) { string? cfg = null; using (var connection = Duplicati.Server.Program.GetDatabaseConnection(opts, true)) @@ -220,9 +220,9 @@ public class Connection return CreateConnectionWithClient(client, accessToken); } } - else if (!string.IsNullOrWhiteSpace(settings.ServerDatafolder)) + else if (!string.IsNullOrWhiteSpace(DataFolderManager.DATAFOLDER)) { - Console.WriteLine($"No database found in {settings.ServerDatafolder}"); + Console.WriteLine($"No database found in {DataFolderManager.DATAFOLDER}"); } } catch (Exception ex) diff --git a/Duplicati/CommandLine/ServerUtil/Settings.cs b/Duplicati/CommandLine/ServerUtil/Settings.cs index 6caacc2bf..1f4528663 100644 --- a/Duplicati/CommandLine/ServerUtil/Settings.cs +++ b/Duplicati/CommandLine/ServerUtil/Settings.cs @@ -19,6 +19,7 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System.Text.Json; +using Duplicati.Library.AutoUpdater; using Duplicati.Library.DynamicLoader; using Duplicati.Library.Encryption; using Duplicati.Library.Interface; @@ -33,7 +34,6 @@ namespace Duplicati.CommandLine.ServerUtil; /// The commandline password /// The saved refresh token /// The host url to connect to -/// The server datafolder for password-free connections /// The settings file where data is loaded/saved /// Whether to disable TLS/SSL certificate trust check /// The encryption key to use for the settings file @@ -44,7 +44,6 @@ public sealed record Settings( string? Password, string? RefreshToken, System.Uri HostUrl, - string? ServerDatafolder, string SettingsFile, bool Insecure, EncryptedFieldHelper.KeyInstance? Key, @@ -64,9 +63,16 @@ public sealed record Settings( System.Uri HostUrl, string? ServerDatafolder ); + + /// + /// Gets the default data folder for the settings file + /// + /// The filename to use + /// The default storage folder private static string GetDefaultStorageFolder(string filename) { - var folder = Library.AutoUpdater.DatabaseLocator.GetDefaultStorageFolderWithDebugSupport(filename); + // Ideally, this should use DataFolderManager.DATAFOLDER, but we cannot due to backwards compatibility + var folder = DataFolderLocator.GetDefaultStorageFolder(filename); if (!Directory.Exists(folder)) Directory.CreateDirectory(folder); @@ -78,7 +84,6 @@ public sealed record Settings( /// /// The password to use /// The host URL to use - /// The server data folder to use /// The settings file to use /// Whether to disable TLS/SSL certificate trust check /// The encryption key to use @@ -87,7 +92,7 @@ public sealed record Settings( /// The secret provider pattern to use /// The SHA1 hash of the host certificate to accept /// The loaded settings - public static Settings Load(string? password, System.Uri? hostUrl, string? serverDataFolder, string settingsFile, bool insecure, string? settingsPassphrase, string? secretProvider, SecretProviderHelper.CachingLevel secretProviderCache, string secretProviderPattern, string? acceptedHostCertificate) + public static Settings Load(string? password, System.Uri? hostUrl, string settingsFile, bool insecure, string? settingsPassphrase, string? secretProvider, SecretProviderHelper.CachingLevel secretProviderCache, string secretProviderPattern, string? acceptedHostCertificate) { hostUrl ??= new System.Uri("http://localhost:8200"); @@ -115,9 +120,6 @@ public sealed record Settings( settingsPassphrase = opts["settings-encryption-key"]; } - if (string.IsNullOrWhiteSpace(serverDataFolder)) - serverDataFolder = GetDefaultStorageFolder("Duplicati-server.sqlite"); - if (!string.IsNullOrWhiteSpace(settingsFile) && !Path.IsPathRooted(settingsFile)) settingsFile = Path.Combine(GetDefaultStorageFolder(settingsFile), settingsFile); @@ -129,7 +131,6 @@ public sealed record Settings( password, persistedSettings?.RefreshToken, hostUrl, - serverDataFolder, settingsFile, insecure, key, @@ -175,7 +176,7 @@ public sealed record Settings( File.WriteAllText(SettingsFile, JsonSerializer.Serialize(LoadSettings(SettingsFile, thisKey) .Where(x => x.HostUrl != HostUrl) - .Append(new PersistedSettings(RefreshToken, HostUrl, ServerDatafolder)) + .Append(new PersistedSettings(RefreshToken, HostUrl, DataFolderManager.DATAFOLDER)) .Select(x => x with { RefreshToken = string.IsNullOrWhiteSpace(x.RefreshToken) || thisKey == null diff --git a/Duplicati/CommandLine/ServerUtil/SettingsBinder.cs b/Duplicati/CommandLine/ServerUtil/SettingsBinder.cs index fb72f2760..e27483d7e 100644 --- a/Duplicati/CommandLine/ServerUtil/SettingsBinder.cs +++ b/Duplicati/CommandLine/ServerUtil/SettingsBinder.cs @@ -20,6 +20,7 @@ // DEALINGS IN THE SOFTWARE. using System.CommandLine; using System.CommandLine.Binding; +using Duplicati.Library.AutoUpdater; using Duplicati.Library.Main; namespace Duplicati.CommandLine.ServerUtil; @@ -40,7 +41,11 @@ public class SettingsBinder : BinderBase /// /// The server datafolder option. /// - public static readonly Option serverDatafolderOption = new Option("--server-datafolder", description: "The server datafolder to use for locating the database", getDefaultValue: () => null); + public static readonly Option serverDatafolderOption = new Option($"--{DataFolderManager.SERVER_DATAFOLDER_OPTION}", description: "The server datafolder to use for locating the database and storing configuration", getDefaultValue: () => new DirectoryInfo(DataFolderManager.DATAFOLDER)); + /// + /// The server portable mode option. + /// + public static readonly Option portableModeOption = new Option($"--{DataFolderManager.PORTABLE_MODE_OPTION}", description: "Use portable mode for locating the database and storing configuration", getDefaultValue: () => DataFolderManager.PORTABLE_MODE); /// /// The settings file option. /// @@ -84,6 +89,7 @@ public class SettingsBinder : BinderBase rootCommand.AddGlobalOption(passwordOption); rootCommand.AddGlobalOption(hostUrlOption); rootCommand.AddGlobalOption(serverDatafolderOption); + rootCommand.AddGlobalOption(portableModeOption); rootCommand.AddGlobalOption(settingsFileOption); rootCommand.AddGlobalOption(insecureOption); rootCommand.AddGlobalOption(settingsEncryptionKeyOption); @@ -103,7 +109,6 @@ public class SettingsBinder : BinderBase Settings.Load( bindingContext.ParseResult.GetValueForOption(passwordOption), bindingContext.ParseResult.GetValueForOption(hostUrlOption), - bindingContext.ParseResult.GetValueForOption(serverDatafolderOption)?.FullName, bindingContext.ParseResult.GetValueForOption(settingsFileOption)?.FullName ?? "settings.json", bindingContext.ParseResult.GetValueForOption(insecureOption), bindingContext.ParseResult.GetValueForOption(settingsEncryptionKeyOption) ?? Environment.GetEnvironmentVariable(Library.Encryption.EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME), diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs index de14695d5..fac67c7ad 100644 --- a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs +++ b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -24,6 +24,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; +using Duplicati.Library.AutoUpdater; using Duplicati.Library.Interface; using Duplicati.Library.RestAPI; using Duplicati.Server; @@ -154,7 +155,7 @@ namespace Duplicati.GUI.TrayIcon } else if (Library.Utility.Utility.ParseBoolOption(options, READCONFIGFROMDB_OPTION)) { - if (File.Exists(Path.Combine(Server.Program.GetDataFolderPath(options), Server.Program.SERVER_DATABASE_FILENAME))) + if (File.Exists(Path.Combine(DataFolderManager.DATAFOLDER, DataFolderManager.SERVER_DATABASE_FILENAME))) { passwordSource = PasswordSource.Database; databaseConnection = Server.Program.GetDatabaseConnection(options, true); diff --git a/Duplicati/Library/AutoUpdater/AutoUpdateSettings.cs b/Duplicati/Library/AutoUpdater/AutoUpdateSettings.cs index 020d598ce..b6b66e191 100644 --- a/Duplicati/Library/AutoUpdater/AutoUpdateSettings.cs +++ b/Duplicati/Library/AutoUpdater/AutoUpdateSettings.cs @@ -1,24 +1,26 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +#nullable enable + using System; using System.Collections.Generic; using System.Linq; @@ -27,28 +29,78 @@ using System.Text.RegularExpressions; namespace Duplicati.Library.AutoUpdater { + /// + /// Contains settings for the auto-updater + /// public static class AutoUpdateSettings { - private static readonly Dictionary _cache = new Dictionary(); + /// + /// The appname file resource name + /// private const string APP_NAME = "AutoUpdateAppName.txt"; + /// + /// The update URL file resource name + /// private const string UPDATE_URL = "AutoUpdateURL.txt"; + /// + /// The update key file resource name + /// private const string UPDATE_KEY = "AutoUpdateSignKeys.txt"; + /// + /// The update channel file resource name + /// private const string UPDATE_CHANNEL = "AutoUpdateBuildChannel.txt"; + /// + /// The update install file template resource name + /// private const string UPDATE_INSTALL_FILE = "AutoUpdateInstallIDTemplate.txt"; + /// + /// The update machine file template resource name + /// private const string UPDATE_MACHINE_FILE = "AutoUpdateMachineIDTemplate.txt"; + /// + /// The OEM file name + /// private const string OEM_APP_NAME = "oem-app-name.txt"; + /// + /// The OEM update URL file name + /// private const string OEM_UPDATE_URL = "oem-update-url.txt"; + /// + /// The OEM update key file name + /// private const string OEM_UPDATE_KEY = "oem-update-key.txt"; + /// + /// The OEM update install template file name + /// private const string OEM_UPDATE_INSTALL_FILE = "oem-update-installid.txt"; + /// + /// The update URL environment variable name template + /// public const string UPDATEURL_ENVNAME_TEMPLATE = "AUTOUPDATER_{0}_URLS"; + /// + /// The update channel environment variable name template + /// public const string UPDATECHANNEL_ENVNAME_TEMPLATE = "AUTOUPDATER_{0}_CHANNEL"; + /// + /// The package prefix group name in the regex + /// internal const string MATCH_UPDATE_URL_PREFIX_GROUP = "prefix"; + /// + /// The channel group name in the regex + /// internal const string MATCH_UPDATE_URL_CHANNEL_GROUP = "channel"; + /// + /// The filename group name in the regex + /// internal const string MATCH_UPDATE_URL_FILENAME_GROUP = "filename"; + /// + /// The regex to match the auto-update URL and replace the channel + /// internal static readonly Regex MATCH_AUTOUPDATE_URL = new Regex(string.Format( "(?<{0}>.+)(?<{1}>{3})(?<{2}>/([^/]+).manifest)", @@ -57,27 +109,21 @@ namespace Duplicati.Library.AutoUpdater MATCH_UPDATE_URL_FILENAME_GROUP, string.Join("|", Enum.GetNames(typeof(ReleaseType)).Union(new[] { "preview" }))), RegexOptions.Compiled | RegexOptions.IgnoreCase); - - static AutoUpdateSettings() + /// + /// Reads a resource text from the assembly, with an optional OEM filename override + /// + /// The embedded resource name + /// The OEM filename override + /// The text of the resource + private static string ReadResourceText(string name, string? oemname) { - ReadResourceText(APP_NAME, OEM_APP_NAME); - ReadResourceText(UPDATE_URL, OEM_UPDATE_URL); - ReadResourceText(UPDATE_KEY, OEM_UPDATE_KEY); - ReadResourceText(UPDATE_INSTALL_FILE, OEM_UPDATE_INSTALL_FILE); - ReadResourceText(UPDATE_MACHINE_FILE, null); - ReadResourceText(UPDATE_CHANNEL, null); - } - - private static string ReadResourceText(string name, string oemname) - { - // First try to read from _cache - if (_cache.TryGetValue(name, out string result)) - return result; - + string? result = null; try { - using (var rd = new System.IO.StreamReader(System.Reflection.Assembly.GetExecutingAssembly().GetManifestResourceStream(typeof(AutoUpdateSettings), name))) - result = rd.ReadToEnd(); + using (var rs = System.Reflection.Assembly.GetExecutingAssembly().GetManifestResourceStream(typeof(AutoUpdateSettings), name)) + if (rs != null) + using (var rd = new System.IO.StreamReader(rs)) + result = rd.ReadToEnd(); } catch (Exception ex) { @@ -99,121 +145,154 @@ namespace Duplicati.Library.AutoUpdater else result = result.Trim(); - _cache[name] = result; return result; } - public static string[] URLs - { - get - { - if (UsesAlternateURLs) - return Environment.GetEnvironmentVariable(string.Format(UPDATEURL_ENVNAME_TEMPLATE, AppName)).Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries); - else - return ReadResourceText(UPDATE_URL, OEM_UPDATE_URL).Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries); - } - } + /// + /// The URLs to check for updates + /// + public static string[] URLs => _URLs.Value; - public static bool UsesAlternateURLs + /// + /// The alternate URLs to check for updates, lazy evaluated + /// + private static readonly Lazy _alternateURLs = new Lazy(() => { - get - { - return !string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable(string.Format(UPDATEURL_ENVNAME_TEMPLATE, AppName))); - } - } + return Environment.GetEnvironmentVariable(string.Format(UPDATEURL_ENVNAME_TEMPLATE, AppName)); + }); - public static ReleaseType DefaultUpdateChannel + /// + /// The URLs to check for updates, lazy evaluated + /// + private static readonly Lazy _URLs = new Lazy(() => { - get - { - var channelstring = Environment.GetEnvironmentVariable(string.Format(UPDATECHANNEL_ENVNAME_TEMPLATE, AppName)); + if (!string.IsNullOrWhiteSpace(_alternateURLs.Value)) + return _alternateURLs.Value.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries); + else + return ReadResourceText(UPDATE_URL, OEM_UPDATE_URL).Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries); + }); - if (UsesAlternateURLs && string.IsNullOrWhiteSpace(channelstring)) + /// + /// Indicates if alternate URLs are in use + /// + public static bool UsesAlternateURLs => !string.IsNullOrWhiteSpace(_alternateURLs.Value); + + /// + /// The default update channel + /// + public static ReleaseType DefaultUpdateChannel => _defaultUpdateChannel.Value; + + /// + /// The default update channel, lazy evaluated + /// + private static readonly Lazy _defaultUpdateChannel = new Lazy(() => + { + var channelstring = Environment.GetEnvironmentVariable(string.Format(UPDATECHANNEL_ENVNAME_TEMPLATE, AppName)); + + if (UsesAlternateURLs && string.IsNullOrWhiteSpace(channelstring)) + { + foreach (var url in URLs) { - foreach (var url in URLs) + var match = AutoUpdateSettings.MATCH_AUTOUPDATE_URL.Match(url); + if (match.Success) { - var match = AutoUpdateSettings.MATCH_AUTOUPDATE_URL.Match(url); - if (match.Success) - { - channelstring = match.Groups[AutoUpdateSettings.MATCH_UPDATE_URL_CHANNEL_GROUP].Value; - break; - } + channelstring = match.Groups[AutoUpdateSettings.MATCH_UPDATE_URL_CHANNEL_GROUP].Value; + break; } } - - if (string.IsNullOrWhiteSpace(channelstring)) - channelstring = BuildUpdateChannel; - - // Update from older builds - if (string.Equals(channelstring, "preview", StringComparison.OrdinalIgnoreCase)) - channelstring = ReleaseType.Experimental.ToString(); - if (string.Equals(channelstring, "rene", StringComparison.OrdinalIgnoreCase)) - channelstring = ReleaseType.Canary.ToString(); - - ReleaseType rt; - if (!Enum.TryParse(channelstring, true, out rt)) - rt = ReleaseType.Stable; - - return rt; } - } - public static string AppName - { - get { return ReadResourceText(APP_NAME, OEM_APP_NAME); } - } + if (string.IsNullOrWhiteSpace(channelstring)) + channelstring = BuildUpdateChannel; - public static string BuildUpdateChannel - { - get { return ReadResourceText(UPDATE_CHANNEL, null); } - } + // Update from older builds + if (string.Equals(channelstring, "preview", StringComparison.OrdinalIgnoreCase)) + channelstring = ReleaseType.Experimental.ToString(); + if (string.Equals(channelstring, "rene", StringComparison.OrdinalIgnoreCase)) + channelstring = ReleaseType.Canary.ToString(); - public static string UpdateInstallFileText - { - get { return string.Format(ReadResourceText(UPDATE_INSTALL_FILE, OEM_UPDATE_INSTALL_FILE), Guid.NewGuid().ToString("N")); } - } + ReleaseType rt; + if (!Enum.TryParse(channelstring, true, out rt)) + rt = ReleaseType.Stable; + return rt; + }); + + /// + /// The application name + /// + public static string AppName => _appName.Value; + + /// + /// The application name, lazy evaluated + /// + private static readonly Lazy _appName = new Lazy(() => ReadResourceText(APP_NAME, OEM_APP_NAME)); + + /// + /// The build update channel + /// + public static string BuildUpdateChannel => _buildUpdateChannel.Value; + + /// + /// The build update channel, lazy evaluated + /// + private static readonly Lazy _buildUpdateChannel = new Lazy(() => ReadResourceText(UPDATE_CHANNEL, null)); + + /// + /// The text for the update install file + /// + public static string UpdateInstallFileText => string.Format(ReadResourceText(UPDATE_INSTALL_FILE, OEM_UPDATE_INSTALL_FILE), Guid.NewGuid().ToString("N")); + + /// + /// Updates the machine file text with the machine ID + /// + /// The machine ID to use + /// The updated text public static string UpdateMachineFileText(string machineid) => string.Format(ReadResourceText(UPDATE_MACHINE_FILE, "{0}"), string.IsNullOrWhiteSpace(machineid) ? Guid.NewGuid().ToString("N") : machineid); - public static RSACryptoServiceProvider[] SignKeys + /// + /// The keys to use for signing + /// + public static RSACryptoServiceProvider[] SignKeys => _signKeys.Value; + + /// + /// The keys to use for signing, lazy evaluated + /// + private static readonly Lazy _signKeys = new Lazy(() => { - get + var keys = new List(); + + try { - var keys = new List(); + var src = ReadResourceText(UPDATE_KEY, OEM_UPDATE_KEY); - try + // Allow multiple keys, one per line + // For fallback, read the whole string as a key, in case there are old ones with line breaks + var keystrings = src.Split(new char[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries) + .Select(x => x.Trim()) + .Prepend(src.Trim()) + .Where(x => !string.IsNullOrWhiteSpace(x)) + .Distinct(); + + foreach (var str in keystrings) { - var src = ReadResourceText(UPDATE_KEY, OEM_UPDATE_KEY); - - // Allow multiple keys, one per line - // For fallback, read the whole string as a key, in case there are old ones with line breaks - - var keystrings = src.Split(new char[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries) - .Select(x => x.Trim()) - .Prepend(src.Trim()) - .Where(x => !string.IsNullOrWhiteSpace(x)) - .Distinct(); - - foreach (var str in keystrings) + try { - try - { - var key = new RSACryptoServiceProvider(); - key.FromXmlString(str); - keys.Add(key); - } - catch - { } + var key = new RSACryptoServiceProvider(); + key.FromXmlString(str); + keys.Add(key); } + catch + { } } - catch - { - } - - return keys.ToArray(); } - } + catch + { + } + + return keys.ToArray(); + }); } } diff --git a/Duplicati/Library/AutoUpdater/DatabaseLocator.cs b/Duplicati/Library/AutoUpdater/DataFolderLocator.cs similarity index 72% rename from Duplicati/Library/AutoUpdater/DatabaseLocator.cs rename to Duplicati/Library/AutoUpdater/DataFolderLocator.cs index 727d91b3b..e9264f2db 100644 --- a/Duplicati/Library/AutoUpdater/DatabaseLocator.cs +++ b/Duplicati/Library/AutoUpdater/DataFolderLocator.cs @@ -27,32 +27,11 @@ using System.Collections.Generic; namespace Duplicati.Library.AutoUpdater; -public static class DatabaseLocator +/// +/// Helper class to locate the default storage folder for the application +/// +public static class DataFolderLocator { - /// - /// The log tag for this class - /// - public static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(DatabaseLocator)); - - /// - /// Finds a default storage folder, using the operating system specific locations. - /// The targetfilename is used to detect locations that are used in previous versions. - /// If the targetfilename is found in an old location, but not the current, the old location is used. - /// If running with DEBUG defined, the storage folder is placed in the same folder as the executable. - /// Note that the folder is not created, only the path is returned. - /// - /// The filename to look for - /// The name of the application - /// The default storage folder - public static string GetDefaultStorageFolderWithDebugSupport(string targetfilename, string appName = "Duplicati") - { -#if DEBUG - return System.IO.Path.GetDirectoryName(typeof(DatabaseLocator).Assembly.Location) ?? string.Empty; -#else - return GetDefaultStorageFolderDirect(targetfilename, appName); -#endif - } - /// /// Finds a default storage folder, using the operating system specific locations. /// The targetfilename is used to detect locations that are used in previous versions. @@ -62,7 +41,7 @@ public static class DatabaseLocator /// The filename to look for /// The name of the application /// The default storage folder - public static string GetDefaultStorageFolderDirect(string targetfilename, string appName = "Duplicati") + public static string GetDefaultStorageFolder(string targetfilename, string appName = "Duplicati") { //Normal mode uses the systems "(Local) Application Data" folder // %LOCALAPPDATA% on Windows, ~/.config on Linux @@ -82,8 +61,7 @@ public static class DatabaseLocator }; // If %LOCALAPPDATA% is inside the Windows folder, prefer a LocalService folder instead - var windowsFolder = Library.Common.IO.Util.AppendDirSeparator(Environment.GetFolderPath(Environment.SpecialFolder.Windows)); - if (newlocation.StartsWith(windowsFolder, StringComparison.OrdinalIgnoreCase)) + if (Common.IO.Util.IsPathUnderWindowsFolder(newlocation)) { var userProfilesFolder = Library.Utility.SHGetFolder.UserProfilesFolder; if (!string.IsNullOrWhiteSpace(userProfilesFolder)) @@ -99,10 +77,6 @@ public static class DatabaseLocator // Use the most recent location found with content // If none are found, use the most recent location folder = matches.FirstOrDefault() ?? folderOrder.Last(); - - // Emit a warning if the database is stored in the Windows folder - if (folder.StartsWith(windowsFolder, StringComparison.OrdinalIgnoreCase)) - Logging.Log.WriteWarningMessage(LOGTAG, "DatabaseInWindowsFolder", null, "The database is stored in the Windows folder, this is not recommended as it will be deleted on Windows upgrades."); } if (OperatingSystem.IsMacOS()) diff --git a/Duplicati/Library/AutoUpdater/DataFolderManager.cs b/Duplicati/Library/AutoUpdater/DataFolderManager.cs new file mode 100644 index 000000000..e3d248510 --- /dev/null +++ b/Duplicati/Library/AutoUpdater/DataFolderManager.cs @@ -0,0 +1,246 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +using System; +using System.IO; +using System.Linq; +using Duplicati.Library.Common.IO; +using Duplicati.Library.Utility; + +namespace Duplicati.Library.AutoUpdater; + +/// +/// Manages the data folder for the application +/// +public static class DataFolderManager +{ + /// + /// The debug flag, set to true if the application is running in debug mode, + /// preventing littering of DEBUG statements in the code + /// +#if DEBUG + private static readonly bool DEBUG_MODE = true; +#else + private static readonly bool DEBUG_MODE = false; +#endif + + /// + /// The folder where the machine id is placed + /// + public static readonly string DATAFOLDER; + + /// + /// The installation ID filename stored in + /// + private const string INSTALL_FILE = "installation.txt"; + + /// + /// The machine ID filename stored in + /// + private const string MACHINE_FILE = "machineid.txt"; + + /// + /// The option name for portable mode + /// + public const string PORTABLE_MODE_OPTION = "portable-mode"; + /// + /// The option anme for the server data folder + /// + public const string SERVER_DATAFOLDER_OPTION = "server-datafolder"; + + /// + /// The app name to use for variables + /// + private static readonly string APPNAME = AutoUpdateSettings.AppName; + + /// + /// The name of the environment variable that allows overriding the path to the data folder used by Duplicati + /// + public static readonly string DATAFOLDER_ENV_NAME = $"{APPNAME}_HOME".ToUpperInvariant(); + + /// + /// Name of the database file + /// + public static readonly string SERVER_DATABASE_FILENAME = $"{APPNAME}-server.sqlite"; + + /// + /// Flag to indicate if the application is running in portable mode + /// + public static readonly bool PORTABLE_MODE; + + /// + /// Replication of the argument parsing from the main Duplicati codebase + /// + /// The option to extract + /// null if the option is not found, otherwise the value of the option + private static string? ExtractOptionSlim(string option) + { + var opt = $"--{option}"; + var args = Environment.GetCommandLineArgs().Skip(1).ToArray(); + var match = args.Select((token, index) => new { token, index }) + .LastOrDefault(x => + string.Equals(x.token, opt, StringComparison.OrdinalIgnoreCase) + || + x.token.StartsWith(opt + "=", StringComparison.OrdinalIgnoreCase) + ); + + // Not found, try the environment variable + if (string.IsNullOrWhiteSpace(match?.token)) + return Environment.GetEnvironmentVariable($"{AutoUpdateSettings.AppName}__{option.Replace('-', '_')}".ToUpperInvariant()); + + // Found in the form --option=value + if (match.token.StartsWith(opt + "=", StringComparison.OrdinalIgnoreCase)) + return match.token.Substring(opt.Length + 1).Trim('"'); + + // Found in the form --option value + if (match.index + 1 < args.Length) + { + var value = args[match.index + 1]; + if (!value.StartsWith("--")) + return value; + } + + // Found, but no value, just the option + return ""; + } + + /// + /// Replication of the boolean parsing from the main Duplicati codebase + /// + /// The value to parse + /// true if the value is a truthy value, otherwise false + private static bool ParseBoolSlim(string? value) + { + // In debug builds, we default to portable mode + if (value == null) + return DEBUG_MODE ? true : false; + + if ( + value.Equals("false", StringComparison.OrdinalIgnoreCase) + || value.Equals("0", StringComparison.OrdinalIgnoreCase) + || value.Equals("no", StringComparison.OrdinalIgnoreCase) + || value.Equals("off", StringComparison.OrdinalIgnoreCase) + ) + return false; + + return true; + } + + static DataFolderManager() + { + // Trigger portable mode, if the flag is set + PORTABLE_MODE = ParseBoolSlim(ExtractOptionSlim(PORTABLE_MODE_OPTION)); + + // The environment variable is a legacy setting + var envOverride = Environment.GetEnvironmentVariable(DATAFOLDER_ENV_NAME); + + // These are mainly supported by the Server + var datafolderArg = ExtractOptionSlim(SERVER_DATAFOLDER_OPTION); + + // Prefer the command line argument over the environment variable + if (!string.IsNullOrWhiteSpace(datafolderArg)) + { + DATAFOLDER = Util.AppendDirSeparator(Environment.ExpandEnvironmentVariables(datafolderArg).Trim('"')); + } + // Portable mode is prefered over the environment variable + else if (PORTABLE_MODE) + { + DATAFOLDER = DEBUG_MODE + // Use the build folder for portable mode in debug builds + ? UpdaterManager.INSTALLATIONDIR + // Use the data folder in the installation directory for portable mode + : Util.AppendDirSeparator(Path.Combine(UpdaterManager.INSTALLATIONDIR, "data")); + } + // Use the legacy environment variable, if set + else if (!string.IsNullOrWhiteSpace(envOverride)) + { + DATAFOLDER = Util.AppendDirSeparator(Environment.ExpandEnvironmentVariables(envOverride).Trim('"')); + } + // Use the default location + else + { + DATAFOLDER = Util.AppendDirSeparator(DataFolderLocator.GetDefaultStorageFolder(SERVER_DATABASE_FILENAME, APPNAME)); + } + + if (!Directory.Exists(DATAFOLDER)) + Directory.CreateDirectory(DATAFOLDER); + + if (!File.Exists(Path.Combine(DATAFOLDER, INSTALL_FILE))) + { + // In case there was already a machine id file from 2.0.8.1 or older, copy it to the new location + if (File.Exists(Path.Combine(DATAFOLDER, "updates", INSTALL_FILE))) + File.Copy(Path.Combine(DATAFOLDER, "updates", INSTALL_FILE), Path.Combine(DATAFOLDER, INSTALL_FILE), true); + else + File.WriteAllText(Path.Combine(DATAFOLDER, INSTALL_FILE), AutoUpdateSettings.UpdateInstallFileText); + } + + if (!File.Exists(Path.Combine(DATAFOLDER, MACHINE_FILE))) + File.WriteAllText(Path.Combine(DATAFOLDER, MACHINE_FILE), AutoUpdateSettings.UpdateMachineFileText(InstallID)); + } + + /// + /// The unique machine installation ID + /// + public static string InstallID => _installID.Value; + + /// + /// The unique machine ID, lazy evaluated + /// + private static readonly Lazy _installID = new(() => + { + try { return File.ReadAllLines(Path.Combine(DATAFOLDER!, INSTALL_FILE)).FirstOrDefault(x => !string.IsNullOrWhiteSpace(x))?.Trim() ?? ""; } + catch { } + + return ""; + }); + + /// + /// The unique machine ID + /// + public static string MachineID => _machineID.Value; + + /// + /// The unique machine ID, lazy evaluated + /// + private static readonly Lazy _machineID = new(() => + { + string? machinedId = null; + try { machinedId = File.ReadAllLines(Path.Combine(DATAFOLDER!, MACHINE_FILE)).FirstOrDefault(x => !string.IsNullOrWhiteSpace(x))?.Trim() ?? ""; } + catch { } + + return string.IsNullOrWhiteSpace(machinedId) + ? InstallID + : machinedId; + }); + + /// + /// The machine name, lazy evaluated + /// + private static readonly Lazy _machineName = new(MachineNameReader.GetMachineName); + + /// + /// The machine name + /// + public static readonly string MachineName = _machineName.Value; + +} \ No newline at end of file diff --git a/Duplicati/Library/AutoUpdater/PreloadSettingsLoader.cs b/Duplicati/Library/AutoUpdater/PreloadSettingsLoader.cs index 667a9dbce..f1e34cbfc 100644 --- a/Duplicati/Library/AutoUpdater/PreloadSettingsLoader.cs +++ b/Duplicati/Library/AutoUpdater/PreloadSettingsLoader.cs @@ -109,7 +109,7 @@ public static class PreloadSettingsLoader // User-context path for preload settings, // the same default path as where other data is stored - Path.Combine(UpdaterManager.DATAFOLDER, FILE_NAME), + Path.Combine(DataFolderManager.DATAFOLDER, FILE_NAME), } .Concat(PortablePreloadPaths) .Where(x => !string.IsNullOrEmpty(x)) @@ -132,7 +132,7 @@ public static class PreloadSettingsLoader /// The database settings public static void ConfigurePreloadSettings(ref string[] arguments, PackageHelper.NamedExecutable executable, out Dictionary dbsettings) { - var (env, args, db) = GetExecutableMergedSettings(executable, IsPortableModeRequested(arguments)); + var (env, args, db) = GetExecutableMergedSettings(executable, DataFolderManager.PORTABLE_MODE); dbsettings = db; ApplyEnvironmentVariables(env); @@ -147,46 +147,6 @@ public static class PreloadSettingsLoader private static string GetArgumentName(string arg) => arg.Split('=', 2)[0]; - /// - /// Helper function to check if portable mode is requested. - /// This is a rudimentary function that mimics the commandline parser, - /// which is not loaded during preload - /// - /// The arguments to check - /// true if portable mode is requested, false otherwise - private static bool IsPortableModeRequested(string[] args) - { - var candiate = args.Select((x, i) => (x, i)) - .Where(x => x.x != null) - .Where(x => x.x.Equals("--portable-mode", StringComparison.OrdinalIgnoreCase) || x.x.StartsWith("--portable-mode=", StringComparison.OrdinalIgnoreCase)) - .LastOrDefault(); - if (candiate.x == null) - return false; - - var value = (string?)null; - if (candiate.x.Contains("=")) - value = candiate.x.Split('=', 2)[1]; - else - { - value = args.ElementAtOrDefault(candiate.i + 1); - if (value == null || value.StartsWith("--")) - value = "true"; - } - - if (value == null) - return false; - - value = value.TrimStart('"').TrimEnd('"').ToLowerInvariant(); - return value switch - { - "1" => true, - "on" => true, - "true" => true, - "yes" => true, - _ => false, - }; - - } /// /// Gets the merged settings for the given executable diff --git a/Duplicati/Library/AutoUpdater/Program.cs b/Duplicati/Library/AutoUpdater/Program.cs index dbdd82bb9..ebfb6d949 100644 --- a/Duplicati/Library/AutoUpdater/Program.cs +++ b/Duplicati/Library/AutoUpdater/Program.cs @@ -53,6 +53,7 @@ namespace Duplicati.Library.AutoUpdater return 0; case "check": { + Console.WriteLine($"Checking for updates on channel {AutoUpdateSettings.DefaultUpdateChannel} ..."); var update = UpdaterManager.CheckForUpdate(); if (update == null) Console.WriteLine("No updates found"); @@ -116,7 +117,7 @@ namespace Duplicati.Library.AutoUpdater Console.WriteLine("{0} - Choose different channel than the default {1}, valid settings: {2}", string.Format(AutoUpdateSettings.UPDATECHANNEL_ENVNAME_TEMPLATE, AutoUpdateSettings.AppName), AutoUpdater.AutoUpdateSettings.DefaultUpdateChannel, string.Join(",", Enum.GetNames(typeof(ReleaseType)).Where(x => x != ReleaseType.Unknown.ToString()))); Console.WriteLine(); Console.WriteLine("Updates are downloaded from: {0}", string.Join(";", AutoUpdateSettings.URLs)); - Console.WriteLine("Machine settings are installed in: {0}", UpdaterManager.DATAFOLDER); + Console.WriteLine("Settings and configuration files are placed in: {0}", DataFolderManager.DATAFOLDER); Console.WriteLine("This version is \"{0}\" ({1}) and is installed in: {2}", UpdaterManager.SelfVersion.Displayname, UpdaterManager.SelfVersion.Version, UpdaterManager.INSTALLATIONDIR); Console.WriteLine(); } diff --git a/Duplicati/Library/AutoUpdater/UpdaterManager.cs b/Duplicati/Library/AutoUpdater/UpdaterManager.cs index a27866911..ce88ae5ad 100644 --- a/Duplicati/Library/AutoUpdater/UpdaterManager.cs +++ b/Duplicati/Library/AutoUpdater/UpdaterManager.cs @@ -19,6 +19,8 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +#nullable enable + using System; using System.Linq; using System.Collections.Generic; @@ -28,32 +30,32 @@ using System.Diagnostics; using System.Text.Json; using System.Net.Http; using System.Threading; +using Duplicati.Library.Common.IO; namespace Duplicati.Library.AutoUpdater { + /// + /// Handles operations related to updating the application + /// public static class UpdaterManager { /// /// The RSA key used to sign the manifest /// - private static readonly System.Security.Cryptography.RSA[] SIGN_KEYS = AutoUpdateSettings.SignKeys; + private static System.Security.Cryptography.RSA[] SIGN_KEYS => AutoUpdateSettings.SignKeys; /// /// Urls to check for updated packages /// - private static readonly string[] MANIFEST_URLS = AutoUpdateSettings.URLs; + private static string[] MANIFEST_URLS => AutoUpdateSettings.URLs; /// /// The app name to show /// - private static readonly string APPNAME = AutoUpdateSettings.AppName; + private static string APPNAME => AutoUpdateSettings.AppName; /// /// The version that the updater supports /// public const int SUPPORTED_PACKAGE_UPDATER_VERSION = 2; /// - /// The folder where the machine id is placed - /// - public static readonly string DATAFOLDER; - /// /// The directory where the program is running from /// public static readonly string INSTALLATIONDIR; @@ -62,6 +64,10 @@ namespace Duplicati.Library.AutoUpdater /// public static readonly bool DISABLE_UPDATE_CHECK = Debugger.IsAttached || Utility.Utility.ParseBool(Environment.GetEnvironmentVariable(string.Format(SKIPUPDATE_ENVNAME_TEMPLATE, APPNAME)), false); /// + /// The operating system display name + /// + public static readonly string OperatingSystemName = OperatingSystem.IsWindows() ? "Windows" : OperatingSystem.IsLinux() ? "Linux" : OperatingSystem.IsMacOS() ? "MacOS" : "Unknown"; + /// /// The update information for the running version /// public static readonly UpdateInfo SelfVersion; @@ -69,17 +75,13 @@ namespace Duplicati.Library.AutoUpdater /// /// Event trigger for errors on update /// - public static event Action OnError; + public static event Action? OnError; /// /// Common formatting string for date-time values /// private const string DATETIME_FORMAT = "yyyymmddhhMMss"; /// - /// The template for the environment variable name that allows an overriden root folder - /// - private const string UPDATEINSTALLDIR_ENVNAME_TEMPLATE = "AUTOUPDATER_{0}_UPDATE_ROOT"; - /// /// The template for the environment variable that toggles disabling updates /// public const string SKIPUPDATE_ENVNAME_TEMPLATE = "AUTOUPDATER_{0}_SKIP_UPDATE"; @@ -91,20 +93,11 @@ namespace Duplicati.Library.AutoUpdater /// The name of the file that contains the package type id, located in the folder /// private const string PACKAGE_TYPE_FILE = "package_type_id.txt"; - /// - /// The installation ID filename stored in - /// - private const string INSTALL_FILE = "installation.txt"; - - /// - /// The machine ID filename stored in - /// - private const string MACHINE_FILE = "machineid.txt"; /// /// Gets the last version found from an update /// - public static UpdateInfo LastUpdateCheckVersion { get; private set; } + public static UpdateInfo? LastUpdateCheckVersion { get; private set; } /// /// The default timeout in seconds for download operations @@ -122,40 +115,10 @@ namespace Duplicati.Library.AutoUpdater static UpdaterManager() { // Set the installation path - INSTALLATIONDIR = Path.GetDirectoryName(Utility.Utility.getEntryAssembly().Location); - - // Check for override - if (string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable(string.Format(UPDATEINSTALLDIR_ENVNAME_TEMPLATE, APPNAME)))) - { - DATAFOLDER = DatabaseLocator.GetDefaultStorageFolderWithDebugSupport(INSTALL_FILE, APPNAME); - } - else - { - // Use override, no checks - DATAFOLDER = Environment.ExpandEnvironmentVariables(Environment.GetEnvironmentVariable(string.Format(UPDATEINSTALLDIR_ENVNAME_TEMPLATE, APPNAME))); - } - - if (!string.IsNullOrWhiteSpace(DATAFOLDER)) - { - if (!Directory.Exists(DATAFOLDER)) - Directory.CreateDirectory(DATAFOLDER); - - if (!File.Exists(Path.Combine(DATAFOLDER, INSTALL_FILE))) - { - // In case there was already a machine id file from 2.0.8.1 or older, copy it to the new location - if (File.Exists(Path.Combine(DATAFOLDER, "updates", INSTALL_FILE))) - File.Copy(Path.Combine(DATAFOLDER, "updates", INSTALL_FILE), Path.Combine(DATAFOLDER, INSTALL_FILE), true); - else - File.WriteAllText(Path.Combine(DATAFOLDER, INSTALL_FILE), AutoUpdateSettings.UpdateInstallFileText); - } - - if (!File.Exists(Path.Combine(DATAFOLDER, MACHINE_FILE))) - File.WriteAllText(Path.Combine(DATAFOLDER, MACHINE_FILE), AutoUpdateSettings.UpdateMachineFileText(InstallID)); - - } + INSTALLATIONDIR = Util.AppendDirSeparator(Path.GetDirectoryName(Utility.Utility.getEntryAssembly().Location)); // Attempt to read the installed manifest file - UpdateInfo selfVersion = null; + UpdateInfo? selfVersion = null; try { selfVersion = ReadInstalledManifest(INSTALLATIONDIR); @@ -165,94 +128,62 @@ namespace Duplicati.Library.AutoUpdater } // In case the installed manifest is broken, try to set some sane values - if (selfVersion == null) - { - SelfVersion = new UpdateInfo( - MinimumCompatibleVersion: 1, - PackageUpdaterVersion: 1, - IncompatibleUpdateUrl: string.Empty, - GenericUpdatePageUrl: "https://duplicati.com/download", - UpdateSeverity: null, - ChangeInfo: null, - Packages: null, - Displayname: string.IsNullOrWhiteSpace(License.VersionNumbers.TAG) ? "Current" : License.VersionNumbers.TAG, - Version: System.Reflection.Assembly.GetExecutingAssembly().GetName().Version.ToString(), - ReleaseTime: new DateTime(0), - ReleaseType: + SelfVersion = selfVersion ?? new UpdateInfo( + MinimumCompatibleVersion: 1, + PackageUpdaterVersion: 1, + IncompatibleUpdateUrl: string.Empty, + GenericUpdatePageUrl: "https://duplicati.com/download", + UpdateSeverity: null, + ChangeInfo: null, + Packages: null, + Displayname: string.IsNullOrWhiteSpace(License.VersionNumbers.TAG) ? "Current" : License.VersionNumbers.TAG, + Version: System.Reflection.Assembly.GetExecutingAssembly().GetName().Version?.ToString(), + ReleaseTime: new DateTime(0), + ReleaseType: #if DEBUG - "Debug" + "Debug" #else string.IsNullOrWhiteSpace(AutoUpdateSettings.BuildUpdateChannel) ? "Nightly" : AutoUpdateSettings.BuildUpdateChannel #endif - ); - } + ); } - public static Version TryParseVersion(string str) + public static Version TryParseVersion(string? str) { - Version v; - if (Version.TryParse(str, out v)) + if (Version.TryParse(str, out var v)) return v; else return new Version(0, 0); } - /// - /// The unique machine installation ID - /// - public static string InstallID - { - get - { - try { return System.IO.File.ReadAllLines(Path.Combine(DATAFOLDER, INSTALL_FILE)).FirstOrDefault(x => !string.IsNullOrWhiteSpace(x))?.Trim() ?? ""; } - catch { } - - return ""; - } - } - - /// - /// The unique machine ID - /// - public static string MachineID - { - get - { - string machinedId = null; - try { machinedId = System.IO.File.ReadAllLines(Path.Combine(DATAFOLDER, MACHINE_FILE)).FirstOrDefault(x => !string.IsNullOrWhiteSpace(x))?.Trim() ?? ""; } - catch { } - - return string.IsNullOrWhiteSpace(machinedId) - ? InstallID - : machinedId; - } - } - - /// - /// The machine name - /// - public static readonly string MachineName = MachineNameReader.GetMachineName(); /// /// The package type ID /// - public static string PackageTypeId + public static string PackageTypeId => _packageTypeId.Value; + + /// + /// The package type ID, lazy evaluated + /// + private static readonly Lazy _packageTypeId = new(() => { - get - { - try { return File.ReadAllLines(Path.Combine(INSTALLATIONDIR, PACKAGE_TYPE_FILE)).FirstOrDefault(x => !string.IsNullOrWhiteSpace(x))?.Trim() ?? ""; } - catch { } + try { return File.ReadAllLines(Path.Combine(INSTALLATIONDIR!, PACKAGE_TYPE_FILE)).FirstOrDefault(x => !string.IsNullOrWhiteSpace(x))?.Trim() ?? ""; } + catch { } #if DEBUG - return "debug"; + return "debug"; #else - return ""; + return ""; #endif - } - } + }); - public static UpdateInfo CheckForUpdate(ReleaseType channel = ReleaseType.Unknown) + /// + /// Checks for updates and returns the update information if available + /// + /// The release channel to check for updates + /// The update information if available, or null if no updates are available + public static UpdateInfo? CheckForUpdate(ReleaseType channel = ReleaseType.Unknown) { if (channel == ReleaseType.Unknown) channel = AutoUpdateSettings.DefaultUpdateChannel; @@ -283,18 +214,19 @@ namespace Duplicati.Library.AutoUpdater if (SIGN_KEYS.Length == 0) throw new Exception("No signing keys are available, cannot check update"); - using (var tmpfile = new Library.Utility.TempFile()) + using (var tmpfile = new TempFile()) { using var request = new HttpRequestMessage(HttpMethod.Get, url); - request.Headers.Add(System.Net.HttpRequestHeader.UserAgent.ToString(), string.Format("{0} v{1}{2}", APPNAME, SelfVersion.Version, string.IsNullOrWhiteSpace(InstallID) ? "" : " -" + InstallID)); - request.Headers.Add("X-Install-ID", InstallID); + request.Headers.Add(System.Net.HttpRequestHeader.UserAgent.ToString(), string.Format("{0} v{1}{2}", APPNAME, SelfVersion.Version, string.IsNullOrWhiteSpace(DataFolderManager.InstallID) ? "" : " -" + DataFolderManager.InstallID)); + request.Headers.Add("X-Install-ID", DataFolderManager.InstallID); request.Headers.Add("X-Package-Type-ID", PackageTypeId); using var timeoutToken = new CancellationTokenSource(); timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS)); - HttpClientHelper.DefaultClient.DownloadFile(request, tmpfile, null, timeoutToken.Token).ConfigureAwait(false).GetAwaiter().GetResult(); + using (var client = HttpClientHelper.CreateClient()) + client.DownloadFile(request, tmpfile, null, timeoutToken.Token).Await(); using (var fs = File.OpenRead(tmpfile)) { @@ -307,6 +239,8 @@ namespace Duplicati.Library.AutoUpdater throw new Exception("No valid signature found in manifest file"); var update = JsonSerializer.Deserialize(fs); + if (update == null) + return null; if (TryParseVersion(update.Version) <= TryParseVersion(SelfVersion.Version)) return null; @@ -347,7 +281,12 @@ namespace Duplicati.Library.AutoUpdater return null; } - private static UpdateInfo ReadInstalledManifest(string folder) + /// + /// Reads the installed manifest file + /// + /// The folder to read the manifest from + /// The manifest if found, or null if not found + private static UpdateInfo? ReadInstalledManifest(string folder) { var manifest = Path.Combine(folder, UPDATE_MANIFEST_FILENAME); if (File.Exists(manifest)) @@ -377,11 +316,16 @@ namespace Duplicati.Library.AutoUpdater return null; } - public static bool DownloadUpdate(UpdateInfo version, PackageEntry package, string targetPath, Action progress = null) + /// + /// Downloads the update package + /// + /// The version to download + /// The package to download + /// The path to save the downloaded package to + /// The progress callback + /// True if the download was successful, otherwise false + public static bool DownloadUpdate(UpdateInfo version, PackageEntry package, string targetPath, Action? progress = null) { - if (DATAFOLDER == null) - return false; - var updates = package.RemoteUrls.ToList(); // If alternate update URLs are specified, @@ -410,19 +354,20 @@ namespace Duplicati.Library.AutoUpdater { using (var tempfile = File.Open(tempfilename, FileMode.Create, FileAccess.ReadWrite, FileShare.None)) { - Action cb = null; + Action? cb = null; if (progress != null) cb = (s) => { progress(Math.Min(1.0, Math.Max(0.0, (double)s / package.Length))); }; using var request = new HttpRequestMessage(HttpMethod.Get, url); request.Headers.Add(System.Net.HttpRequestHeader.UserAgent.ToString(), string.Format("{0} v{1}", APPNAME, SelfVersion.Version)); - request.Headers.Add("X-Install-ID", InstallID); + request.Headers.Add("X-Install-ID", DataFolderManager.InstallID); using var timeoutToken = new CancellationTokenSource(); timeoutToken.CancelAfter(TimeSpan.FromSeconds(DOWNLOAD_OPERATION_TIMEOUT_SECONDS)); - HttpClientHelper.DefaultClient.DownloadFile(request, tempfile, cb, timeoutToken.Token).ConfigureAwait(false).GetAwaiter().GetResult(); + using (var client = HttpClientHelper.CreateClient()) + client.DownloadFile(request, tempfile, cb, timeoutToken.Token).Await(); var sha256 = System.Security.Cryptography.SHA256.Create(); var md5 = System.Security.Cryptography.MD5.Create(); @@ -465,10 +410,11 @@ namespace Duplicati.Library.AutoUpdater /// The version of the manifest /// The URL to use for incompatible updates /// The URL to use for generic updates - public static void CreateSignedManifest(IEnumerable keys, string sourcedata, string outputfolder, string version = null, string incompatibleUpdateUrl = null, string genericUpdatePageUrl = null, string releaseType = null, IEnumerable packages = null) + public static void CreateSignedManifest(IEnumerable keys, string sourcedata, string outputfolder, string? version = null, string? incompatibleUpdateUrl = null, string? genericUpdatePageUrl = null, string? releaseType = null, IEnumerable? packages = null) { // Read the existing manifest - var remoteManifest = JsonSerializer.Deserialize(string.IsNullOrWhiteSpace(sourcedata) ? "{}" : sourcedata); + var remoteManifest = JsonSerializer.Deserialize(string.IsNullOrWhiteSpace(sourcedata) ? "{}" : sourcedata) + ?? throw new Exception("Failed to deserialize the manifest from source data"); if (remoteManifest.ReleaseTime.Ticks == 0) remoteManifest = remoteManifest with { ReleaseTime = DateTime.UtcNow }; diff --git a/Duplicati/Library/Common/IO/Util.cs b/Duplicati/Library/Common/IO/Util.cs index 0d0f8ee75..381648a15 100644 --- a/Duplicati/Library/Common/IO/Util.cs +++ b/Duplicati/Library/Common/IO/Util.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -66,5 +66,18 @@ namespace Duplicati.Library.Common.IO { return string.IsNullOrWhiteSpace(path) || path.StartsWith("/", StringComparison.Ordinal) ? "/" : "\\"; } + + /// + /// Checks if the path is inside the Windows folder + /// + /// The path to check + /// true if the path is inside the Windows folder, false otherwise + public static bool IsPathUnderWindowsFolder(string path) + { + if (!OperatingSystem.IsWindows()) + return false; + + return path.StartsWith(AppendDirSeparator(Environment.GetFolderPath(Environment.SpecialFolder.Windows)), StringComparison.OrdinalIgnoreCase); + } } } diff --git a/Duplicati/Library/Main/CLIDatabaseLocator.cs b/Duplicati/Library/Main/CLIDatabaseLocator.cs index 7c5255e4c..fe3ab3304 100644 --- a/Duplicati/Library/Main/CLIDatabaseLocator.cs +++ b/Duplicati/Library/Main/CLIDatabaseLocator.cs @@ -22,6 +22,7 @@ using System; using System.Linq; using System.Collections.Generic; +using Duplicati.Library.Common.IO; #nullable enable @@ -95,7 +96,12 @@ namespace Duplicati.Library.Main if (!string.IsNullOrEmpty(options.Dbpath)) return options.Dbpath; - var folder = AutoUpdater.DatabaseLocator.GetDefaultStorageFolderWithDebugSupport(CONFIG_FILE); + // Ideally, this should use DataFolderManager.DATAFOLDER, but we cannot due to backwards compatibility + var folder = AutoUpdater.DataFolderLocator.GetDefaultStorageFolder(CONFIG_FILE); + // Emit a warning if the database is stored in the Windows folder + if (Util.IsPathUnderWindowsFolder(folder)) + Logging.Log.WriteWarningMessage(LOGTAG, "DatabaseInWindowsFolder", null, "The database config is stored in the Windows folder, this is not recommended as it will be deleted on Windows upgrades."); + var file = System.IO.Path.Combine(folder, CONFIG_FILE); List configs; @@ -242,7 +248,8 @@ namespace Duplicati.Library.Main /// true if the path is in use, false otherwise public static bool IsDatabasePathInUse(string path) { - var file = System.IO.Path.Combine(AutoUpdater.DatabaseLocator.GetDefaultStorageFolderWithDebugSupport(CONFIG_FILE), CONFIG_FILE); + // Ideally, this should use DataFolderManager.DATAFOLDER, but we cannot due to backwards compatibility + var file = System.IO.Path.Combine(AutoUpdater.DataFolderLocator.GetDefaultStorageFolder(CONFIG_FILE), CONFIG_FILE); if (!System.IO.File.Exists(file)) return false; diff --git a/Duplicati/Library/Main/Controller.cs b/Duplicati/Library/Main/Controller.cs index beff76f0c..7883ddfc8 100644 --- a/Duplicati/Library/Main/Controller.cs +++ b/Duplicati/Library/Main/Controller.cs @@ -559,26 +559,6 @@ namespace Duplicati.Library.Main } } - /// - /// Attempts to get the locale, but delays linking to the calls as they are missing in some environments - /// - [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] - private static void DoGetLocale(out System.Globalization.CultureInfo locale, out System.Globalization.CultureInfo uiLocale) - { - locale = System.Globalization.CultureInfo.DefaultThreadCurrentCulture; - uiLocale = System.Globalization.CultureInfo.DefaultThreadCurrentUICulture; - } - - /// - /// Attempts to set the locale, but delays linking to the calls as they are missing in some environments - /// - [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] - private static void DoSetLocale(System.Globalization.CultureInfo locale, System.Globalization.CultureInfo uiLocale) - { - System.Globalization.CultureInfo.DefaultThreadCurrentCulture = locale; - System.Globalization.CultureInfo.DefaultThreadCurrentUICulture = uiLocale; - } - private void OperationComplete(IBasicResults result, Exception exception) { if (m_options != null && m_options.LoadedModules != null) @@ -783,20 +763,20 @@ namespace Duplicati.Library.Main } //Keep a list of all supplied options - Dictionary ropts = new Dictionary(m_options.RawOptions); + var ropts = new Dictionary(m_options.RawOptions); //Keep a list of all supported options - Dictionary supportedOptions = new Dictionary(); + var supportedOptions = new Dictionary(); //There are a few internal options that are not accessible from outside, and thus not listed foreach (string s in Options.InternalOptions) supportedOptions[s] = null; //Figure out what module options are supported in the current setup - List moduleOptions = new List(); - Dictionary disabledModuleOptions = new Dictionary(); + var moduleOptions = new List(); + var disabledModuleOptions = new Dictionary(); - foreach (KeyValuePair m in m_options.LoadedModules) + foreach (var m in m_options.LoadedModules) if (m.Value.SupportedCommands != null) if (m.Key) moduleOptions.AddRange(m.Value.SupportedCommands); @@ -863,7 +843,7 @@ namespace Duplicati.Library.Main } //Now look for options that were supplied but not supported - foreach (string s in ropts.Keys) + foreach (var s in ropts.Keys) if (!supportedOptions.ContainsKey(s)) if (disabledModuleOptions.ContainsKey(s)) Logging.Log.WriteWarningMessage(LOGTAG, "UnsupportedDisabledModule", null, Strings.Controller.UnsupportedOptionDisabledModuleWarning(s, disabledModuleOptions[s]), null); @@ -871,10 +851,9 @@ namespace Duplicati.Library.Main Logging.Log.WriteWarningMessage(LOGTAG, "UnsupportedOption", null, Strings.Controller.UnsupportedOptionWarning(s), null); //Look at the value supplied for each argument and see if is valid according to its type - foreach (string s in ropts.Keys) + foreach (var s in ropts.Keys) { - Library.Interface.ICommandLineArgument arg; - if (supportedOptions.TryGetValue(s, out arg) && arg != null) + if (supportedOptions.TryGetValue(s, out var arg) && arg != null) { string validationMessage = ValidateOptionValue(arg, s, ropts[s]); if (validationMessage != null) diff --git a/Duplicati/Library/Main/Operation/SystemInfoHandler.cs b/Duplicati/Library/Main/Operation/SystemInfoHandler.cs index 069f34e1f..04d9f0d94 100644 --- a/Duplicati/Library/Main/Operation/SystemInfoHandler.cs +++ b/Duplicati/Library/Main/Operation/SystemInfoHandler.cs @@ -38,13 +38,13 @@ namespace Duplicati.Library.Main.Operation yield return string.Format("Duplicati: {0} ({1})", Library.Utility.Utility.getEntryAssembly().FullName, System.Reflection.Assembly.GetExecutingAssembly().FullName); yield return string.Format("Autoupdate urls: {0}", string.Join(";", AutoUpdater.AutoUpdateSettings.URLs)); - yield return string.Format("Default data folder: {0}", AutoUpdater.UpdaterManager.DATAFOLDER); + yield return string.Format("Default data folder: {0}", AutoUpdater.DataFolderManager.DATAFOLDER); yield return string.Format("Install folder: {0}", AutoUpdater.UpdaterManager.INSTALLATIONDIR); yield return string.Format("Version name: \"{0}\" ({1})", AutoUpdater.UpdaterManager.SelfVersion.Displayname, System.Reflection.Assembly.GetExecutingAssembly().GetName().Version); yield return string.Format("Current Version folder {0}", System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location)); yield return string.Format("OS: {0}", Library.UsageReporter.OSInfoHelper.PlatformString); - yield return string.Format("OSType: {0}", OperatingSystem.IsWindows() ? "Windows" : OperatingSystem.IsLinux() ? "Linux" : OperatingSystem.IsMacOS() ? "MacOS" : "Unknown"); + yield return string.Format("OSType: {0}", AutoUpdater.UpdaterManager.OperatingSystemName); yield return string.Format("64bit: {0} ({1})", Environment.Is64BitOperatingSystem, Environment.Is64BitProcess); yield return string.Format("Machinename: {0}", Environment.MachineName); diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index 6131d1dd1..c0d681aaa 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -384,8 +384,8 @@ namespace Duplicati.Library.Main new CommandLineArgument("exclude-files-attributes", CommandLineArgument.ArgumentType.String, Strings.Options.ExcludefilesattributesShort, Strings.Options.ExcludefilesattributesLong(Enum.GetNames(typeof(System.IO.FileAttributes)))), new CommandLineArgument("compression-extension-file", CommandLineArgument.ArgumentType.Path, Strings.Options.CompressionextensionfileShort, Strings.Options.CompressionextensionfileLong(DEFAULT_COMPRESSED_EXTENSION_FILE), DEFAULT_COMPRESSED_EXTENSION_FILE), - new CommandLineArgument("machine-id", CommandLineArgument.ArgumentType.String, Strings.Options.MachineidShort, Strings.Options.MachineidLong, Library.AutoUpdater.UpdaterManager.InstallID), - new CommandLineArgument("machine-name", CommandLineArgument.ArgumentType.String, Strings.Options.MachinenameShort, Strings.Options.MachinenameLong, Library.AutoUpdater.UpdaterManager.MachineName), + new CommandLineArgument("machine-id", CommandLineArgument.ArgumentType.String, Strings.Options.MachineidShort, Strings.Options.MachineidLong, Library.AutoUpdater.DataFolderManager.InstallID), + new CommandLineArgument("machine-name", CommandLineArgument.ArgumentType.String, Strings.Options.MachinenameShort, Strings.Options.MachinenameLong, Library.AutoUpdater.DataFolderManager.MachineName), new CommandLineArgument("backup-id", CommandLineArgument.ArgumentType.String, Strings.Options.BackupidShort, Strings.Options.BackupidLong, ""), new CommandLineArgument("backup-name", CommandLineArgument.ArgumentType.String, Strings.Options.BackupnameShort, Strings.Options.BackupnameLong, DefaultBackupName), new CommandLineArgument("next-scheduled-run", CommandLineArgument.ArgumentType.String, Strings.Options.NextscheduledrunShort, Strings.Options.NextscheduledrunLong), @@ -1367,7 +1367,7 @@ namespace Duplicati.Library.Main { if (m_options.TryGetValue("machine-id", out var tmp)) return tmp; - return Library.AutoUpdater.UpdaterManager.InstallID; + return Library.AutoUpdater.DataFolderManager.InstallID; } } /// diff --git a/Duplicati/Library/Modules/Builtin/ReportHelper.cs b/Duplicati/Library/Modules/Builtin/ReportHelper.cs index 5e9db9a23..78cbfae29 100644 --- a/Duplicati/Library/Modules/Builtin/ReportHelper.cs +++ b/Duplicati/Library/Modules/Builtin/ReportHelper.cs @@ -397,15 +397,15 @@ namespace Duplicati.Library.Modules.Builtin case PARSEDRESULT: return m_parsedresultlevel; case MACHINE_ID: - return AutoUpdater.UpdaterManager.MachineID; + return AutoUpdater.DataFolderManager.MachineID; case BACKUP_ID: return Utility.Utility.ByteArrayAsHexString(Utility.Utility.RepeatedHashWithSalt(m_remoteurl, SALT)); case BACKUP_NAME: return System.IO.Path.GetFileNameWithoutExtension(Utility.Utility.getEntryAssembly().Location); case MACHINE_NAME: - return AutoUpdater.UpdaterManager.MachineName; + return AutoUpdater.DataFolderManager.MachineName; case OPERATING_SYSTEM: - return OperatingSystem.IsWindows() ? "Windows" : OperatingSystem.IsLinux() ? "Linux" : OperatingSystem.IsMacOS() ? "MacOS" : "Unknown"; + return AutoUpdater.UpdaterManager.OperatingSystemName; case INSTALLATION_TYPE: return AutoUpdater.UpdaterManager.PackageTypeId; case DESTINATION_TYPE: diff --git a/Duplicati/Library/RemoteControl/RegisterForRemote.cs b/Duplicati/Library/RemoteControl/RegisterForRemote.cs index 85ea09776..6de30c610 100644 --- a/Duplicati/Library/RemoteControl/RegisterForRemote.cs +++ b/Duplicati/Library/RemoteControl/RegisterForRemote.cs @@ -219,12 +219,13 @@ public class RegisterForRemote : IDisposable var basics = new Dictionary { { "instanceId", ClientInstanceId }, - { "machineId", AutoUpdater.UpdaterManager.MachineID }, - { "machineName", AutoUpdater.UpdaterManager.MachineName }, - { "installId", AutoUpdater.UpdaterManager.InstallID }, + { "machineId", AutoUpdater.DataFolderManager.MachineID }, + { "machineName", AutoUpdater.DataFolderManager.MachineName }, + { "installId", AutoUpdater.DataFolderManager.InstallID }, { "localTime", DateTimeOffset.Now.ToString("o") }, { "version", AutoUpdater.UpdaterManager.SelfVersion?.Version }, - { "packageTypeId", AutoUpdater.UpdaterManager.PackageTypeId } + { "packageTypeId", AutoUpdater.UpdaterManager.PackageTypeId }, + { "operatingSystem", AutoUpdater.UpdaterManager.OperatingSystemName } }; // Pass any query parameters from the registration URL into the JSON body diff --git a/Duplicati/Library/UsageReporter/ReportSet.cs b/Duplicati/Library/UsageReporter/ReportSet.cs index d80b5549a..55ac80dcb 100644 --- a/Duplicati/Library/UsageReporter/ReportSet.cs +++ b/Duplicati/Library/UsageReporter/ReportSet.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -43,7 +43,7 @@ namespace Duplicati.Library.UsageReporter [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] private static void DoInitUID() { - Cached_UserID = Library.AutoUpdater.UpdaterManager.InstallID; + Cached_UserID = Library.AutoUpdater.DataFolderManager.InstallID; } [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] diff --git a/Duplicati/Library/UsageReporter/ReportSetUploader.cs b/Duplicati/Library/UsageReporter/ReportSetUploader.cs index 2b7d57d9b..9c396f760 100644 --- a/Duplicati/Library/UsageReporter/ReportSetUploader.cs +++ b/Duplicati/Library/UsageReporter/ReportSetUploader.cs @@ -1,31 +1,28 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using CoCoL; using System.Threading.Tasks; -using System.Collections.Generic; -using System.Linq; using System.IO; -using System.Net; using System.Net.Http; using Duplicati.Library.Utility; using System.Threading; @@ -62,7 +59,7 @@ namespace Duplicati.Library.UsageReporter public static Tuple> Run() { var channel = ChannelManager.CreateChannel( - buffersize: MAX_PENDING_UPLOADS, + buffersize: MAX_PENDING_UPLOADS, pendingWritersOverflowStrategy: QueueOverflowStrategy.LIFO ); @@ -85,13 +82,14 @@ namespace Duplicati.Library.UsageReporter if (fs.Length > 0) { using var request = new HttpRequestMessage(HttpMethod.Post, UPLOAD_URL); - + request.Content = new StreamContent(fs); using var timeoutToken = new CancellationTokenSource(); timeoutToken.CancelAfter(TimeSpan.FromSeconds(UPLOAD_OPERATION_TIMEOUT_SECONDS)); - - var response = await HttpClientHelper.DefaultClient.UploadStream(request, timeoutToken.Token); + + using var client = HttpClientHelper.CreateClient(); + using var response = await client.UploadStream(request, timeoutToken.Token); rc = (int)response.StatusCode; } else diff --git a/Duplicati/Library/Utility/HttpClientExtensions.cs b/Duplicati/Library/Utility/HttpClientExtensions.cs index b02633d63..1247818b1 100644 --- a/Duplicati/Library/Utility/HttpClientExtensions.cs +++ b/Duplicati/Library/Utility/HttpClientExtensions.cs @@ -25,6 +25,8 @@ using System.Net.Http; using System.Threading; using System.Threading.Tasks; +#nullable enable + namespace Duplicati.Library.Utility; /// @@ -43,20 +45,20 @@ public static class HttpClientExtensions /// Action for progress reporting /// Cancelation token /// - public static async Task DownloadFile(this HttpClient client, HttpRequestMessage request, string filename, Action progressReportingAction = null, CancellationToken cancellationToken = default) + public static async Task DownloadFile(this HttpClient client, HttpRequestMessage request, string filename, Action? progressReportingAction = null, CancellationToken cancellationToken = default) { - using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken); + using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); response.EnsureSuccessStatusCode(); - using var stream = await response.Content.ReadAsStreamAsync(cancellationToken); - using var fileStream = System.IO.File.Create(filename); + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + await using var fileStream = System.IO.File.Create(filename); if (progressReportingAction != null) { using var ProgressReportingStream = new ProgressReportingStream(stream, progressReportingAction); - await ProgressReportingStream.CopyToAsync(fileStream,cancellationToken); + await ProgressReportingStream.CopyToAsync(fileStream, cancellationToken).ConfigureAwait(false); } else { - await stream.CopyToAsync(fileStream, cancellationToken); + await stream.CopyToAsync(fileStream, cancellationToken).ConfigureAwait(false); } } @@ -69,19 +71,19 @@ public static class HttpClientExtensions /// Action for progress reporting /// Cancelation token /// - public static async Task DownloadFile(this HttpClient client, HttpRequestMessage request, Stream fileStream, Action progressReportingAction = null, CancellationToken cancellationToken = default) + public static async Task DownloadFile(this HttpClient client, HttpRequestMessage request, Stream fileStream, Action? progressReportingAction = null, CancellationToken cancellationToken = default) { - using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken); + using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); response.EnsureSuccessStatusCode(); - using var stream = await response.Content.ReadAsStreamAsync(cancellationToken); + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); if (progressReportingAction != null) { using var ProgressReportingStream = new ProgressReportingStream(stream, progressReportingAction); - await ProgressReportingStream.CopyToAsync(fileStream, cancellationToken); + await ProgressReportingStream.CopyToAsync(fileStream, cancellationToken).ConfigureAwait(false); } else { - await stream.CopyToAsync(fileStream, cancellationToken); + await stream.CopyToAsync(fileStream, cancellationToken).ConfigureAwait(false); } } @@ -91,8 +93,6 @@ public static class HttpClientExtensions /// The Http client reference /// A prepared HttpRequestMessage (Presumably with a stream) /// Cancelation token - public static async Task UploadStream(this HttpClient client, HttpRequestMessage request,CancellationToken cancellationToken = default) - { - return await client.SendAsync(request, HttpCompletionOption.ResponseContentRead, cancellationToken); - } + public static Task UploadStream(this HttpClient client, HttpRequestMessage request, CancellationToken cancellationToken = default) + => client.SendAsync(request, HttpCompletionOption.ResponseContentRead, cancellationToken); } diff --git a/Duplicati/Library/Utility/HttpClientHelper.cs b/Duplicati/Library/Utility/HttpClientHelper.cs index c4c21d3e9..411eda3a3 100644 --- a/Duplicati/Library/Utility/HttpClientHelper.cs +++ b/Duplicati/Library/Utility/HttpClientHelper.cs @@ -19,10 +19,9 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System; -using System.Collections.Generic; +#nullable enable + using System.Net.Http; -using Duplicati.Library.Interface; namespace Duplicati.Library.Utility; @@ -33,25 +32,11 @@ namespace Duplicati.Library.Utility; /// public static class HttpClientHelper { - - /// - /// Globally configured certificate validator to be used across the application. - /// Each backend can choose when creating the httpclient/requesthandler if it will - /// use this or the default (null) validator. - /// - public static SslCertificateValidator CertificateValidator { get; set; } - - /// - /// Central HttpClient singleton instance to be used across the application as - /// per the HttpClientFactory recommended pattern. - /// - public static HttpClient DefaultClient { get; private set; } - /// /// Reference to the HttpClientFactory instance so we can create specific clients /// when the singleton pattnern is not desirable. /// - private static IHttpClientFactory _factory { get; set; } + private static IHttpClientFactory? _factory { get; set; } /// /// Sets the factory reference. @@ -60,7 +45,6 @@ public static class HttpClientHelper public static void Configure(IHttpClientFactory factory) { _factory = factory; - DefaultClient = factory.CreateClient(); } /// @@ -82,7 +66,7 @@ public static class HttpClientHelper { return new HttpClient(handler); } - + /// /// Configures the httphandler with the certificate validator, if needed. /// diff --git a/Duplicati/License/VersionNumbers.cs b/Duplicati/License/VersionNumbers.cs index 0a458ea02..98a4bb71a 100644 --- a/Duplicati/License/VersionNumbers.cs +++ b/Duplicati/License/VersionNumbers.cs @@ -1,35 +1,43 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - -using System; -using System.Collections.Generic; -using System.Text; +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. namespace Duplicati.License { + /// + /// The version numbers for the application, read from the assembly + /// public static class VersionNumbers { + /// + /// The Version Tag read from the embedded resource VersionTag.txt + /// public static readonly string TAG; + /// + /// The version number of the assembly + /// public static readonly string VERSION_NAME; - + + /// + /// Static constructor to read the version numbers from the assembly + /// static VersionNumbers() { string tag = ""; @@ -41,13 +49,13 @@ namespace Duplicati.License catch { } - + if (string.IsNullOrWhiteSpace(tag)) tag = ""; - + TAG = tag.Trim(); - var v = VersionNumber; + var v = System.Reflection.Assembly.GetExecutingAssembly().GetName().Version.ToString(); if (!string.IsNullOrWhiteSpace(TAG)) v = " - " + TAG; #if DEBUG @@ -56,21 +64,5 @@ namespace Duplicati.License VERSION_NAME = v; } - - public static string Version - { - get - { - return VERSION_NAME; - } - } - - private static string VersionNumber - { - get - { - return System.Reflection.Assembly.GetExecutingAssembly().GetName().Version.ToString(); - } - } } } diff --git a/Duplicati/Server/Program.cs b/Duplicati/Server/Program.cs index 40bac2ee7..09ce2f012 100644 --- a/Duplicati/Server/Program.cs +++ b/Duplicati/Server/Program.cs @@ -71,20 +71,10 @@ namespace Duplicati.Server /// public static readonly string StartupPath = Duplicati.Library.AutoUpdater.UpdaterManager.INSTALLATIONDIR; - /// - /// Name of the database file - /// - public const string SERVER_DATABASE_FILENAME = "Duplicati-server.sqlite"; - /// /// The environment variable prefix /// - private static readonly string ENV_NAME_PREFIX = Duplicati.Library.AutoUpdater.AutoUpdateSettings.AppName.ToUpper(CultureInfo.InvariantCulture); - - /// - /// The name of the environment variable that holds the path to the data folder used by Duplicati - /// - private static readonly string DATAFOLDER_ENV_NAME = ENV_NAME_PREFIX + "_HOME"; + private static readonly string ENV_NAME_PREFIX = AutoUpdateSettings.AppName.ToUpperInvariant(); /// /// Gets the folder where Duplicati data is stored @@ -756,46 +746,14 @@ namespace Duplicati.Server throw new Exception("Server invoked with --help"); } - public static string GetDataFolderPath(Dictionary commandlineOptions) + public static Connection GetDatabaseConnection(Dictionary commandlineOptions, bool silentConsole) { - var serverDataFolder = Environment.GetEnvironmentVariable(DATAFOLDER_ENV_NAME); - if (commandlineOptions.ContainsKey("server-datafolder")) - serverDataFolder = commandlineOptions["server-datafolder"]; + DataFolder = DataFolderManager.DATAFOLDER; - if (string.IsNullOrEmpty(serverDataFolder)) - { - bool portableMode = commandlineOptions.ContainsKey("portable-mode") - ? Library.Utility.Utility.ParseBool(commandlineOptions["portable-mode"], true) - : (DEBUG_MODE ? true : false); // Default to portable mode in debug mode + // Emit a warning if the database is stored in the Windows folder + if (Util.IsPathUnderWindowsFolder(DataFolder)) + Log.WriteWarningMessage(LOGTAG, "DatabaseInWindowsFolder", null, "The database is stored in the Windows folder, this is not recommended as it will be deleted on Windows upgrades."); - if (DEBUG_MODE && portableMode) - { - //debug mode uses a lock file located in the app folder - return StartupPath; - } - else if (portableMode) - { - //Portable mode uses a data folder in the application home dir - System.IO.Directory.SetCurrentDirectory(StartupPath); - return System.IO.Path.Combine(StartupPath, "data"); - } - else - { - //Normal release mode uses - // %LOCALAPPDATA% on Windows, - // ~/.config on Linux, - // ~/Library/Application\ Support on MacOS - // with additional rules for special cases and probing for older locations - return DatabaseLocator.GetDefaultStorageFolderDirect(SERVER_DATABASE_FILENAME, AutoUpdateSettings.AppName); - } - } - else - return Util.AppendDirSeparator(Environment.ExpandEnvironmentVariables(serverDataFolder).Trim('"')); - } - - public static Database.Connection GetDatabaseConnection(Dictionary commandlineOptions, bool silentConsole) - { - DataFolder = GetDataFolderPath(commandlineOptions); CrashlogHelper.DefaultLogDir = DataFolder; var sqliteVersion = new Version(Duplicati.Library.SQLiteHelper.SQLiteLoader.SQLiteVersion); @@ -810,7 +768,7 @@ namespace Duplicati.Server try { - DatabasePath = System.IO.Path.Combine(DataFolder, SERVER_DATABASE_FILENAME); + DatabasePath = System.IO.Path.Combine(DataFolder, DataFolderManager.SERVER_DATABASE_FILENAME); if (!System.IO.Directory.Exists(System.IO.Path.GetDirectoryName(DatabasePath))) System.IO.Directory.CreateDirectory(System.IO.Path.GetDirectoryName(DatabasePath)); @@ -985,7 +943,7 @@ namespace Duplicati.Server new Duplicati.Library.Interface.CommandLineArgument("tempdir", Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Path, Strings.Program.TempdirShort, Strings.Program.TempdirLong, System.IO.Path.GetTempPath()), new Duplicati.Library.Interface.CommandLineArgument("help", Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.Program.HelpCommandDescription, Strings.Program.HelpCommandDescription), new Duplicati.Library.Interface.CommandLineArgument("parameters-file", Library.Interface.CommandLineArgument.ArgumentType.Path, Strings.Program.ParametersFileOptionShort, Strings.Program.ParametersFileOptionLong2, "", ParameterFileOptionStrings), - new Duplicati.Library.Interface.CommandLineArgument("portable-mode", Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.Program.PortablemodeCommandDescription, Strings.Program.PortablemodeCommandDescription), + new Duplicati.Library.Interface.CommandLineArgument(DataFolderManager.PORTABLE_MODE_OPTION, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.Program.PortablemodeCommandDescription, Strings.Program.PortablemodeCommandDescription, DataFolderManager.PORTABLE_MODE.ToString().ToLowerInvariant()), new Duplicati.Library.Interface.CommandLineArgument("log-file", Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Path, Strings.Program.LogfileCommandDescription, Strings.Program.LogfileCommandDescription), new Duplicati.Library.Interface.CommandLineArgument("log-level", Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Enumeration, Strings.Program.LoglevelCommandDescription, Strings.Program.LoglevelCommandDescription, Library.Logging.LogMessageType.Warning.ToString(), null, Enum.GetNames(typeof(Duplicati.Library.Logging.LogMessageType))), new Duplicati.Library.Interface.CommandLineArgument(WebServerLoader.OPTION_WEBROOT, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Path, Strings.Program.WebserverWebrootDescription, Strings.Program.WebserverWebrootDescription, WebServerLoader.DEFAULT_OPTION_WEBROOT), @@ -1006,7 +964,7 @@ namespace Duplicati.Server new Duplicati.Library.Interface.CommandLineArgument(WebServerLoader.OPTION_WEBSERVICE_TIMEZONE, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.String, Strings.Program.WebserverTimezoneDescription, Strings.Program.WebserverTimezoneDescription, TimeZoneHelper.GetLocalTimeZone(), null, TimeZoneHelper.GetTimeZones().Select(x => x.Id).ToArray()), new Duplicati.Library.Interface.CommandLineArgument(PING_PONG_KEEPALIVE_OPTION, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.Program.PingpongkeepaliveShort, Strings.Program.PingpongkeepaliveLong), new Duplicati.Library.Interface.CommandLineArgument("log-retention", Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Timespan, Strings.Program.LogretentionShort, Strings.Program.LogretentionLong, DEFAULT_LOG_RETENTION), - new Duplicati.Library.Interface.CommandLineArgument("server-datafolder", Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Path, Strings.Program.ServerdatafolderShort, Strings.Program.ServerdatafolderLong(DATAFOLDER_ENV_NAME), System.IO.Path.Combine(System.Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), Library.AutoUpdater.AutoUpdateSettings.AppName)), + new Duplicati.Library.Interface.CommandLineArgument(DataFolderManager.SERVER_DATAFOLDER_OPTION, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Path, Strings.Program.ServerdatafolderShort, Strings.Program.ServerdatafolderLong(DataFolderManager.DATAFOLDER_ENV_NAME), DataFolderManager.DATAFOLDER), new Duplicati.Library.Interface.CommandLineArgument(DISABLE_DB_ENCRYPTION_OPTION, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.Program.DisabledbencryptionShort, Strings.Program.DisabledbencryptionLong), new Duplicati.Library.Interface.CommandLineArgument(REQUIRE_DB_ENCRYPTION_KEY_OPTION, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.Program.RequiredbencryptionShort, Strings.Program.RequiredbencryptionLong), new Duplicati.Library.Interface.CommandLineArgument(SETTINGS_ENCRYPTION_KEY_OPTION, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Password, Strings.Program.SettingsencryptionkeyShort, Strings.Program.SettingsencryptionkeyLong(EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME)), diff --git a/Duplicati/WebserverCore/Services/SystemInfoProvider.cs b/Duplicati/WebserverCore/Services/SystemInfoProvider.cs index 02296cab7..1662b3352 100644 --- a/Duplicati/WebserverCore/Services/SystemInfoProvider.cs +++ b/Duplicati/WebserverCore/Services/SystemInfoProvider.cs @@ -212,19 +212,19 @@ public class SystemInfoProvider : ISystemInfoProvider APIVersion = 1, PasswordPlaceholder = FIXMEGlobal.PASSWORD_PLACEHOLDER, ServerVersion = UpdaterManager.SelfVersion.Version, - ServerVersionName = License.VersionNumbers.Version, - ServerVersionType = Library.AutoUpdater.UpdaterManager.SelfVersion.ReleaseType, - RemoteControlRegistrationUrl = Duplicati.Library.RemoteControl.RegisterForRemote.DefaultRegisterationUrl, + ServerVersionName = License.VersionNumbers.VERSION_NAME, + ServerVersionType = UpdaterManager.SelfVersion.ReleaseType, + RemoteControlRegistrationUrl = Library.RemoteControl.RegisterForRemote.DefaultRegisterationUrl, StartedBy = FIXMEGlobal.Origin, - DefaultUpdateChannel = Library.AutoUpdater.AutoUpdateSettings.DefaultUpdateChannel.ToString(), + DefaultUpdateChannel = AutoUpdateSettings.DefaultUpdateChannel.ToString(), DefaultUsageReportLevel = Library.UsageReporter.Reporter.DefaultReportLevel, - OSType = OperatingSystem.IsWindows() ? "Windows" : OperatingSystem.IsLinux() ? "Linux" : OperatingSystem.IsMacOS() ? "MacOS" : "Unknown", + OSType = UpdaterManager.OperatingSystemName, OSVersion = Library.UsageReporter.OSInfoHelper.PlatformString, DirectorySeparator = Path.DirectorySeparatorChar, PathSeparator = Path.PathSeparator, CaseSensitiveFilesystem = Library.Utility.Utility.IsFSCaseSensitive, MachineName = Environment.MachineName, - PackageTypeId = Library.AutoUpdater.UpdaterManager.PackageTypeId, + PackageTypeId = UpdaterManager.PackageTypeId, UserName = OperatingSystem.IsWindows() ? System.Security.Principal.WindowsIdentity.GetCurrent().Name : Environment.UserName, NewLine = Environment.NewLine, CLRVersion = Environment.Version.ToString(), @@ -237,7 +237,7 @@ public class SystemInfoProvider : ISystemInfoProvider ConnectionModules = Server.Serializable.ServerSettings.ConnectionModules, ServerModules = Server.Serializable.ServerSettings.ServerModules, SecretProviderModules = Server.Serializable.ServerSettings.SecretProviderModules, - UsingAlternateUpdateURLs = Library.AutoUpdater.AutoUpdateSettings.UsesAlternateURLs, + UsingAlternateUpdateURLs = AutoUpdateSettings.UsesAlternateURLs, LogLevels = Enum.GetNames(typeof(Library.Logging.LogMessageType)), SpecialFolders = SpecialFolders.Nodes.Select(n => new Dto.SystemInfoDto.SpecialFolderDto { ID = n.id, Path = n.resolvedpath }).ToArray(), SupportedLocales = Library.Localization.LocalizationService.SupportedCultures From de00eee3deb9073a3577294ac70ee988ba126334 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 28 Jan 2025 13:23:57 +0100 Subject: [PATCH 045/362] Changed logic so overrides to server-datafolder will affect any operation, including CLI, Agent, and ServerUtil --- .../Library/AutoUpdater/DataFolderLocator.cs | 17 +++++++++- .../Library/AutoUpdater/DataFolderManager.cs | 33 +++++++++---------- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/Duplicati/Library/AutoUpdater/DataFolderLocator.cs b/Duplicati/Library/AutoUpdater/DataFolderLocator.cs index e9264f2db..1176ee530 100644 --- a/Duplicati/Library/AutoUpdater/DataFolderLocator.cs +++ b/Duplicati/Library/AutoUpdater/DataFolderLocator.cs @@ -32,6 +32,21 @@ namespace Duplicati.Library.AutoUpdater; /// public static class DataFolderLocator { + + /// + /// Finds a default storage folder, using the operating system specific locations. + /// The targetfilename is used to detect locations that are used in previous versions. + /// If the targetfilename is found in an old location, but not the current, the old location is used. + /// Note that the folder is not created, only the path is returned. + /// If the data folder is overriden, the overriden folder is used, and no search is performed. + /// + /// The filename to look for + /// The default storage folder + public static string GetDefaultStorageFolder(string targetfilename) + => DataFolderManager.OVERRIDEN_DATAFOLDER + ? DataFolderManager.DATAFOLDER + : GetDefaultStorageFolderInternal(targetfilename, AutoUpdateSettings.AppName); + /// /// Finds a default storage folder, using the operating system specific locations. /// The targetfilename is used to detect locations that are used in previous versions. @@ -41,7 +56,7 @@ public static class DataFolderLocator /// The filename to look for /// The name of the application /// The default storage folder - public static string GetDefaultStorageFolder(string targetfilename, string appName = "Duplicati") + internal static string GetDefaultStorageFolderInternal(string targetfilename, string appName) { //Normal mode uses the systems "(Local) Application Data" folder // %LOCALAPPDATA% on Windows, ~/.config on Linux diff --git a/Duplicati/Library/AutoUpdater/DataFolderManager.cs b/Duplicati/Library/AutoUpdater/DataFolderManager.cs index e3d248510..db64ba9d2 100644 --- a/Duplicati/Library/AutoUpdater/DataFolderManager.cs +++ b/Duplicati/Library/AutoUpdater/DataFolderManager.cs @@ -34,16 +34,6 @@ namespace Duplicati.Library.AutoUpdater; /// public static class DataFolderManager { - /// - /// The debug flag, set to true if the application is running in debug mode, - /// preventing littering of DEBUG statements in the code - /// -#if DEBUG - private static readonly bool DEBUG_MODE = true; -#else - private static readonly bool DEBUG_MODE = false; -#endif - /// /// The folder where the machine id is placed /// @@ -88,6 +78,11 @@ public static class DataFolderManager /// public static readonly bool PORTABLE_MODE; + /// + /// Flag to indicate if the data folder was overriden + /// + public static readonly bool OVERRIDEN_DATAFOLDER; + /// /// Replication of the argument parsing from the main Duplicati codebase /// @@ -133,7 +128,11 @@ public static class DataFolderManager { // In debug builds, we default to portable mode if (value == null) - return DEBUG_MODE ? true : false; +#if DEBUG + return true; +#else + return false; +#endif if ( value.Equals("false", StringComparison.OrdinalIgnoreCase) @@ -160,26 +159,26 @@ public static class DataFolderManager // Prefer the command line argument over the environment variable if (!string.IsNullOrWhiteSpace(datafolderArg)) { + OVERRIDEN_DATAFOLDER = true; DATAFOLDER = Util.AppendDirSeparator(Environment.ExpandEnvironmentVariables(datafolderArg).Trim('"')); } // Portable mode is prefered over the environment variable else if (PORTABLE_MODE) { - DATAFOLDER = DEBUG_MODE - // Use the build folder for portable mode in debug builds - ? UpdaterManager.INSTALLATIONDIR - // Use the data folder in the installation directory for portable mode - : Util.AppendDirSeparator(Path.Combine(UpdaterManager.INSTALLATIONDIR, "data")); + OVERRIDEN_DATAFOLDER = true; + DATAFOLDER = Util.AppendDirSeparator(Path.Combine(UpdaterManager.INSTALLATIONDIR, "data")); } // Use the legacy environment variable, if set else if (!string.IsNullOrWhiteSpace(envOverride)) { + OVERRIDEN_DATAFOLDER = true; DATAFOLDER = Util.AppendDirSeparator(Environment.ExpandEnvironmentVariables(envOverride).Trim('"')); } // Use the default location else { - DATAFOLDER = Util.AppendDirSeparator(DataFolderLocator.GetDefaultStorageFolder(SERVER_DATABASE_FILENAME, APPNAME)); + OVERRIDEN_DATAFOLDER = false; + DATAFOLDER = Util.AppendDirSeparator(DataFolderLocator.GetDefaultStorageFolderInternal(SERVER_DATABASE_FILENAME, APPNAME)); } if (!Directory.Exists(DATAFOLDER)) From ae58d65de38d61e7b4bcbe5eb57b50d052ee7cfb Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 28 Jan 2025 13:25:51 +0100 Subject: [PATCH 046/362] Add force option to Remote Synchronization Tool --- Tools/RemoteSynchronization/Program.cs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 313cc2dcd..ccdc37517 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -38,6 +38,7 @@ namespace RemoteSynchronization private const bool DEFAULT_DRY_RUN = false; private const bool DEFAULT_VERIFY = false; private const int DEFAULT_RETRY = 3; + private const bool DEFAULT_FORCE = false; public static async Task Main(string[] args) { @@ -48,6 +49,7 @@ namespace RemoteSynchronization var dst_opts = OptionWithMultipleTokens(aliases: ["--dst-options"], description: "Options for the destination backend"); var verify_opt = new Option(aliases: ["--verify"], description: "Verify the files after copying", getDefaultValue: () => DEFAULT_VERIFY); var retry_opt = new Option(aliases: ["--retry"], description: "Number of times to retry on errors", getDefaultValue: () => DEFAULT_RETRY) { Arity = ArgumentArity.ExactlyOne }; + var force_opt = new Option(aliases: ["--force"], description: "Force the synchronization", getDefaultValue: () => DEFAULT_FORCE); var root_cmd = new RootCommand("Remote Synchronization Tool"); root_cmd.AddArgument(src_arg); @@ -57,6 +59,7 @@ namespace RemoteSynchronization root_cmd.AddOption(dst_opts); root_cmd.AddOption(verify_opt); root_cmd.AddOption(retry_opt); + root_cmd.AddOption(force_opt); root_cmd.SetHandler((InvocationContext ctx) => { @@ -75,6 +78,7 @@ namespace RemoteSynchronization var dry_run = options["dry-run"] as bool? ?? DEFAULT_DRY_RUN; var verify = options["verify"] as bool? ?? DEFAULT_VERIFY; var retries = options["retry"] as int? ?? DEFAULT_RETRY; + var force = options["force"] as bool? ?? DEFAULT_FORCE; var duplicati_options = new Dictionary() { ["dry-run"] = dry_run.ToString() @@ -98,7 +102,10 @@ namespace RemoteSynchronization var b2s = b2 as IStreamingBackend; System.Diagnostics.Debug.Assert(b2s != null); - var (to_copy, to_delete, to_verify) = PrepareFileLists(b1s, b2s); + var (to_copy, to_delete, to_verify) = PrepareFileLists(b1s, b2s, force); + + // TODO if confirm is not set, present the plan to the user and ask for confirmation + var deleted = await DeleteAsync(b2s, to_delete, dry_run); Console.WriteLine($"Deleted {deleted} files from {dst}"); var (copied, copy_errors) = await CopyAsync(b1s, b2s, to_copy, dry_run, verify); @@ -141,7 +148,6 @@ namespace RemoteSynchronization // TODO maximum retention?: don't delete files, only rename old ones. // TODO have concurrency parameters: uploaders, downloaders // TODO low memory mode, where things aren't kept in memory. Maybe utilize SQLite? - // TODO Force parameter // TODO Progress reporting // TODO Logging @@ -225,11 +231,17 @@ namespace RemoteSynchronization }; } - private static (IEnumerable, IEnumerable, IEnumerable) PrepareFileLists(IStreamingBackend b_src, IStreamingBackend b_dst) + private static (IEnumerable, IEnumerable, IEnumerable) PrepareFileLists(IStreamingBackend b_src, IStreamingBackend b_dst, bool force) { var files_src = b_src.List(); var files_dst = b_dst.List(); + // Shortcut for force + if (force) + { + return (files_src, files_dst, []); + } + // Shortcut for empty destination if (!files_dst.Any()) { From 2c727ab1322e60be78af58912165c5f06fbc30c2 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 28 Jan 2025 14:23:00 +0100 Subject: [PATCH 047/362] Add retention option to Remote Synchronization Tool --- Tools/RemoteSynchronization/Program.cs | 27 +++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index ccdc37517..35c424305 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2025, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a @@ -19,7 +19,6 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using System; using System.Collections.Generic; @@ -35,10 +34,12 @@ namespace RemoteSynchronization { public class Program { + // Default values for the options private const bool DEFAULT_DRY_RUN = false; private const bool DEFAULT_VERIFY = false; private const int DEFAULT_RETRY = 3; private const bool DEFAULT_FORCE = false; + private const bool DEFAULT_RETENTION = false; public static async Task Main(string[] args) { @@ -50,6 +51,7 @@ namespace RemoteSynchronization var verify_opt = new Option(aliases: ["--verify"], description: "Verify the files after copying", getDefaultValue: () => DEFAULT_VERIFY); var retry_opt = new Option(aliases: ["--retry"], description: "Number of times to retry on errors", getDefaultValue: () => DEFAULT_RETRY) { Arity = ArgumentArity.ExactlyOne }; var force_opt = new Option(aliases: ["--force"], description: "Force the synchronization", getDefaultValue: () => DEFAULT_FORCE); + var retention_opt = new Option(aliases: ["--retention"], description: "Toggles whether to keep old files. Any deletes will be renames instead", getDefaultValue: () => DEFAULT_RETENTION); var root_cmd = new RootCommand("Remote Synchronization Tool"); root_cmd.AddArgument(src_arg); @@ -60,6 +62,7 @@ namespace RemoteSynchronization root_cmd.AddOption(verify_opt); root_cmd.AddOption(retry_opt); root_cmd.AddOption(force_opt); + root_cmd.AddOption(retention_opt); root_cmd.SetHandler((InvocationContext ctx) => { @@ -79,6 +82,7 @@ namespace RemoteSynchronization var verify = options["verify"] as bool? ?? DEFAULT_VERIFY; var retries = options["retry"] as int? ?? DEFAULT_RETRY; var force = options["force"] as bool? ?? DEFAULT_FORCE; + var retention = options["retention"] as bool? ?? DEFAULT_RETENTION; var duplicati_options = new Dictionary() { ["dry-run"] = dry_run.ToString() @@ -106,8 +110,18 @@ namespace RemoteSynchronization // TODO if confirm is not set, present the plan to the user and ask for confirmation - var deleted = await DeleteAsync(b2s, to_delete, dry_run); - Console.WriteLine($"Deleted {deleted} files from {dst}"); + // Delete or rename the files that are not needed + if (retention) + { + var renamed = await RenameAsync(b2s, to_delete, dry_run); + Console.WriteLine($"Renamed {renamed} files in {dst}"); + } + else + { + var deleted = await DeleteAsync(b2s, to_delete, dry_run); + Console.WriteLine($"Deleted {deleted} files from {dst}"); + } + var (copied, copy_errors) = await CopyAsync(b1s, b2s, to_copy, dry_run, verify); Console.WriteLine($"Copied {copied} files from {src} to {dst}"); @@ -145,7 +159,6 @@ namespace RemoteSynchronization return 0; } - // TODO maximum retention?: don't delete files, only rename old ones. // TODO have concurrency parameters: uploaders, downloaders // TODO low memory mode, where things aren't kept in memory. Maybe utilize SQLite? // TODO Progress reporting @@ -294,10 +307,10 @@ namespace RemoteSynchronization return (to_copy, to_delete.Select(x => lookup_dst[x]), to_verify); } - private static async Task RenameAsync(IStreamingBackend b, IEnumerable files, bool dry_run) + private static async Task RenameAsync(IStreamingBackend b, IEnumerable files, bool dry_run) { long successful_renames = 0; - string suffix = $"{DateTime.Now:yyyyMMddHHmmss}.old"; + string suffix = $"{System.DateTime.Now:yyyyMMddHHmmss}.old"; using var downloaded = new MemoryStream(); foreach (var f in files) { From 1c8985b7e957d56ba63e1df14ec87bf7d62ab7e4 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 28 Jan 2025 14:31:20 +0100 Subject: [PATCH 048/362] Add confirmation option to Remote Synchronization Tool --- Tools/RemoteSynchronization/Program.cs | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 35c424305..55518994a 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -40,6 +40,7 @@ namespace RemoteSynchronization private const int DEFAULT_RETRY = 3; private const bool DEFAULT_FORCE = false; private const bool DEFAULT_RETENTION = false; + private const bool DEFAULT_CONFIRM = false; public static async Task Main(string[] args) { @@ -52,6 +53,7 @@ namespace RemoteSynchronization var retry_opt = new Option(aliases: ["--retry"], description: "Number of times to retry on errors", getDefaultValue: () => DEFAULT_RETRY) { Arity = ArgumentArity.ExactlyOne }; var force_opt = new Option(aliases: ["--force"], description: "Force the synchronization", getDefaultValue: () => DEFAULT_FORCE); var retention_opt = new Option(aliases: ["--retention"], description: "Toggles whether to keep old files. Any deletes will be renames instead", getDefaultValue: () => DEFAULT_RETENTION); + var confirm_opt = new Option(aliases: ["--confirm"], description: "Ask for confirmation before starting the synchronization", getDefaultValue: () => DEFAULT_CONFIRM); var root_cmd = new RootCommand("Remote Synchronization Tool"); root_cmd.AddArgument(src_arg); @@ -63,6 +65,7 @@ namespace RemoteSynchronization root_cmd.AddOption(retry_opt); root_cmd.AddOption(force_opt); root_cmd.AddOption(retention_opt); + root_cmd.AddOption(confirm_opt); root_cmd.SetHandler((InvocationContext ctx) => { @@ -83,6 +86,8 @@ namespace RemoteSynchronization var retries = options["retry"] as int? ?? DEFAULT_RETRY; var force = options["force"] as bool? ?? DEFAULT_FORCE; var retention = options["retention"] as bool? ?? DEFAULT_RETENTION; + var confirm = options["confirm"] as bool? ?? DEFAULT_CONFIRM; + var duplicati_options = new Dictionary() { ["dry-run"] = dry_run.ToString() @@ -108,7 +113,25 @@ namespace RemoteSynchronization var (to_copy, to_delete, to_verify) = PrepareFileLists(b1s, b2s, force); - // TODO if confirm is not set, present the plan to the user and ask for confirmation + // As this is a potentially destructive operation, ask for confirmation + if (!confirm) + { + var delete_rename = retention ? "Rename" : "Delete"; + Console.WriteLine($"This will perform the following actions (in order):"); + Console.WriteLine($" - {delete_rename} {to_delete.Count()} files from {dst}"); + Console.WriteLine($" - Copy {to_copy.Count()} files from {src} to {dst}"); + if (verify) + Console.WriteLine($" - Download and verify {to_verify.Count()} files in {dst}"); + Console.WriteLine(); + Console.WriteLine("Do you want to continue? [y/N]"); + + var response = Console.ReadLine(); + if (!response?.Equals("y", StringComparison.CurrentCultureIgnoreCase) ?? true) + { + Console.WriteLine("Aborted"); + return -1; + } + } // Delete or rename the files that are not needed if (retention) From ae94cc6a73a938717856a696af62e8f0903a862e Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 28 Jan 2025 14:31:40 +0100 Subject: [PATCH 049/362] Remote synchronization flavortext --- Tools/RemoteSynchronization/Program.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 55518994a..65e61b26f 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2025, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a @@ -145,9 +145,11 @@ namespace RemoteSynchronization Console.WriteLine($"Deleted {deleted} files from {dst}"); } + // Copy the files var (copied, copy_errors) = await CopyAsync(b1s, b2s, to_copy, dry_run, verify); Console.WriteLine($"Copied {copied} files from {src} to {dst}"); + // If there are still errors, retry a few times if (copy_errors.Any()) { if (retries > 0) @@ -170,6 +172,7 @@ namespace RemoteSynchronization } } + // Verify the files if requested if (verify) { var not_verified = await VerifyAsync(b1s, b2s, to_verify); @@ -177,7 +180,7 @@ namespace RemoteSynchronization return not_verified.Count(); } - Console.WriteLine("Done"); + Console.WriteLine($"Remote synchronization completed successfully"); return 0; } From 36db29dc773e8de42974a7c55e0db31b8a3b1545 Mon Sep 17 00:00:00 2001 From: "Marcelo C." Date: Tue, 28 Jan 2025 10:35:32 -0300 Subject: [PATCH 050/362] Change timeout configuration for BackendTester This change will allow an environment variable READ_WRITE_TIMEOUT_SECONDS to control the read write timeout used during the tests on streaming backends. -1 will set it to infinite. Additionally all console logs during test execution have a timestamp with millisecond precision to aid in benchmarking. --- .../CommandLine/BackendTester/Program.cs | 113 ++++++++++-------- 1 file changed, 63 insertions(+), 50 deletions(-) diff --git a/Duplicati/CommandLine/BackendTester/Program.cs b/Duplicati/CommandLine/BackendTester/Program.cs index a93873ddd..5bee5cc52 100644 --- a/Duplicati/CommandLine/BackendTester/Program.cs +++ b/Duplicati/CommandLine/BackendTester/Program.cs @@ -118,16 +118,17 @@ namespace Duplicati.CommandLine.BackendTester for (int i = 0; i < reruns; i++) { - Console.WriteLine("Starting run no {0}", i); + Console.WriteLine(DateTime.Now.ToShortDateString() + " " + DateTime.Now.ToLongTimeString()); + Console.WriteLine(LogTimeStamp + "Starting run no {0}", i); if (!Run(args, options, i == 0)) return 1; } - Console.WriteLine("Unittest complete!"); + Console.WriteLine(LogTimeStamp + "Unittest complete!"); return 0; } catch (Exception ex) { - Console.WriteLine("Unittest failed: " + ex); + Console.WriteLine(LogTimeStamp + "Unittest failed: " + ex); } return 1; @@ -185,7 +186,7 @@ namespace Duplicati.CommandLine.BackendTester } catch (Exception ex) { - Console.WriteLine("Autocreate folder failed with message: " + ex.Message); + Console.WriteLine(LogTimeStamp + "Autocreate folder failed with message: " + ex.Message); } } @@ -199,7 +200,7 @@ namespace Duplicati.CommandLine.BackendTester if (Library.Utility.Utility.ParseBoolOption(options, "auto-clean") && first) if (Library.Utility.Utility.ParseBoolOption(options, "force")) { - Console.WriteLine("Auto clean, removing file: {0}", fe.Name); + Console.WriteLine(LogTimeStamp + "Auto clean, removing file: {0}", fe.Name); backend.DeleteAsync(fe.Name, CancellationToken.None).Await(); continue; } @@ -208,10 +209,10 @@ namespace Duplicati.CommandLine.BackendTester var fileCount = curlist.Where(x => !x.IsFolder).Count(); var filenames = curlist.Where(x => !x.IsFolder).Select(x => x.Name).Take(10).ToList(); - Console.WriteLine($"*** Remote folder contains {fileCount} file(s), aborting"); - Console.WriteLine($"*** First {filenames.Count} file(s): {Environment.NewLine}{string.Join(Environment.NewLine, filenames)}"); + Console.WriteLine(LogTimeStamp + $"*** Remote folder contains {fileCount} file(s), aborting"); + Console.WriteLine(LogTimeStamp + $"*** First {filenames.Count} file(s): {Environment.NewLine}{string.Join(Environment.NewLine, filenames)}"); if (fileCount > filenames.Count) - Console.WriteLine($"*** ... and {fileCount - filenames.Count} more file(s)"); + Console.WriteLine(LogTimeStamp + $"*** ... and {fileCount - filenames.Count} more file(s)"); return false; } @@ -232,7 +233,7 @@ namespace Duplicati.CommandLine.BackendTester { if (!(backend is IStreamingBackend) || disableStreaming) { - Console.WriteLine("Warning: Throttling is only supported in this tool on streaming backends"); + Console.WriteLine(LogTimeStamp + "Warning: Throttling is only supported in this tool on streaming backends"); } throttleUpload = Duplicati.Library.Utility.Sizeparser.ParseSize(throttleUploadString, "kb"); @@ -243,16 +244,26 @@ namespace Duplicati.CommandLine.BackendTester { if (!(backend is IStreamingBackend) || disableStreaming) { - Console.WriteLine("Warning: Throttling is only supported in this tool on streaming backends"); + Console.WriteLine(LogTimeStamp + "Warning: Throttling is only supported in this tool on streaming backends"); } throttleDownload = Duplicati.Library.Utility.Sizeparser.ParseSize(throttleDownloadString, "kb"); } - var readWriteTimeout = (int)TimeSpan.FromSeconds(10).TotalMilliseconds; - if (backend is ITimeoutExemptBackend) - readWriteTimeout = Timeout.Infinite; + int readWriteTimeout = backend is ITimeoutExemptBackend || + (Environment.GetEnvironmentVariable("READ_WRITE_TIMEOUT_SECONDS") is { } timeout + && int.TryParse(timeout, out var seconds) + && seconds == -1) + ? Timeout.Infinite + : Environment.GetEnvironmentVariable("READ_WRITE_TIMEOUT_SECONDS") is { } timeoutRetry + && int.TryParse(timeoutRetry, out var secondsRetry) + ? (int)TimeSpan.FromSeconds(secondsRetry).TotalMilliseconds + : (int)TimeSpan.FromSeconds(10).TotalMilliseconds; + Console.WriteLine(LogTimeStamp + "Read Write Timeout set to {0}{1}", + readWriteTimeout == Timeout.Infinite ? "infinite" : readWriteTimeout + " ms", + backend is ITimeoutExemptBackend ? " (because implements ITimeoutExemptBackend)" : ""); + // Allow overriding the timeout for the backend here, even if timeouts are disabled if (options.TryGetValue("read-write-timeout", out var readWriteTimeoutString)) readWriteTimeout = (int)Timeparser.ParseTimeSpan(readWriteTimeoutString).TotalMilliseconds; @@ -298,7 +309,7 @@ namespace Duplicati.CommandLine.BackendTester byte[] dummyFileHash = null; if (!skipOverwriteTest) { - Console.WriteLine("Uploading wrong files ..."); + Console.WriteLine(LogTimeStamp + "Uploading wrong files ..."); using (var dummy = Library.Utility.TempFile.WrapExistingFile(CreateRandomFile(tf, files.Count, 1024, 2048, rnd))) { using (var fs = new System.IO.FileStream(dummy, System.IO.FileMode.Open, System.IO.FileAccess.Read)) @@ -313,7 +324,7 @@ namespace Duplicati.CommandLine.BackendTester } - Console.WriteLine("Uploading files ..."); + Console.WriteLine(LogTimeStamp + "Uploading files ..."); for (int i = 0; i < files.Count; i++) Uploadfile(files[i].localfilename, i, files[i].remotefilename, backend, disableStreaming, throttleUpload, readWriteTimeout); @@ -328,7 +339,7 @@ namespace Duplicati.CommandLine.BackendTester renamedFileNewName = CreateRandomRemoteFileName(min_filename_size, max_filename_size, allowedChars, trimFilenameSpaces, rnd); - Console.WriteLine("Renaming file {0} from {1} to {2}", renameIndex, originalRenamedFile.remotefilename, renamedFileNewName); + Console.WriteLine(LogTimeStamp + "Renaming file {0} from {1} to {2}", renameIndex, originalRenamedFile.remotefilename, renamedFileNewName); renameEnabledBackend.RenameAsync(originalRenamedFile.remotefilename, renamedFileNewName, CancellationToken.None).Await(); files[renameIndex] = new TempFile(renamedFileNewName, originalRenamedFile.localfilename, originalRenamedFile.hash, originalRenamedFile.length); @@ -336,11 +347,11 @@ namespace Duplicati.CommandLine.BackendTester if (waitAfterUpload > TimeSpan.Zero) { - Console.WriteLine("Waiting {0} after upload", waitAfterUpload); + Console.WriteLine(LogTimeStamp + "Waiting {0} after upload", waitAfterUpload); Thread.Sleep(waitAfterUpload); } - Console.WriteLine("Verifying file list ..."); + Console.WriteLine(LogTimeStamp + "Verifying file list ..."); curlist = backend.List(); foreach (var fe in curlist) @@ -351,12 +362,12 @@ namespace Duplicati.CommandLine.BackendTester if (tx.remotefilename == fe.Name) { if (tx.found) - Console.WriteLine("*** File with name {0} was found more than once", tx.remotefilename); + Console.WriteLine(LogTimeStamp + "*** File with name {0} was found more than once", tx.remotefilename); found = true; tx.found = true; if (fe.Size > 0 && tx.length != fe.Size) - Console.WriteLine("*** File with name {0} has size {1} but the size was reported as {2}", tx.remotefilename, tx.length, fe.Size); + Console.WriteLine(LogTimeStamp + "*** File with name {0} has size {1} but the size was reported as {2}", tx.remotefilename, tx.length, fe.Size); break; } @@ -364,26 +375,26 @@ namespace Duplicati.CommandLine.BackendTester if (!found) if (originalRenamedFile != null && renamedFileNewName != null && originalRenamedFile.remotefilename == fe.Name) { - Console.WriteLine("*** File with name {0} was found on server but was supposed to have been renamed to {1}!", fe.Name, renamedFileNewName); + Console.WriteLine(LogTimeStamp + "*** File with name {0} was found on server but was supposed to have been renamed to {1}!", fe.Name, renamedFileNewName); } else { - Console.WriteLine("*** File with name {0} was found on server but not uploaded!", fe.Name); + Console.WriteLine(LogTimeStamp + "*** File with name {0} was found on server but not uploaded!", fe.Name); } } foreach (var tx in files) if (!tx.found) - Console.WriteLine("*** File with name {0} was uploaded but not found afterwards", tx.remotefilename); + Console.WriteLine(LogTimeStamp + "*** File with name {0} was uploaded but not found afterwards", tx.remotefilename); - Console.WriteLine("Downloading files"); + Console.WriteLine(LogTimeStamp + "Downloading files"); for (int i = 0; i < files.Count; i++) { using (var cf = new Duplicati.Library.Utility.TempFile()) { Exception e = null; - Console.Write("Downloading file {0} ... ", i); + Console.Write(LogTimeStamp + "Downloading file {0} ... ", i); var sw = Stopwatch.StartNew(); @@ -410,22 +421,22 @@ namespace Duplicati.CommandLine.BackendTester if (e != null) { failAfterFinished = true; - Console.WriteLine($"failed\n*** Error: {e} after {sw.ElapsedMilliseconds} ms"); + Console.WriteLine(LogTimeStamp + $"failed\n*** Error: {e} after {sw.ElapsedMilliseconds} ms"); } else - Console.WriteLine($"done in {sw.ElapsedMilliseconds} ms"); + Console.WriteLine($" done in {sw.ElapsedMilliseconds} ms"); - Console.Write("Checking hash ... "); + Console.Write(LogTimeStamp + "Checking hash ... "); using (var fs = new System.IO.FileStream(cf, System.IO.FileMode.Open, System.IO.FileAccess.Read)) if (Convert.ToBase64String(sha.ComputeHash(fs)) != Convert.ToBase64String(files[i].hash)) { if (dummyFileHash != null && Convert.ToBase64String(sha.ComputeHash(fs)) == Convert.ToBase64String(dummyFileHash)) - Console.WriteLine("failed\n*** Downloaded file was the dummy file"); // Should this be failed? + Console.WriteLine(LogTimeStamp + "failed\n*** Downloaded file was the dummy file"); // Should this be failed? else { failAfterFinished = true; - Console.WriteLine("failed\n*** Downloaded file was corrupt"); + Console.WriteLine(LogTimeStamp + "failed\n*** Downloaded file was corrupt"); } } else @@ -433,19 +444,19 @@ namespace Duplicati.CommandLine.BackendTester } } - Console.WriteLine("Deleting files..."); + Console.WriteLine(LogTimeStamp + "Deleting files..."); foreach (var tx in files) try { backend.DeleteAsync(tx.remotefilename, CancellationToken.None).Await(); } catch (Exception ex) { failAfterFinished = true; - Console.WriteLine("*** Failed to delete file {0}, message: {1}", tx.remotefilename, ex); + Console.WriteLine(LogTimeStamp + "*** Failed to delete file {0}, message: {1}", tx.remotefilename, ex); } if (waitAfterDelete > TimeSpan.Zero) { - Console.WriteLine("Waiting {0} after delete", waitAfterDelete); + Console.WriteLine(LogTimeStamp + "Waiting {0} after delete", waitAfterDelete); Thread.Sleep(waitAfterDelete); } @@ -453,11 +464,11 @@ namespace Duplicati.CommandLine.BackendTester foreach (var fe in curlist) if (!fe.IsFolder) { - Console.WriteLine("*** Remote folder contains {0} after cleanup", fe.Name); + Console.WriteLine(LogTimeStamp + "*** Remote folder contains {0} after cleanup", fe.Name); } // Test some error cases - Console.WriteLine("Checking retrieval of non-existent file..."); + Console.WriteLine(LogTimeStamp + "Checking retrieval of non-existent file..."); var caughtExpectedException = false; try { @@ -468,25 +479,25 @@ namespace Duplicati.CommandLine.BackendTester } catch (FileMissingException) { - Console.WriteLine("Caught expected FileMissingException"); + Console.WriteLine(LogTimeStamp + "Caught expected FileMissingException"); caughtExpectedException = true; } catch (Exception ex) { - Console.WriteLine("*** Retrieval of non-existent file failed: {0}", ex); + Console.WriteLine(LogTimeStamp + "*** Retrieval of non-existent file failed: {0}", ex); } if (!caughtExpectedException) { failAfterFinished = true; - Console.WriteLine("*** Retrieval of non-existent file should have failed with FileMissingException"); + Console.WriteLine(LogTimeStamp + "*** Retrieval of non-existent file should have failed with FileMissingException"); } } // Test quota retrieval if (backend is IQuotaEnabledBackend quotaEnabledBackend) { - Console.WriteLine("Checking quota..."); + Console.WriteLine(LogTimeStamp + "Checking quota..."); IQuotaInfo quota = null; bool noException; try @@ -496,7 +507,7 @@ namespace Duplicati.CommandLine.BackendTester } catch (Exception ex) { - Console.WriteLine("*** Checking quota information failed: {0}", ex); + Console.WriteLine(LogTimeStamp + "*** Checking quota information failed: {0}", ex); noException = false; } @@ -504,18 +515,18 @@ namespace Duplicati.CommandLine.BackendTester { if (quota != null) { - Console.WriteLine("Free Space: {0}", Library.Utility.Utility.FormatSizeString(quota.FreeQuotaSpace)); - Console.WriteLine("Total Space: {0}", Library.Utility.Utility.FormatSizeString(quota.TotalQuotaSpace)); + Console.WriteLine(LogTimeStamp + "Free Space: {0}", Library.Utility.Utility.FormatSizeString(quota.FreeQuotaSpace)); + Console.WriteLine(LogTimeStamp + "Total Space: {0}", Library.Utility.Utility.FormatSizeString(quota.TotalQuotaSpace)); } else { - Console.WriteLine("Unable to retrieve quota information"); + Console.WriteLine(LogTimeStamp + "Unable to retrieve quota information"); } } } // Test DNSName lookup - Console.WriteLine("Checking DNS names used by this backend..."); + Console.WriteLine(LogTimeStamp + "Checking DNS names used by this backend..."); try { var dnsNames = backend.GetDNSNamesAsync(CancellationToken.None).Await(); @@ -528,12 +539,12 @@ namespace Duplicati.CommandLine.BackendTester } else { - Console.WriteLine("No DNS names reported"); + Console.WriteLine(LogTimeStamp + "No DNS names reported"); } } catch (Exception ex) { - Console.WriteLine("*** Checking DNSName failed: {0}", ex); + Console.WriteLine(LogTimeStamp + "*** Checking DNSName failed: {0}", ex); } } finally @@ -548,7 +559,7 @@ namespace Duplicati.CommandLine.BackendTester private static void Uploadfile(string localfilename, int i, string remotefilename, IBackend backend, bool disableStreaming, long throttle, int readWriteTimeout) { - Console.Write("Uploading file {0}, {1} ... ", i, Duplicati.Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(localfilename).Length)); + Console.Write(LogTimeStamp + "Uploading file {0}, {1} ... ", i, Duplicati.Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(localfilename).Length)); Exception e = null; var sw = Stopwatch.StartNew(); @@ -575,11 +586,11 @@ namespace Duplicati.CommandLine.BackendTester if (e != null) { - Console.WriteLine($"Failed to upload file {i}, error message: {e}, remote name: {remotefilename} after {sw.ElapsedMilliseconds} ms"); + Console.WriteLine(LogTimeStamp + $"Failed to upload file {i}, error message: {e}, remote name: {remotefilename} after {sw.ElapsedMilliseconds} ms"); while (e.InnerException != null) { e = e.InnerException; - Console.WriteLine(" Inner exception: {0}", e); + Console.WriteLine(LogTimeStamp + " Inner exception: {0}", e); } } else @@ -604,7 +615,7 @@ namespace Duplicati.CommandLine.BackendTester private static string CreateRandomFile(Library.Utility.TempFolder tf, int i, int min_file_size, int max_file_size, Random rnd) { - Console.Write("Generating file {0}", i); + Console.Write(LogTimeStamp + "Generating file {0}", i); string filename = System.IO.Path.Combine(tf, i.ToString()); using (System.IO.FileStream fs = new System.IO.FileStream(filename, System.IO.FileMode.CreateNew, System.IO.FileAccess.Write)) { @@ -625,6 +636,8 @@ namespace Duplicati.CommandLine.BackendTester return filename; } + private static string LogTimeStamp => $"[{DateTime.Now:HH:mm:ss fff}] "; + public static IList SupportedCommands { get From 3d194e8da57a38e1bf89eeb311ce3751e46fcb6c Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 28 Jan 2025 14:39:29 +0100 Subject: [PATCH 051/362] Reworded confirm commandline option description --- Duplicati/UnitTest/ToolTests.cs | 3 ++- Tools/RemoteSynchronization/Program.cs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Duplicati/UnitTest/ToolTests.cs b/Duplicati/UnitTest/ToolTests.cs index bd57e6506..4d8072151 100644 --- a/Duplicati/UnitTest/ToolTests.cs +++ b/Duplicati/UnitTest/ToolTests.cs @@ -76,7 +76,8 @@ namespace Duplicati.UnitTest $"file://{l1}", $"file://{l2}", // Pass along multi token options to test that the parser won't fail "--src-options", "ssh-accept-any-fingerprints=true", "ssh-keyfile=/path/to/keyfile", - "--dst-options", "some-other-key=value", "another-key=value2" + "--dst-options", "some-other-key=value", "another-key=value2", + "--confirm" }; var async_call = exe(args); async_call.Wait(); diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 65e61b26f..eaad0acfe 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -53,7 +53,7 @@ namespace RemoteSynchronization var retry_opt = new Option(aliases: ["--retry"], description: "Number of times to retry on errors", getDefaultValue: () => DEFAULT_RETRY) { Arity = ArgumentArity.ExactlyOne }; var force_opt = new Option(aliases: ["--force"], description: "Force the synchronization", getDefaultValue: () => DEFAULT_FORCE); var retention_opt = new Option(aliases: ["--retention"], description: "Toggles whether to keep old files. Any deletes will be renames instead", getDefaultValue: () => DEFAULT_RETENTION); - var confirm_opt = new Option(aliases: ["--confirm"], description: "Ask for confirmation before starting the synchronization", getDefaultValue: () => DEFAULT_CONFIRM); + var confirm_opt = new Option(aliases: ["--confirm"], description: "Automatically confirm the operation", getDefaultValue: () => DEFAULT_CONFIRM); var root_cmd = new RootCommand("Remote Synchronization Tool"); root_cmd.AddArgument(src_arg); From 186a1659dbbdb777e9ee6fc1dda35e9b4b00a55e Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 28 Jan 2025 14:47:46 +0100 Subject: [PATCH 052/362] Add global options to the Remote Synchronization tool --- Tools/RemoteSynchronization/Program.cs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index eaad0acfe..f7e13c356 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2025, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a @@ -54,6 +54,7 @@ namespace RemoteSynchronization var force_opt = new Option(aliases: ["--force"], description: "Force the synchronization", getDefaultValue: () => DEFAULT_FORCE); var retention_opt = new Option(aliases: ["--retention"], description: "Toggles whether to keep old files. Any deletes will be renames instead", getDefaultValue: () => DEFAULT_RETENTION); var confirm_opt = new Option(aliases: ["--confirm"], description: "Automatically confirm the operation", getDefaultValue: () => DEFAULT_CONFIRM); + var global_opts = OptionWithMultipleTokens(aliases: ["--global-options"], description: "Global options all backends. May be overridden by backend specific options (src-options, dst-options)"); var root_cmd = new RootCommand("Remote Synchronization Tool"); root_cmd.AddArgument(src_arg); @@ -66,6 +67,7 @@ namespace RemoteSynchronization root_cmd.AddOption(force_opt); root_cmd.AddOption(retention_opt); root_cmd.AddOption(confirm_opt); + root_cmd.AddOption(global_opts); root_cmd.SetHandler((InvocationContext ctx) => { @@ -88,10 +90,10 @@ namespace RemoteSynchronization var retention = options["retention"] as bool? ?? DEFAULT_RETENTION; var confirm = options["confirm"] as bool? ?? DEFAULT_CONFIRM; - var duplicati_options = new Dictionary() - { - ["dry-run"] = dry_run.ToString() - }; + Dictionary global_options = (options["global-options"] as List) + ?.Select(x => x.Split("=")) + .ToDictionary(x => x[0], x => x[1]) + ?? []; Dictionary src_opts = (options["src-options"] as List) ?.Select(x => x.Split("=")) .ToDictionary(x => x[0], x => x[1]) @@ -100,7 +102,9 @@ namespace RemoteSynchronization ?.Select(x => x.Split("=")) .ToDictionary(x => x[0], x => x[1]) ?? []; - foreach (var x in duplicati_options) + + // Merge the global options into the source and destination options + foreach (var x in global_options) src_opts[x.Key] = dst_opts[x.Key] = x.Value; using var b1 = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(src, src_opts); From dd6e250087edae248491d66f23b55a8074efbd15 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 28 Jan 2025 15:05:37 +0100 Subject: [PATCH 053/362] Verification should be before copy, as it verifies the files that are already there. --- Tools/RemoteSynchronization/Program.cs | 45 +++++++++++++++++--------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index f7e13c356..e09ff28c4 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -115,8 +115,34 @@ namespace RemoteSynchronization var b2s = b2 as IStreamingBackend; System.Diagnostics.Debug.Assert(b2s != null); + // Prepare the operations var (to_copy, to_delete, to_verify) = PrepareFileLists(b1s, b2s, force); + // Verify the files if requested. If the files are not verified, they will be deleted and copied again. + if (verify) + { + // As this is a potentially slow operation, ask for confirmation of the verification) + if (!confirm) + { + Console.WriteLine($"This will verify {to_verify.Count()} files before copying them. Do you want to continue? [y/N]"); + var response = Console.ReadLine(); + if (!response?.Equals("y", StringComparison.CurrentCultureIgnoreCase) ?? true) + { + Console.WriteLine("Aborted"); + return -1; + } + } + + var not_verified = await VerifyAsync(b1s, b2s, to_verify); + + if (not_verified.Any()) + { + Console.WriteLine($"{not_verified.Count()} files failed verification. They will be deleted and copied again."); + to_delete = to_delete.Concat(not_verified); + to_copy = to_copy.Concat(not_verified); + } + } + // As this is a potentially destructive operation, ask for confirmation if (!confirm) { @@ -124,8 +150,6 @@ namespace RemoteSynchronization Console.WriteLine($"This will perform the following actions (in order):"); Console.WriteLine($" - {delete_rename} {to_delete.Count()} files from {dst}"); Console.WriteLine($" - Copy {to_copy.Count()} files from {src} to {dst}"); - if (verify) - Console.WriteLine($" - Download and verify {to_verify.Count()} files in {dst}"); Console.WriteLine(); Console.WriteLine("Do you want to continue? [y/N]"); @@ -176,14 +200,6 @@ namespace RemoteSynchronization } } - // Verify the files if requested - if (verify) - { - var not_verified = await VerifyAsync(b1s, b2s, to_verify); - Console.WriteLine($"Could not verify {not_verified.Count()} files: {string.Join(", ", not_verified)}"); - return not_verified.Count(); - } - Console.WriteLine($"Remote synchronization completed successfully"); return 0; @@ -367,10 +383,9 @@ namespace RemoteSynchronization return successful_renames; } - // Post comparison - private static async Task> VerifyAsync(IStreamingBackend b_src, IStreamingBackend b_dst, IEnumerable files) + private static async Task> VerifyAsync(IStreamingBackend b_src, IStreamingBackend b_dst, IEnumerable files) { - var errors = new List(); + var errors = new List(); using var s_src = new MemoryStream(); using var s_dst = new MemoryStream(); @@ -386,7 +401,7 @@ namespace RemoteSynchronization // Compare the contents if (s_src.Length != s_dst.Length || !s_src.ToArray().SequenceEqual(s_dst.ToArray())) { - errors.Add(f.Name); + errors.Add(f); } // Reset the streams @@ -395,7 +410,7 @@ namespace RemoteSynchronization } catch (Exception e) { - errors.Add($"{f.Name}"); + errors.Add(f); Console.WriteLine($"Error verifying {f.Name}: {e.Message}"); } } From 97b6ed88b691f491bc802e12863fcd1f92238564 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 28 Jan 2025 15:06:42 +0100 Subject: [PATCH 054/362] More descriptive wording in descriptions. --- Tools/RemoteSynchronization/Program.cs | 33 ++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index e09ff28c4..7a97bedb3 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2025, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a @@ -32,6 +32,9 @@ using System.Threading.Tasks; namespace RemoteSynchronization { + /// + /// Remote synchronization tool. + /// public class Program { // Default values for the options @@ -56,7 +59,25 @@ namespace RemoteSynchronization var confirm_opt = new Option(aliases: ["--confirm"], description: "Automatically confirm the operation", getDefaultValue: () => DEFAULT_CONFIRM); var global_opts = OptionWithMultipleTokens(aliases: ["--global-options"], description: "Global options all backends. May be overridden by backend specific options (src-options, dst-options)"); - var root_cmd = new RootCommand("Remote Synchronization Tool"); + var root_cmd = new RootCommand(@"Remote Synchronization Tool + +This tool synchronizes two remote backends. The tool assumes that the intent is +to have the destination match the source. + +If the destination has files that are not in the source, they will be deleted +(or renamed if the retention option is set). + +If the destination has files that are in the source but are different in size or +has a newer timestamp, they will be overwritten by the source files. + +If the force option is set, the destination will be overwritten by the source, +regardless of the state of the files. It will also skip the initial comparison, +and delete (or rename) all files in the destination. + +If the verify option is set, the files will be downloaded and compared after +uploading to ensure that the files are correct. Files that already exist in the +destination will be verified before being overwritten (if they seemingly match). + "); root_cmd.AddArgument(src_arg); root_cmd.AddArgument(dst_arg); root_cmd.AddOption(dry_run_opt); @@ -83,6 +104,7 @@ namespace RemoteSynchronization private static async Task Run(string src, string dst, Dictionary options) { + // Parse the known options for this tool var dry_run = options["dry-run"] as bool? ?? DEFAULT_DRY_RUN; var verify = options["verify"] as bool? ?? DEFAULT_VERIFY; var retries = options["retry"] as int? ?? DEFAULT_RETRY; @@ -90,6 +112,7 @@ namespace RemoteSynchronization var retention = options["retention"] as bool? ?? DEFAULT_RETENTION; var confirm = options["confirm"] as bool? ?? DEFAULT_CONFIRM; + // Unpack and parse the multi token options Dictionary global_options = (options["global-options"] as List) ?.Select(x => x.Split("=")) .ToDictionary(x => x[0], x => x[1]) @@ -107,6 +130,7 @@ namespace RemoteSynchronization foreach (var x in global_options) src_opts[x.Key] = dst_opts[x.Key] = x.Value; + // Load the backends using var b1 = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(src, src_opts); var b1s = b1 as IStreamingBackend; System.Diagnostics.Debug.Assert(b1s != null); @@ -214,7 +238,6 @@ namespace RemoteSynchronization // introduce these checks as a post processing step? Especially the database consistency check, as that is often recreated from the index files. // TODO This tool shouldn't handle it, but for convenience, it should support making the seperate call to the regular Duplicati on the destination backend, which alread carries this functionality. - // Forcefully synchronize the remote backends private static async Task<(long, IEnumerable)> CopyAsync(IStreamingBackend b_src, IStreamingBackend b_dst, IEnumerable files, bool dry_run, bool verify) { long successful_copies = 0; @@ -377,7 +400,7 @@ namespace RemoteSynchronization } catch (Exception e) { - Console.WriteLine($"Error renaming {f.Name}: {e.Message}"); + Console.WriteLine($"Error during renaming of {f.Name}: {e.Message}"); } } return successful_renames; @@ -411,7 +434,7 @@ namespace RemoteSynchronization catch (Exception e) { errors.Add(f); - Console.WriteLine($"Error verifying {f.Name}: {e.Message}"); + Console.WriteLine($"Error during verification of {f.Name}: {e.Message}"); } } From ba4a67a9eaa53b995eaa5d3a702067be90fc76db Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 28 Jan 2025 15:17:26 +0100 Subject: [PATCH 055/362] Add docstrings in the Remote Synchronization tool --- Tools/RemoteSynchronization/Program.cs | 61 ++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 7a97bedb3..82b1f88d2 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -45,6 +45,11 @@ namespace RemoteSynchronization private const bool DEFAULT_RETENTION = false; private const bool DEFAULT_CONFIRM = false; + /// + /// Main entry point for the tool. + /// + /// The commandline arguments + /// 0 on success, -1 on abort, and the number of errors encountered otherwise. public static async Task Main(string[] args) { var src_arg = new Argument(name: "backend_src", description: "The source backend string"); @@ -102,6 +107,13 @@ destination will be verified before being overwritten (if they seemingly match). return await root_cmd.InvokeAsync(args); } + /// + /// The main logic of the tool. + /// + /// The connection string for the source backend. + /// The connection string for the destination backend. + /// Various options for the tool + /// The return code for the main entry; 0 on success. private static async Task Run(string src, string dst, Dictionary options) { // Parse the known options for this tool @@ -238,6 +250,16 @@ destination will be verified before being overwritten (if they seemingly match). // introduce these checks as a post processing step? Especially the database consistency check, as that is often recreated from the index files. // TODO This tool shouldn't handle it, but for convenience, it should support making the seperate call to the regular Duplicati on the destination backend, which alread carries this functionality. + /// + /// Copies the files from one backend to another. + /// The files are copied one by one, and each file is verified after uploading if the verify flag is set. + /// + /// The source backend. + /// The destination backend. + /// The files that will be copied. + /// Flag for whether destructive actions (writes) should be printed rather than performed. Downloads (reads) are still being performed. + /// Flag for whether to verify each upload to the destination backend. + /// A tuple holding the number of succesful copies and a List of the files that failed. private static async Task<(long, IEnumerable)> CopyAsync(IStreamingBackend b_src, IStreamingBackend b_dst, IEnumerable files, bool dry_run, bool verify) { long successful_copies = 0; @@ -279,6 +301,13 @@ destination will be verified before being overwritten (if they seemingly match). return (successful_copies, errors); } + /// + /// Deletes the files from a backend. + /// + /// The backend to delete the files from. + /// The files to delete. + /// Flag for whether the deletion should be printed rather than performed. + /// The number of successful deletions. private static async Task DeleteAsync(IStreamingBackend b, IEnumerable files, bool dry_run) { long successful_deletes = 0; @@ -304,6 +333,12 @@ destination will be verified before being overwritten (if they seemingly match). return successful_deletes; } + /// + /// Creates an option that allows multiple tokens and multiple arguments per token. + /// + /// The aliases for the option. + /// The description for the option. + /// The created option. private static Option> OptionWithMultipleTokens(string[] aliases, string description) { return new Option>(aliases: aliases, description: description) @@ -313,6 +348,16 @@ destination will be verified before being overwritten (if they seemingly match). }; } + /// + /// Prepares the lists of files to copy, delete and verify. + /// The files to copy are the files that are not in the destination, have a different size or have a more recent modification date. + /// The files to delete are the files that are found in the destination but not found in the source. + /// The files to verify are the files that are found in both the source and the destination, and that have the same size and modification date. + /// + /// The source backend. + /// The destination backend. + /// Flag for whether to force the synchronization. + /// A tuple of Lists each holding the files to copy, delete and verify. private static (IEnumerable, IEnumerable, IEnumerable) PrepareFileLists(IStreamingBackend b_src, IStreamingBackend b_dst, bool force) { var files_src = b_src.List(); @@ -376,6 +421,14 @@ destination will be verified before being overwritten (if they seemingly match). return (to_copy, to_delete.Select(x => lookup_dst[x]), to_verify); } + /// + /// Renames the files in a backend. + /// The renaming is done by deleting the file and re-uploading it with a new name. + /// + /// The backend to rename the files in. + /// The files to rename. + /// Flag for whether the renaming should be printed rather than performed. + /// The number of successful renames. private static async Task RenameAsync(IStreamingBackend b, IEnumerable files, bool dry_run) { long successful_renames = 0; @@ -406,6 +459,14 @@ destination will be verified before being overwritten (if they seemingly match). return successful_renames; } + /// + /// Verifies the files in the destination backend. + /// The verification is done by downloading the files from the destination backend and comparing them to the source files. + /// + /// The source backend. + /// The destination backend. + /// The files to verify. + /// A list of the files that failed verification. private static async Task> VerifyAsync(IStreamingBackend b_src, IStreamingBackend b_dst, IEnumerable files) { var errors = new List(); From 36f809460b4ee6e24477ae6a241bdc0d2db51818 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 28 Jan 2025 15:41:03 +0100 Subject: [PATCH 056/362] Updated the Remote Synchronization tool to do proper logging. --- Tools/RemoteSynchronization/Program.cs | 81 +++++++++++++++++--------- 1 file changed, 53 insertions(+), 28 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 82b1f88d2..eccf0deb0 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -45,6 +45,11 @@ namespace RemoteSynchronization private const bool DEFAULT_RETENTION = false; private const bool DEFAULT_CONFIRM = false; + /// + /// The log tag for this tool. + /// + private static readonly string LOGTAG = Duplicati.Library.Logging.Log.LogTagFromType(); + /// /// Main entry point for the tool. /// @@ -164,7 +169,7 @@ destination will be verified before being overwritten (if they seemingly match). var response = Console.ReadLine(); if (!response?.Equals("y", StringComparison.CurrentCultureIgnoreCase) ?? true) { - Console.WriteLine("Aborted"); + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Aborted"); return -1; } } @@ -173,7 +178,7 @@ destination will be verified before being overwritten (if they seemingly match). if (not_verified.Any()) { - Console.WriteLine($"{not_verified.Count()} files failed verification. They will be deleted and copied again."); + Duplicati.Library.Logging.Log.WriteWarningMessage(LOGTAG, "rsync", null, "{0} files failed verification. They will be deleted and copied again.", not_verified.Count()); to_delete = to_delete.Concat(not_verified); to_copy = to_copy.Concat(not_verified); } @@ -192,7 +197,7 @@ destination will be verified before being overwritten (if they seemingly match). var response = Console.ReadLine(); if (!response?.Equals("y", StringComparison.CurrentCultureIgnoreCase) ?? true) { - Console.WriteLine("Aborted"); + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Aborted"); return -1; } } @@ -201,29 +206,30 @@ destination will be verified before being overwritten (if they seemingly match). if (retention) { var renamed = await RenameAsync(b2s, to_delete, dry_run); - Console.WriteLine($"Renamed {renamed} files in {dst}"); + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Renamed {0} files in {1}", renamed, b2s.DisplayName); } else { var deleted = await DeleteAsync(b2s, to_delete, dry_run); - Console.WriteLine($"Deleted {deleted} files from {dst}"); + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Deleted {0} files from {1}", deleted, b2s.DisplayName); } // Copy the files var (copied, copy_errors) = await CopyAsync(b1s, b2s, to_copy, dry_run, verify); - Console.WriteLine($"Copied {copied} files from {src} to {dst}"); + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Copied {0} files from {1} to {2}", copied, b1s, b2s); // If there are still errors, retry a few times if (copy_errors.Any()) { + Duplicati.Library.Logging.Log.WriteWarningMessage(LOGTAG, "rsync", null, "Could not copy {0} files.", copy_errors.Count()); if (retries > 0) { - Console.WriteLine($"Retrying {retries} more times to copy the {copy_errors.Count()} files that failed"); + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Retrying {0} more times to copy the {1} files that failed", retries, copy_errors.Count()); for (int i = 0; i < retries; i++) { Thread.Sleep(5000); // Wait 5 seconds before retrying (copied, copy_errors) = await CopyAsync(b1s, b2s, copy_errors, dry_run, verify); - Console.WriteLine($"Copied {copied} files from {src} to {dst}"); + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Copied {0} files from {1} to {2}", copied, b1s, b2s); if (!copy_errors.Any()) break; } @@ -231,12 +237,12 @@ destination will be verified before being overwritten (if they seemingly match). if (copy_errors.Any()) { - Console.WriteLine($"Could not copy {copy_errors.Count()} files: {string.Join(", ", copy_errors)}"); + Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", null, "Could not copy {0} files. Not retrying any more.", copy_errors.Count()); return copy_errors.Count(); } } - Console.WriteLine($"Remote synchronization completed successfully"); + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Remote synchronization completed successfully"); return 0; } @@ -244,7 +250,6 @@ destination will be verified before being overwritten (if they seemingly match). // TODO have concurrency parameters: uploaders, downloaders // TODO low memory mode, where things aren't kept in memory. Maybe utilize SQLite? // TODO Progress reporting - // TODO Logging // check database consistency. I.e. have both databases, check that the block, volume, files, etc match up. // introduce these checks as a post processing step? Especially the database consistency check, as that is often recreated from the index files. @@ -273,7 +278,7 @@ destination will be verified before being overwritten (if they seemingly match). await b_src.GetAsync(f.Name, s_src, CancellationToken.None); if (dry_run) { - Console.WriteLine($"Would write {s_src.Length} bytes of {f.Name} to {b_dst.DisplayName}"); + Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", "Would write {0} bytes of {1} to {2}", s_src.Length, f.Name, b_dst.DisplayName); } else { @@ -281,22 +286,36 @@ destination will be verified before being overwritten (if they seemingly match). if (verify) { await b_dst.GetAsync(f.Name, s_dst, CancellationToken.None); - if (s_src.Length != s_dst.Length || !s_src.ToArray().SequenceEqual(s_dst.ToArray())) + + string? err_string = null; + if (s_src.Length != s_dst.Length) { - Console.WriteLine($"Error verifying {f.Name}: The file was not copied correctly"); - errors.Add(f); + err_string = $"The sizes of the files do not match: {s_src.Length} != {s_dst.Length}."; + } + + if (!s_src.ToArray().SequenceEqual(s_dst.ToArray())) + { + err_string = (err_string is null ? "" : err_string + " ") + "The contents of the files do not match."; + } + + if (err_string is not null) + { + throw new Exception(err_string); } - s_dst.SetLength(0); } } - s_src.SetLength(0); successful_copies++; } catch (Exception e) { - Console.WriteLine($"Error copying {f.Name}: {e.Message}"); + Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e, "Error copying {0}: {1}", f.Name, e.Message); errors.Add(f); } + finally + { + s_src.SetLength(0); + s_dst.SetLength(0); + } } return (successful_copies, errors); } @@ -317,7 +336,7 @@ destination will be verified before being overwritten (if they seemingly match). { if (dry_run) { - Console.WriteLine($"Would delete {f.Name} from {b.DisplayName}"); + Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", "Would delete {0} from {1}", f.Name, b.DisplayName); } else { @@ -327,7 +346,7 @@ destination will be verified before being overwritten (if they seemingly match). } catch (Exception e) { - Console.WriteLine($"Error deleting {f.Name}: {e.Message}"); + Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e, "Error deleting {0}: {1}", f.Name, e.Message); } } return successful_deletes; @@ -441,19 +460,23 @@ destination will be verified before being overwritten (if they seemingly match). await b.GetAsync(f.Name, downloaded, CancellationToken.None); if (dry_run) { - Console.WriteLine($"Would rename {f.Name} to {f.Name}.{suffix} by deleting and re-uploading {downloaded.Length} bytes to {b.DisplayName}"); + Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", "Would rename {0} to {0}.{1} by deleting and re-uploading {2} bytes to {3}", f.Name, suffix, downloaded.Length, b.DisplayName); } else { await b.PutAsync($"{f.Name}.{suffix}", downloaded, CancellationToken.None); await b.DeleteAsync(f.Name, CancellationToken.None); } - downloaded.SetLength(0); successful_renames++; } catch (Exception e) { - Console.WriteLine($"Error during renaming of {f.Name}: {e.Message}"); + Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e, "Error renaming {0}: {1}", f.Name, e.Message); + } + finally + { + // Reset the stream + downloaded.SetLength(0); } } return successful_renames; @@ -487,15 +510,17 @@ destination will be verified before being overwritten (if they seemingly match). { errors.Add(f); } - - // Reset the streams - s_src.SetLength(0); - s_dst.SetLength(0); } catch (Exception e) { errors.Add(f); - Console.WriteLine($"Error during verification of {f.Name}: {e.Message}"); + Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e, "Error during verification of {0}: {1}", f.Name, e.Message); + } + finally + { + // Reset the streams + s_src.SetLength(0); + s_dst.SetLength(0); } } From dc75b4c09cd478bdeaf5d81818224f6c92a1b007 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 29 Jan 2025 04:32:51 +0100 Subject: [PATCH 057/362] Proper OS independant paths in the rsync tool test --- Duplicati/UnitTest/ToolTests.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Duplicati/UnitTest/ToolTests.cs b/Duplicati/UnitTest/ToolTests.cs index 4d8072151..c4997c9b8 100644 --- a/Duplicati/UnitTest/ToolTests.cs +++ b/Duplicati/UnitTest/ToolTests.cs @@ -43,10 +43,10 @@ namespace Duplicati.UnitTest [Category("Tools")] public void TestRemoteSynchronization() { - var l1 = $"{TARGETFOLDER}/l1"; - var l2 = $"{TARGETFOLDER}/l2"; - var l1r = $"{RESTOREFOLDER}/l1_restore"; - var l2r = $"{RESTOREFOLDER}/l2_restore"; + var l1 = Path.Combine(TARGETFOLDER, "l1"); + var l2 = Path.Combine(TARGETFOLDER, "l2"); + var l1r = Path.Combine(RESTOREFOLDER, "l1_restore"); + var l2r = Path.Combine(RESTOREFOLDER, "l2_restore"); var options = TestOptions; From 35f75ce70e0f7d2aed508bcc954bd041025b8e32 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 29 Jan 2025 04:33:29 +0100 Subject: [PATCH 058/362] Proper forwarding of the test options to the rsync tool during test --- Duplicati/UnitTest/ToolTests.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Duplicati/UnitTest/ToolTests.cs b/Duplicati/UnitTest/ToolTests.cs index c4997c9b8..a3312be75 100644 --- a/Duplicati/UnitTest/ToolTests.cs +++ b/Duplicati/UnitTest/ToolTests.cs @@ -72,13 +72,14 @@ namespace Duplicati.UnitTest // Call the tool var exe = RemoteSynchronization.Program.Main; - var args = new string[] { + string[] args = [ $"file://{l1}", $"file://{l2}", + "--global-options", ..options.Select(x => $"{x.Key}={x.Value}"), // Pass along multi token options to test that the parser won't fail "--src-options", "ssh-accept-any-fingerprints=true", "ssh-keyfile=/path/to/keyfile", "--dst-options", "some-other-key=value", "another-key=value2", "--confirm" - }; + ]; var async_call = exe(args); async_call.Wait(); Assert.AreEqual(0, async_call.Result, "Remote synchronization tool did not return 0."); From c40ee2e1565ee58e7416decc7511c324c39e4031 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 29 Jan 2025 04:33:54 +0100 Subject: [PATCH 059/362] Added light profiling to the console output of the rsync tool test --- Duplicati/UnitTest/ToolTests.cs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Duplicati/UnitTest/ToolTests.cs b/Duplicati/UnitTest/ToolTests.cs index a3312be75..66a9d09ba 100644 --- a/Duplicati/UnitTest/ToolTests.cs +++ b/Duplicati/UnitTest/ToolTests.cs @@ -64,13 +64,15 @@ namespace Duplicati.UnitTest // Backup the first level using (var c = new Controller($"file://{l1}", options, null)) { + now = DateTime.Now; var results = c.Backup([DATAFOLDER]); Assert.AreEqual(0, results.Errors.Count()); Assert.AreEqual(0, results.Warnings.Count()); - Console.WriteLine($"Backed up {results.AddedFiles} files to {l1}"); + Console.WriteLine($"Backed up {results.AddedFiles} files to {l1} in {DateTime.Now - now}"); } // Call the tool + now = DateTime.Now; var exe = RemoteSynchronization.Program.Main; string[] args = [ $"file://{l1}", $"file://{l2}", @@ -83,6 +85,7 @@ namespace Duplicati.UnitTest var async_call = exe(args); async_call.Wait(); Assert.AreEqual(0, async_call.Result, "Remote synchronization tool did not return 0."); + Console.WriteLine($"Remote synchronization tool returned 0 in {DateTime.Now - now}"); // Verify that the directories are equal Assert.IsTrue(DirectoriesAndContentsAreEqual(l1, l2), "Synchronized directories are not equal"); @@ -91,10 +94,11 @@ namespace Duplicati.UnitTest options["restore-path"] = l1r; using (var c = new Controller($"file://{l1}", options, null)) { + now = DateTime.Now; var results = c.Restore([Path.Combine(DATAFOLDER, "*")]); Assert.AreEqual(0, results.Errors.Count()); Assert.AreEqual(0, results.Warnings.Count()); - Console.WriteLine($"Restored {results.RestoredFiles} files to {options["restore-path"]}"); + Console.WriteLine($"Restored {results.RestoredFiles} files to {options["restore-path"]} in {DateTime.Now - now}"); } Assert.IsTrue(DirectoriesAndContentsAreEqual(DATAFOLDER, l1r), "Restored first level files is not equal to original files"); @@ -102,10 +106,11 @@ namespace Duplicati.UnitTest options["restore-path"] = l2r; using (var c = new Controller($"file://{l2}", options, null)) { + now = DateTime.Now; var results = c.Restore([]); Assert.AreEqual(0, results.Errors.Count()); Assert.AreEqual(0, results.Warnings.Count()); - Console.WriteLine($"Restored {results.RestoredFiles} files to {options["restore-path"]}"); + Console.WriteLine($"Restored {results.RestoredFiles} files to {options["restore-path"]} in {DateTime.Now - now}"); } Assert.IsTrue(DirectoriesAndContentsAreEqual(DATAFOLDER, l2r), "Restored second level files is not equal to original files"); } From b1e23da40cc6cdf051dabd2543b760ff1e34884b Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 29 Jan 2025 11:04:13 +0100 Subject: [PATCH 060/362] Implemented correct shutdown of backend manager handler --- .../Main/Backend/BackendManager.Handler.cs | 101 ++++++++++++------ 1 file changed, 68 insertions(+), 33 deletions(-) diff --git a/Duplicati/Library/Main/Backend/BackendManager.Handler.cs b/Duplicati/Library/Main/Backend/BackendManager.Handler.cs index b059a0f1f..915d12f4d 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.Handler.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.Handler.cs @@ -218,40 +218,72 @@ partial class BackendManager /// An awaitable task private async Task Run(IReadChannel requestChannel) { - while (true) + using var tcs = new CancellationTokenSource(); + try { - // Get next operation - var op = await requestChannel.ReadAsync().ConfigureAwait(false); - - try + while (true) { - // Clean up completed uploads, if any - await ReclaimCompletedUploads().ConfigureAwait(false); + // Get next operation + var op = await requestChannel.ReadAsync().ConfigureAwait(false); - // Allow PUT operations to be queued, if requested - if (op is PutOperation putOp && !putOp.WaitForComplete) + try { - await EnsureAtMostNPendingUploads(maxParallelUploads).ConfigureAwait(false); + // Clean up completed uploads, if any + await ReclaimCompletedUploads().ConfigureAwait(false); - // Operation is accepted into queue, so we can signal completion - putOp.SetComplete(true); - activeUploads.Add(ExecuteWithRetry(putOp)); + // Allow PUT operations to be queued, if requested + if (op is PutOperation putOp && !putOp.WaitForComplete) + { + await EnsureAtMostNPendingUploads(maxParallelUploads).ConfigureAwait(false); + + // Operation is accepted into queue, so we can signal completion + putOp.SetComplete(true); + activeUploads.Add(ExecuteWithRetry(putOp, tcs.Token)); + } + else + { + // Wait for all pending uploads to complete + await EnsureAtMostNPendingUploads(1).ConfigureAwait(false); + // Execute the operation + await ExecuteWithRetry(op, tcs.Token).ConfigureAwait(false); + } } - else + catch (Exception ex) { - // Wait for all pending uploads to complete - await EnsureAtMostNPendingUploads(1).ConfigureAwait(false); - // Execute the operation - await ExecuteWithRetry(op).ConfigureAwait(false); + Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerHandlerFailure", ex, "Error in handler: {0}", ex.Message); + // If we fail, the task may "hang", so we ensure it is completed here + op.SetFailed(ex); + throw; } } - catch (Exception ex) + } + finally + { + // Terminate any pending uploads + tcs.Cancel(); + + if (activeUploads.Count > 0) { - Logging.Log.WriteWarningMessage(LOGTAG, "HandlerError", ex, "Error in handler: {0}", ex.Message); - // If we fail, the task may "hang", so we ensure it is completed here - op.SetFailed(ex); - throw; + Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerDisposeWhileActive", null, "Terminating {0} pending uploads", activeUploads.Count); + + // Wait for all pending uploads to complete + await Task.WhenAny(Task.Delay(1000), Task.WhenAll(activeUploads)).ConfigureAwait(false); + for (int i = activeUploads.Count - 1; i >= 0; i--) + { + var t = activeUploads[i]; + activeUploads.RemoveAt(i); + if (t.IsCompleted && !t.IsCanceled && t.Exception != null) + Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerDisposeError", t.Exception, "Error in pending upload: {0}", t.Exception.Message); + } + + if (activeUploads.Count > 0) + Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerDisposeError", null, "Terminating, but {0} pending uploads are still active", activeUploads.Count); } + + // Dispose of any remaining backends + while (backendPool.TryDequeue(out var backend)) + try { backend.Dispose(); } + catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerDisposeError", ex, "Failed to dispose backend instance: {0}", ex.Message); } } } @@ -282,8 +314,9 @@ partial class BackendManager /// Executes an operation with retries and error handling /// /// The operation to execute + /// The cancellation token /// An awaitable task - private async Task ExecuteWithRetry(PendingOperationBase op) + private async Task ExecuteWithRetry(PendingOperationBase op, CancellationToken cancellationToken) { // Once in this method, we MUST set the op result, // or the program will hang waiting for the operation to complete @@ -296,7 +329,7 @@ partial class BackendManager try { // Happy case is execute and return - await Execute(op).ConfigureAwait(false); + await Execute(op, cancellationToken).ConfigureAwait(false); return; } catch (Exception ex) @@ -369,29 +402,30 @@ partial class BackendManager /// but avoids a reflection-based dispatch. /// /// The operation to execute + /// The cancellation token /// An awaitable task - private async Task Execute(PendingOperationBase op) + private async Task Execute(PendingOperationBase op, CancellationToken cancellationToken) { await context.TaskReader.ProgressRendevouz().ConfigureAwait(false); using (new Logging.Timer(LOGTAG, $"RemoteOperation{op.Operation}", $"RemoteOperation{op.Operation}")) switch (op) { case PutOperation putOp: - await Execute(putOp).ConfigureAwait(false); + await Execute(putOp, cancellationToken).ConfigureAwait(false); anyUploaded = true; return; case GetOperation getOp: - await Execute(getOp).ConfigureAwait(false); + await Execute(getOp, cancellationToken).ConfigureAwait(false); anyDownloaded = true; return; case DeleteOperation deleteOp: - await Execute(deleteOp).ConfigureAwait(false); + await Execute(deleteOp, cancellationToken).ConfigureAwait(false); return; case ListOperation listOp: - await Execute(listOp).ConfigureAwait(false); + await Execute(listOp, cancellationToken).ConfigureAwait(false); return; case QuotaInfoOperation quotaOp: - await Execute(quotaOp).ConfigureAwait(false); + await Execute(quotaOp, cancellationToken).ConfigureAwait(false); return; case WaitForEmptyOperation waitOp: waitOp.SetComplete(true); @@ -406,11 +440,12 @@ partial class BackendManager /// /// The return value type of the operation /// The operation to execute + /// The cancellation token /// An awaitable task - private async Task Execute(PendingOperation op) + private async Task Execute(PendingOperation op, CancellationToken cancellationToken) { using var backend = CreateBackend(); - using var token = CancellationTokenSource.CreateLinkedTokenSource(op.CancelToken, context.TaskReader.TransferToken); + using var token = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, op.CancelToken, context.TaskReader.TransferToken); try { From 8b37599c39f1dd39b9640544512d1ed5c65ae6af Mon Sep 17 00:00:00 2001 From: Stavros Kois Date: Wed, 29 Jan 2025 20:37:57 +0200 Subject: [PATCH 061/362] add docker tag $version-$channel Signed-off-by: Stavros Kois --- ReleaseBuilder/Build/Command.CreatePackage.cs | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/ReleaseBuilder/Build/Command.CreatePackage.cs b/ReleaseBuilder/Build/Command.CreatePackage.cs index e7ef0f9ba..d4d029bad 100644 --- a/ReleaseBuilder/Build/Command.CreatePackage.cs +++ b/ReleaseBuilder/Build/Command.CreatePackage.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System.Globalization; using System.IO.Compression; @@ -748,7 +748,7 @@ public static partial class Command { // The approach here is based on: // https://www.internalpointers.com/post/build-binary-deb-package-practical-guide - // + // // It is not the recommended way to build a package, // but since the build is from a pre-build binary, // it is easier than trying to hack debhelper. @@ -1195,7 +1195,7 @@ public static partial class Command await PackageSupport.SetPermissionFlags(tgfolder, rtcfg); } - var tags = new List { rtcfg.ReleaseInfo.Channel.ToString(), rtcfg.ReleaseInfo.Version.ToString() }; + var tags = new List { rtcfg.ReleaseInfo.Channel.ToString(), rtcfg.ReleaseInfo.Version.ToString(), $"{rtcfg.ReleaseInfo.Version}-{rtcfg.ReleaseInfo.Channel}" }; if (rtcfg.ReleaseInfo.Channel == ReleaseChannel.Stable) tags.Add("latest"); From 0fca2ba96b8bb8b6a1bffcc23cf3af8f35b7c901 Mon Sep 17 00:00:00 2001 From: "Marcelo C." Date: Wed, 29 Jan 2025 20:37:40 -0300 Subject: [PATCH 062/362] Changing default streaming timeout from 10 to 60 seconds --- Duplicati/CommandLine/BackendTester/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/CommandLine/BackendTester/Program.cs b/Duplicati/CommandLine/BackendTester/Program.cs index 5bee5cc52..63fadd9fa 100644 --- a/Duplicati/CommandLine/BackendTester/Program.cs +++ b/Duplicati/CommandLine/BackendTester/Program.cs @@ -258,7 +258,7 @@ namespace Duplicati.CommandLine.BackendTester : Environment.GetEnvironmentVariable("READ_WRITE_TIMEOUT_SECONDS") is { } timeoutRetry && int.TryParse(timeoutRetry, out var secondsRetry) ? (int)TimeSpan.FromSeconds(secondsRetry).TotalMilliseconds - : (int)TimeSpan.FromSeconds(10).TotalMilliseconds; + : (int)TimeSpan.FromSeconds(60).TotalMilliseconds; Console.WriteLine(LogTimeStamp + "Read Write Timeout set to {0}{1}", readWriteTimeout == Timeout.Infinite ? "infinite" : readWriteTimeout + " ms", From de9ebfa01016ca92fc34f5f9c77c76257590a110 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 30 Jan 2025 06:50:28 +0100 Subject: [PATCH 063/362] RenameAsync now calls the backends rename functionality, if it supports it. --- Tools/RemoteSynchronization/Program.cs | 79 ++++++++++++++++++-------- 1 file changed, 55 insertions(+), 24 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index eccf0deb0..02ba61222 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2025, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a @@ -205,7 +205,7 @@ destination will be verified before being overwritten (if they seemingly match). // Delete or rename the files that are not needed if (retention) { - var renamed = await RenameAsync(b2s, to_delete, dry_run); + var renamed = await RenameAsync(b2, to_delete, dry_run); Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Renamed {0} files in {1}", renamed, b2s.DisplayName); } else @@ -448,38 +448,69 @@ destination will be verified before being overwritten (if they seemingly match). /// The files to rename. /// Flag for whether the renaming should be printed rather than performed. /// The number of successful renames. - private static async Task RenameAsync(IStreamingBackend b, IEnumerable files, bool dry_run) + private static async Task RenameAsync(IBackend b, IEnumerable files, bool dry_run) { long successful_renames = 0; string suffix = $"{System.DateTime.Now:yyyyMMddHHmmss}.old"; using var downloaded = new MemoryStream(); - foreach (var f in files) + switch (b) { - try - { - await b.GetAsync(f.Name, downloaded, CancellationToken.None); - if (dry_run) + case IStreamingBackend sb: { - Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", "Would rename {0} to {0}.{1} by deleting and re-uploading {2} bytes to {3}", f.Name, suffix, downloaded.Length, b.DisplayName); + foreach (var f in files) + { + try + { + await sb.GetAsync(f.Name, downloaded, CancellationToken.None); + if (dry_run) + { + Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", "Would rename {0} to {0}.{1} by deleting and re-uploading {2} bytes to {3}", f.Name, suffix, downloaded.Length, sb.DisplayName); + } + else + { + await sb.PutAsync($"{f.Name}.{suffix}", downloaded, CancellationToken.None); + await sb.DeleteAsync(f.Name, CancellationToken.None); + } + successful_renames++; + } + catch (Exception e) + { + Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e, "Error renaming {0}: {1}", f.Name, e.Message); + } + finally + { + // Reset the stream + downloaded.SetLength(0); + } + } + return successful_renames; } - else + case IRenameEnabledBackend rb: { - await b.PutAsync($"{f.Name}.{suffix}", downloaded, CancellationToken.None); - await b.DeleteAsync(f.Name, CancellationToken.None); + foreach (var f in files) + { + try + { + if (dry_run) + { + Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", "Would rename {0} to {0}.{1} by calling Rename on {2}", f.Name, suffix, rb.DisplayName); + } + else + { + await rb.RenameAsync(f.Name, $"{f.Name}.{suffix}", CancellationToken.None); + } + successful_renames++; + } + catch (Exception e) + { + Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e, "Error renaming {0}: {1}", f.Name, e.Message); + } + } + return successful_renames; } - successful_renames++; - } - catch (Exception e) - { - Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e, "Error renaming {0}: {1}", f.Name, e.Message); - } - finally - { - // Reset the stream - downloaded.SetLength(0); - } + default: + throw new NotSupportedException("The backend does not support renaming"); } - return successful_renames; } /// From bf0808466259da0d1ea5e3be69088b1744e3dac0 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 30 Jan 2025 06:54:25 +0100 Subject: [PATCH 064/362] Added additonal TODOs --- Tools/RemoteSynchronization/Program.cs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 02ba61222..a2a4efd09 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -249,11 +249,13 @@ destination will be verified before being overwritten (if they seemingly match). // TODO have concurrency parameters: uploaders, downloaders // TODO low memory mode, where things aren't kept in memory. Maybe utilize SQLite? - // TODO Progress reporting + // TODO For convenience, have the option to launch a "duplicati test" on the destination backend after the synchronization + // TODO Save hash to minimize redownload - // check database consistency. I.e. have both databases, check that the block, volume, files, etc match up. - // introduce these checks as a post processing step? Especially the database consistency check, as that is often recreated from the index files. - // TODO This tool shouldn't handle it, but for convenience, it should support making the seperate call to the regular Duplicati on the destination backend, which alread carries this functionality. + // TODO Profiling logging + // TODO Progress reporting + // TODO Duplicati Results + // TODO Log-level and log-file /// /// Copies the files from one backend to another. From 40271166eec77a13293a9697e057f04db1640b00 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 30 Jan 2025 07:12:36 +0100 Subject: [PATCH 065/362] Push before changing machine --- Tools/RemoteSynchronization/Program.cs | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index a2a4efd09..1d8035671 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2025, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a @@ -44,6 +44,8 @@ namespace RemoteSynchronization private const bool DEFAULT_FORCE = false; private const bool DEFAULT_RETENTION = false; private const bool DEFAULT_CONFIRM = false; + private const string DEFAULT_LOG_LEVEL = "Information"; + private const string DEFAULT_LOG_FILE = ""; /// /// The log tag for this tool. @@ -59,7 +61,7 @@ namespace RemoteSynchronization { var src_arg = new Argument(name: "backend_src", description: "The source backend string"); var dst_arg = new Argument(name: "backend_dst", description: "The destination backend string"); - var dry_run_opt = new Option(aliases: ["--dry-run", "-d"], description: "Do not actually write or delete files", getDefaultValue: () => DEFAULT_DRY_RUN); + var dry_run_opt = new Option(aliases: ["--dry-run", "-d"], description: "Do not actually write or delete files. If not set here, the global options will be checked", getDefaultValue: () => DEFAULT_DRY_RUN); var src_opts = OptionWithMultipleTokens(aliases: ["--src-options"], description: "Options for the source backend"); var dst_opts = OptionWithMultipleTokens(aliases: ["--dst-options"], description: "Options for the destination backend"); var verify_opt = new Option(aliases: ["--verify"], description: "Verify the files after copying", getDefaultValue: () => DEFAULT_VERIFY); @@ -68,6 +70,8 @@ namespace RemoteSynchronization var retention_opt = new Option(aliases: ["--retention"], description: "Toggles whether to keep old files. Any deletes will be renames instead", getDefaultValue: () => DEFAULT_RETENTION); var confirm_opt = new Option(aliases: ["--confirm"], description: "Automatically confirm the operation", getDefaultValue: () => DEFAULT_CONFIRM); var global_opts = OptionWithMultipleTokens(aliases: ["--global-options"], description: "Global options all backends. May be overridden by backend specific options (src-options, dst-options)"); + var log_level_opt = new Option(aliases: ["--log-level"], description: "The log level to use. If not set here, global options will be checked", getDefaultValue: () => DEFAULT_LOG_LEVEL) { Arity = ArgumentArity.ExactlyOne }; + var log_file_opt = new Option(aliases: ["--log-file"], description: "The log file to write to. If not set here, global options will be checked", getDefaultValue: () => DEFAULT_LOG_FILE) { Arity = ArgumentArity.ExactlyOne }; var root_cmd = new RootCommand(@"Remote Synchronization Tool @@ -99,6 +103,8 @@ destination will be verified before being overwritten (if they seemingly match). root_cmd.AddOption(retention_opt); root_cmd.AddOption(confirm_opt); root_cmd.AddOption(global_opts); + root_cmd.AddOption(log_level_opt); + root_cmd.AddOption(log_file_opt); root_cmd.SetHandler((InvocationContext ctx) => { @@ -134,6 +140,22 @@ destination will be verified before being overwritten (if they seemingly match). ?.Select(x => x.Split("=")) .ToDictionary(x => x[0], x => x[1]) ?? []; + + var log_file = options["log-file"] as string ?? DEFAULT_LOG_FILE; + + // Parse the log level + var log_level = options["log-level"] as string ?? DEFAULT_LOG_LEVEL; + Enum.TryParse(log_level, true, out var log_level_enum); + // TODO Du er naaet hertil + Duplicati.Library.Logging.Log.LogLevel = log_level_enum; + else + foreach (string s in Enum.GetNames(typeof(Duplicati.Library.Logging.LogMessageType))) + if (s.Equals(log_level, StringComparison.OrdinalIgnoreCase)) + return (Duplicati.Library.Logging.LogMessageType)Enum.Parse(typeof(Duplicati.Library.Logging.LogMessageType), s); + + //var log_target = string.IsNullOrEmpty(log_file) ? new Duplicati.Library.Logging. + using Duplicati.Library.Logging.Log.StartScope(); + Dictionary src_opts = (options["src-options"] as List) ?.Select(x => x.Split("=")) .ToDictionary(x => x[0], x => x[1]) From bc8f6dd0c84c7a6d1a51b1249d78082f4c7ffef5 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 30 Jan 2025 10:52:17 +0100 Subject: [PATCH 066/362] This adds the option to parse flags, where an enum can have multiple values. It is only used by the FTP backend. --- Duplicati/Library/Backend/FTP/FTPBackend.cs | 2 +- Duplicati/Library/Utility/Utility.cs | 40 +++++++++++++++++++-- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Backend/FTP/FTPBackend.cs b/Duplicati/Library/Backend/FTP/FTPBackend.cs index 41a17fb94..939311ab5 100644 --- a/Duplicati/Library/Backend/FTP/FTPBackend.cs +++ b/Duplicati/Library/Backend/FTP/FTPBackend.cs @@ -324,7 +324,7 @@ namespace Duplicati.Library.Backend var dataConnectionType = CoreUtility.ParseEnumOption(options, CONFIG_KEY_FTP_DATA_CONNECTION_TYPE, DEFAULT_DATA_CONNECTION_TYPE); var encryptionMode = CoreUtility.ParseEnumOption(options, CONFIG_KEY_FTP_ENCRYPTION_MODE, DEFAULT_ENCRYPTION_MODE); - var sslProtocols = CoreUtility.ParseEnumOption(options, CONFIG_KEY_FTP_SSL_PROTOCOLS, DEFAULT_SSL_PROTOCOLS); + var sslProtocols = CoreUtility.ParseFlagsOption(options, CONFIG_KEY_FTP_SSL_PROTOCOLS, DEFAULT_SSL_PROTOCOLS); // Process options of the legacy FTP backend if (ProtocolKey == "ftp") diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index 8525e1d27..4d2d8e423 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -758,7 +758,20 @@ namespace Duplicati.Library.Utility /// The value to look for in the settings /// The default value to return if there are no matches. /// The enum type parameter. - public static T ParseEnumOption(IReadOnlyDictionary options, string value, T @default) + public static T ParseEnumOption(IReadOnlyDictionary options, string value, T @default) where T : struct, Enum + { + return options.TryGetValue(value, out var opt) ? ParseEnum(opt, @default) : @default; + } + + /// + /// Parses a flags-type enum found in the options dictionary + /// + /// The parsed or default enum value. + /// The set of options to look for the setting in + /// The value to look for in the settings + /// The default value to return if there are no matches. + /// The enum type parameter. + public static T ParseFlagsOption(IReadOnlyDictionary options, string value, T @default) where T : struct, Enum { return options.TryGetValue(value, out var opt) ? ParseEnum(opt, @default) : @default; } @@ -770,7 +783,7 @@ namespace Duplicati.Library.Utility /// The string to parse. /// The default value to return if there are no matches. /// The enum type parameter. - public static T ParseEnum(string? value, T @default) + public static T ParseEnum(string? value, T @default) where T : struct, Enum { if (string.IsNullOrWhiteSpace(value)) return @default; @@ -781,6 +794,29 @@ namespace Duplicati.Library.Utility return @default; } + /// + /// Parses a string into a flags enum value. + /// + /// The enum type to parse. + /// The value to parse. + /// The default value to return if there are no matches. + /// + public static T ParseFlags(string? value, T @default) where T : struct, Enum + { + if (string.IsNullOrWhiteSpace(value)) + return @default; + + var flags = 0; + foreach (var s in value.Split([','], StringSplitOptions.RemoveEmptyEntries)) + { + var trimmed = s.Trim(); + if (Enum.TryParse(trimmed, true, out T flag)) + flags = flags | (int)(object)flag; + } + + return (T)(object)flags; + } + /// /// Converts a sequence of bytes to a hex string /// From 4b6d73ddbdcf059a72f8012a7a49a6c935133d03 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 30 Jan 2025 13:22:45 +0100 Subject: [PATCH 067/362] Included Duplicati.CommandLine to get the console output sink for logging. --- Duplicati/Tools/Duplicati.Tools.csproj | 3 --- Duplicati/UnitTest/Duplicati.UnitTest.csproj | 7 ++++--- Tools/RemoteSynchronization/RemoteSynchronization.csproj | 1 + 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Duplicati/Tools/Duplicati.Tools.csproj b/Duplicati/Tools/Duplicati.Tools.csproj index 28ca8eea6..50e4b572c 100644 --- a/Duplicati/Tools/Duplicati.Tools.csproj +++ b/Duplicati/Tools/Duplicati.Tools.csproj @@ -24,9 +24,6 @@ utility-scripts\pause-resume.py - - - diff --git a/Duplicati/UnitTest/Duplicati.UnitTest.csproj b/Duplicati/UnitTest/Duplicati.UnitTest.csproj index d8a930cdf..ee728f479 100644 --- a/Duplicati/UnitTest/Duplicati.UnitTest.csproj +++ b/Duplicati/UnitTest/Duplicati.UnitTest.csproj @@ -8,7 +8,7 @@ Duplicati.UnitTest Copyright © 2024 Team Duplicati, MIT license - + @@ -35,14 +35,15 @@ + - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/Tools/RemoteSynchronization/RemoteSynchronization.csproj b/Tools/RemoteSynchronization/RemoteSynchronization.csproj index 7324b2069..7e044266f 100644 --- a/Tools/RemoteSynchronization/RemoteSynchronization.csproj +++ b/Tools/RemoteSynchronization/RemoteSynchronization.csproj @@ -9,6 +9,7 @@ + From 22252d48abcc20a87a7ab6a228c282d5afdcd5d1 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 30 Jan 2025 13:23:48 +0100 Subject: [PATCH 068/362] Implemented proper Duplicati logging in the remote synchronization tool --- Tools/RemoteSynchronization/Program.cs | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 1d8035671..f31721228 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -141,20 +141,20 @@ destination will be verified before being overwritten (if they seemingly match). .ToDictionary(x => x[0], x => x[1]) ?? []; - var log_file = options["log-file"] as string ?? DEFAULT_LOG_FILE; - // Parse the log level var log_level = options["log-level"] as string ?? DEFAULT_LOG_LEVEL; - Enum.TryParse(log_level, true, out var log_level_enum); - // TODO Du er naaet hertil - Duplicati.Library.Logging.Log.LogLevel = log_level_enum; - else - foreach (string s in Enum.GetNames(typeof(Duplicati.Library.Logging.LogMessageType))) - if (s.Equals(log_level, StringComparison.OrdinalIgnoreCase)) - return (Duplicati.Library.Logging.LogMessageType)Enum.Parse(typeof(Duplicati.Library.Logging.LogMessageType), s); + var log_level_parsed = Enum.TryParse(log_level, true, out var log_level_enum); + log_level_enum = log_level_parsed ? log_level_enum : Duplicati.Library.Logging.LogMessageType.Information; - //var log_target = string.IsNullOrEmpty(log_file) ? new Duplicati.Library.Logging. - using Duplicati.Library.Logging.Log.StartScope(); + // Parse the log file + var log_file = options["log-file"] as string ?? DEFAULT_LOG_FILE; + var file_sink = (Duplicati.Library.Main.IMessageSink?)(log_file != "" ? new Duplicati.Library.Logging.StreamLogDestination(log_file) : null); + var console_sink = new Duplicati.CommandLine.ConsoleOutput(Console.Out, global_options); + Duplicati.Library.Main.IMessageSink[] sinks = file_sink is null ? [console_sink] : [file_sink, console_sink]; + var multi_sink = new Duplicati.Library.Main.MultiMessageSink(sinks); + + // Start the logging scope + using var _ = Duplicati.Library.Logging.Log.StartScope(multi_sink, log_level_enum); Dictionary src_opts = (options["src-options"] as List) ?.Select(x => x.Split("=")) @@ -277,7 +277,6 @@ destination will be verified before being overwritten (if they seemingly match). // TODO Profiling logging // TODO Progress reporting // TODO Duplicati Results - // TODO Log-level and log-file /// /// Copies the files from one backend to another. From 23e72336919977b6eb4ec482af8b5e6c954951a5 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 30 Jan 2025 21:39:41 +0100 Subject: [PATCH 069/362] Do not encode http form extra parameters. This changes the encoding that broke because the encoding function was changed. The new logic is to not encode the parameters, but just use them as provided. Any escaping needs to then be applied to the input by the user/caller. This fixes #5900 --- Duplicati/Library/Modules/Builtin/SendHttpMessage.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Modules/Builtin/SendHttpMessage.cs b/Duplicati/Library/Modules/Builtin/SendHttpMessage.cs index 952dea7fc..94f97bc56 100644 --- a/Duplicati/Library/Modules/Builtin/SendHttpMessage.cs +++ b/Duplicati/Library/Modules/Builtin/SendHttpMessage.cs @@ -289,7 +289,7 @@ namespace Duplicati.Library.Modules.Builtin contenttype = new MediaTypeHeaderValue("application/x-www-form-urlencoded"); var postData = $"{m_messageParameterName}={System.Uri.EscapeDataString(body)}"; if (!string.IsNullOrEmpty(m_extraParameters)) - postData += $"&{System.Uri.EscapeDataString(m_extraParameters)}"; + postData += $"&{m_extraParameters}"; data = Encoding.UTF8.GetBytes(postData); } From 31f506c6a1c305912aec6c3709c77d1ebca7af89 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 31 Jan 2025 13:57:45 +0100 Subject: [PATCH 070/362] Moved configuration handling to use a global config class to simplify method signatures --- Tools/RemoteSynchronization/Program.cs | 56 +++++++++++++++++--------- 1 file changed, 36 insertions(+), 20 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index f31721228..56672127d 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2025, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a @@ -47,6 +47,18 @@ namespace RemoteSynchronization private const string DEFAULT_LOG_LEVEL = "Information"; private const string DEFAULT_LOG_FILE = ""; + /// + /// Global configuration for the tool. Should be set after parsing the commandline arguments. + /// + private class GlobalConfig + { + internal static bool DryRun = DEFAULT_DRY_RUN; + internal static bool Force = DEFAULT_FORCE; + internal static string LogLevel = DEFAULT_LOG_LEVEL; + internal static string LogFile = DEFAULT_LOG_FILE; + internal static bool Verify = DEFAULT_VERIFY; + } + /// /// The log tag for this tool. /// @@ -128,10 +140,12 @@ destination will be verified before being overwritten (if they seemingly match). private static async Task Run(string src, string dst, Dictionary options) { // Parse the known options for this tool - var dry_run = options["dry-run"] as bool? ?? DEFAULT_DRY_RUN; - var verify = options["verify"] as bool? ?? DEFAULT_VERIFY; + GlobalConfig.Force = options["force"] as bool? ?? DEFAULT_FORCE; + GlobalConfig.DryRun = options["dry-run"] as bool? ?? DEFAULT_DRY_RUN; + GlobalConfig.LogLevel = options["log-level"] as string ?? DEFAULT_LOG_LEVEL; + GlobalConfig.LogFile = options["log-file"] as string ?? DEFAULT_LOG_FILE; + GlobalConfig.Verify = options["verify"] as bool? ?? DEFAULT_VERIFY; var retries = options["retry"] as int? ?? DEFAULT_RETRY; - var force = options["force"] as bool? ?? DEFAULT_FORCE; var retention = options["retention"] as bool? ?? DEFAULT_RETENTION; var confirm = options["confirm"] as bool? ?? DEFAULT_CONFIRM; @@ -179,10 +193,10 @@ destination will be verified before being overwritten (if they seemingly match). System.Diagnostics.Debug.Assert(b2s != null); // Prepare the operations - var (to_copy, to_delete, to_verify) = PrepareFileLists(b1s, b2s, force); + var (to_copy, to_delete, to_verify) = PrepareFileLists(b1s, b2s); // Verify the files if requested. If the files are not verified, they will be deleted and copied again. - if (verify) + if (GlobalConfig.Verify) { // As this is a potentially slow operation, ask for confirmation of the verification) if (!confirm) @@ -227,17 +241,17 @@ destination will be verified before being overwritten (if they seemingly match). // Delete or rename the files that are not needed if (retention) { - var renamed = await RenameAsync(b2, to_delete, dry_run); + var renamed = await RenameAsync(b2, to_delete); Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Renamed {0} files in {1}", renamed, b2s.DisplayName); } else { - var deleted = await DeleteAsync(b2s, to_delete, dry_run); + var deleted = await DeleteAsync(b2s, to_delete); Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Deleted {0} files from {1}", deleted, b2s.DisplayName); } // Copy the files - var (copied, copy_errors) = await CopyAsync(b1s, b2s, to_copy, dry_run, verify); + var (copied, copy_errors) = await CopyAsync(b1s, b2s, to_copy); Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Copied {0} files from {1} to {2}", copied, b1s, b2s); // If there are still errors, retry a few times @@ -250,7 +264,7 @@ destination will be verified before being overwritten (if they seemingly match). for (int i = 0; i < retries; i++) { Thread.Sleep(5000); // Wait 5 seconds before retrying - (copied, copy_errors) = await CopyAsync(b1s, b2s, copy_errors, dry_run, verify); + (copied, copy_errors) = await CopyAsync(b1s, b2s, copy_errors); Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Copied {0} files from {1} to {2}", copied, b1s, b2s); if (!copy_errors.Any()) break; @@ -278,6 +292,8 @@ destination will be verified before being overwritten (if they seemingly match). // TODO Progress reporting // TODO Duplicati Results + // TODO Should the "get after put" verification be extracted out to be its own flag, as the "verify destination matches source" check is a desiredly separate operation? + /// /// Copies the files from one backend to another. /// The files are copied one by one, and each file is verified after uploading if the verify flag is set. @@ -288,7 +304,7 @@ destination will be verified before being overwritten (if they seemingly match). /// Flag for whether destructive actions (writes) should be printed rather than performed. Downloads (reads) are still being performed. /// Flag for whether to verify each upload to the destination backend. /// A tuple holding the number of succesful copies and a List of the files that failed. - private static async Task<(long, IEnumerable)> CopyAsync(IStreamingBackend b_src, IStreamingBackend b_dst, IEnumerable files, bool dry_run, bool verify) + private static async Task<(long, IEnumerable)> CopyAsync(IStreamingBackend b_src, IStreamingBackend b_dst, IEnumerable files) { long successful_copies = 0; List errors = []; @@ -299,14 +315,14 @@ destination will be verified before being overwritten (if they seemingly match). try { await b_src.GetAsync(f.Name, s_src, CancellationToken.None); - if (dry_run) + if (GlobalConfig.DryRun) { Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", "Would write {0} bytes of {1} to {2}", s_src.Length, f.Name, b_dst.DisplayName); } else { await b_dst.PutAsync(f.Name, s_src, CancellationToken.None); - if (verify) + if (GlobalConfig.Verify) { await b_dst.GetAsync(f.Name, s_dst, CancellationToken.None); @@ -350,14 +366,14 @@ destination will be verified before being overwritten (if they seemingly match). /// The files to delete. /// Flag for whether the deletion should be printed rather than performed. /// The number of successful deletions. - private static async Task DeleteAsync(IStreamingBackend b, IEnumerable files, bool dry_run) + private static async Task DeleteAsync(IStreamingBackend b, IEnumerable files) { long successful_deletes = 0; foreach (var f in files) { try { - if (dry_run) + if (GlobalConfig.DryRun) { Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", "Would delete {0} from {1}", f.Name, b.DisplayName); } @@ -400,13 +416,13 @@ destination will be verified before being overwritten (if they seemingly match). /// The destination backend. /// Flag for whether to force the synchronization. /// A tuple of Lists each holding the files to copy, delete and verify. - private static (IEnumerable, IEnumerable, IEnumerable) PrepareFileLists(IStreamingBackend b_src, IStreamingBackend b_dst, bool force) + private static (IEnumerable, IEnumerable, IEnumerable) PrepareFileLists(IStreamingBackend b_src, IStreamingBackend b_dst) { var files_src = b_src.List(); var files_dst = b_dst.List(); // Shortcut for force - if (force) + if (GlobalConfig.Force) { return (files_src, files_dst, []); } @@ -471,7 +487,7 @@ destination will be verified before being overwritten (if they seemingly match). /// The files to rename. /// Flag for whether the renaming should be printed rather than performed. /// The number of successful renames. - private static async Task RenameAsync(IBackend b, IEnumerable files, bool dry_run) + private static async Task RenameAsync(IBackend b, IEnumerable files) { long successful_renames = 0; string suffix = $"{System.DateTime.Now:yyyyMMddHHmmss}.old"; @@ -485,7 +501,7 @@ destination will be verified before being overwritten (if they seemingly match). try { await sb.GetAsync(f.Name, downloaded, CancellationToken.None); - if (dry_run) + if (GlobalConfig.DryRun) { Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", "Would rename {0} to {0}.{1} by deleting and re-uploading {2} bytes to {3}", f.Name, suffix, downloaded.Length, sb.DisplayName); } @@ -514,7 +530,7 @@ destination will be verified before being overwritten (if they seemingly match). { try { - if (dry_run) + if (GlobalConfig.DryRun) { Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", "Would rename {0} to {0}.{1} by calling Rename on {2}", f.Name, suffix, rb.DisplayName); } From 733e20d77c855bc553854be4ce1ac28130edd423 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 31 Jan 2025 14:19:53 +0100 Subject: [PATCH 071/362] Added the progress option --- Tools/RemoteSynchronization/Program.cs | 56 ++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 4 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 56672127d..5bc99017e 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2025, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a @@ -46,6 +46,7 @@ namespace RemoteSynchronization private const bool DEFAULT_CONFIRM = false; private const string DEFAULT_LOG_LEVEL = "Information"; private const string DEFAULT_LOG_FILE = ""; + private const bool DEFAULT_PROGRESS = false; /// /// Global configuration for the tool. Should be set after parsing the commandline arguments. @@ -56,6 +57,7 @@ namespace RemoteSynchronization internal static bool Force = DEFAULT_FORCE; internal static string LogLevel = DEFAULT_LOG_LEVEL; internal static string LogFile = DEFAULT_LOG_FILE; + internal static bool Progress = DEFAULT_PROGRESS; internal static bool Verify = DEFAULT_VERIFY; } @@ -84,6 +86,7 @@ namespace RemoteSynchronization var global_opts = OptionWithMultipleTokens(aliases: ["--global-options"], description: "Global options all backends. May be overridden by backend specific options (src-options, dst-options)"); var log_level_opt = new Option(aliases: ["--log-level"], description: "The log level to use. If not set here, global options will be checked", getDefaultValue: () => DEFAULT_LOG_LEVEL) { Arity = ArgumentArity.ExactlyOne }; var log_file_opt = new Option(aliases: ["--log-file"], description: "The log file to write to. If not set here, global options will be checked", getDefaultValue: () => DEFAULT_LOG_FILE) { Arity = ArgumentArity.ExactlyOne }; + var progress_opt = new Option(aliases: ["--progress"], description: "Print progress to STDOUT", getDefaultValue: () => DEFAULT_PROGRESS); var root_cmd = new RootCommand(@"Remote Synchronization Tool @@ -117,6 +120,7 @@ destination will be verified before being overwritten (if they seemingly match). root_cmd.AddOption(global_opts); root_cmd.AddOption(log_level_opt); root_cmd.AddOption(log_file_opt); + root_cmd.AddOption(progress_opt); root_cmd.SetHandler((InvocationContext ctx) => { @@ -287,11 +291,9 @@ destination will be verified before being overwritten (if they seemingly match). // TODO low memory mode, where things aren't kept in memory. Maybe utilize SQLite? // TODO For convenience, have the option to launch a "duplicati test" on the destination backend after the synchronization // TODO Save hash to minimize redownload - - // TODO Profiling logging - // TODO Progress reporting // TODO Duplicati Results + // TODO Profiling logging // TODO Should the "get after put" verification be extracted out to be its own flag, as the "verify destination matches source" check is a desiredly separate operation? /// @@ -310,8 +312,12 @@ destination will be verified before being overwritten (if they seemingly match). List errors = []; using var s_src = new MemoryStream(); using var s_dst = new MemoryStream(); + long i = 0, n = files.Count(); + foreach (var f in files) { + if (GlobalConfig.Progress) + Console.Write($"\nCopying: {i}/{n}"); try { await b_src.GetAsync(f.Name, s_src, CancellationToken.None); @@ -354,8 +360,13 @@ destination will be verified before being overwritten (if they seemingly match). { s_src.SetLength(0); s_dst.SetLength(0); + i++; } } + + if (GlobalConfig.Progress) + Console.WriteLine($"\rCopying: {n}/{n}"); + return (successful_copies, errors); } @@ -369,8 +380,14 @@ destination will be verified before being overwritten (if they seemingly match). private static async Task DeleteAsync(IStreamingBackend b, IEnumerable files) { long successful_deletes = 0; + long i = 0, n = files.Count(); + foreach (var f in files) { + if (n > 1) + { + Console.Write($"\nDeleting: {i}/{n}"); + } try { if (GlobalConfig.DryRun) @@ -387,7 +404,9 @@ destination will be verified before being overwritten (if they seemingly match). { Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e, "Error deleting {0}: {1}", f.Name, e.Message); } + i++; } + Console.WriteLine($"\rDeleting: {n}/{n}"); return successful_deletes; } @@ -492,12 +511,17 @@ destination will be verified before being overwritten (if they seemingly match). long successful_renames = 0; string suffix = $"{System.DateTime.Now:yyyyMMddHHmmss}.old"; using var downloaded = new MemoryStream(); + long i = 0, n = files.Count(); + switch (b) { case IStreamingBackend sb: { foreach (var f in files) { + if (GlobalConfig.Progress) + Console.Write($"\nRenaming: {i}/{n}"); + try { await sb.GetAsync(f.Name, downloaded, CancellationToken.None); @@ -521,13 +545,22 @@ destination will be verified before being overwritten (if they seemingly match). // Reset the stream downloaded.SetLength(0); } + + i++; } + + if (GlobalConfig.Progress) + Console.WriteLine($"\rRenaming: {n}/{n}"); + return successful_renames; } case IRenameEnabledBackend rb: { foreach (var f in files) { + if (GlobalConfig.Progress) + Console.Write($"\nRenaming: {i}/{n}"); + try { if (GlobalConfig.DryRun) @@ -544,7 +577,13 @@ destination will be verified before being overwritten (if they seemingly match). { Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e, "Error renaming {0}: {1}", f.Name, e.Message); } + + i++; } + + if (GlobalConfig.Progress) + Console.WriteLine($"\rRenaming: {n}/{n}"); + return successful_renames; } default: @@ -565,9 +604,13 @@ destination will be verified before being overwritten (if they seemingly match). var errors = new List(); using var s_src = new MemoryStream(); using var s_dst = new MemoryStream(); + long i = 0, n = files.Count(); foreach (var f in files) { + if (GlobalConfig.Progress) + Console.Write($"\nVerifying: {i}/{n}"); + try { // Get both files @@ -592,8 +635,13 @@ destination will be verified before being overwritten (if they seemingly match). s_src.SetLength(0); s_dst.SetLength(0); } + + i++; } + if (GlobalConfig.Progress) + Console.WriteLine($"\rVerifying: {n}/{n}"); + return errors; } From 94a19f9abc1ed89a373f2397d9b616ac31ee43fe Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 31 Jan 2025 14:32:54 +0100 Subject: [PATCH 072/362] Added logging a status report towards the end of remote synchronization --- Tools/RemoteSynchronization/Program.cs | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 5bc99017e..db939182c 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -200,6 +200,7 @@ destination will be verified before being overwritten (if they seemingly match). var (to_copy, to_delete, to_verify) = PrepareFileLists(b1s, b2s); // Verify the files if requested. If the files are not verified, they will be deleted and copied again. + long verified = 0, failed_verify = 0; if (GlobalConfig.Verify) { // As this is a potentially slow operation, ask for confirmation of the verification) @@ -215,6 +216,8 @@ destination will be verified before being overwritten (if they seemingly match). } var not_verified = await VerifyAsync(b1s, b2s, to_verify); + failed_verify = not_verified.Count(); + verified = to_verify.Count() - failed_verify; if (not_verified.Any()) { @@ -243,14 +246,15 @@ destination will be verified before being overwritten (if they seemingly match). } // Delete or rename the files that are not needed + long renamed = 0, deleted = 0; if (retention) { - var renamed = await RenameAsync(b2, to_delete); + renamed = await RenameAsync(b2, to_delete); Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Renamed {0} files in {1}", renamed, b2s.DisplayName); } else { - var deleted = await DeleteAsync(b2s, to_delete); + deleted = await DeleteAsync(b2s, to_delete); Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Deleted {0} files from {1}", deleted, b2s.DisplayName); } @@ -282,6 +286,18 @@ destination will be verified before being overwritten (if they seemingly match). } } + // Results reporting + if (verified > 0) + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Verified {0} files in {1} that didn't need to be copied", verified, b2s.DisplayName); + if (failed_verify > 0) + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Failed to verify {0} files in {1}, which were then attempted to be copied", failed_verify, b2s.DisplayName); + if (copied > 0) + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Copied {0} files from {1} to {2}", copied, b1s.DisplayName, b2s.DisplayName); + if (deleted > 0) + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Deleted {0} files from {1}", deleted, b2s.DisplayName); + if (renamed > 0) + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Renamed {0} files in {1}", renamed, b2s.DisplayName); + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Remote synchronization completed successfully"); return 0; From 3b94cd8039ac34e428e579327274934489826ea7 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 31 Jan 2025 14:39:21 +0100 Subject: [PATCH 073/362] Split the verify option into two separate options --- Tools/RemoteSynchronization/Program.cs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index db939182c..30cb717f2 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -39,7 +39,8 @@ namespace RemoteSynchronization { // Default values for the options private const bool DEFAULT_DRY_RUN = false; - private const bool DEFAULT_VERIFY = false; + private const bool DEFAULT_VERIFY_CONTENTS = false; + private const bool DEFAULT_VERIFY_GET_AFTER_PUT = false; private const int DEFAULT_RETRY = 3; private const bool DEFAULT_FORCE = false; private const bool DEFAULT_RETENTION = false; @@ -58,7 +59,8 @@ namespace RemoteSynchronization internal static string LogLevel = DEFAULT_LOG_LEVEL; internal static string LogFile = DEFAULT_LOG_FILE; internal static bool Progress = DEFAULT_PROGRESS; - internal static bool Verify = DEFAULT_VERIFY; + internal static bool VerifyContents = DEFAULT_VERIFY_CONTENTS; + internal static bool VerifyGetAfterPut = DEFAULT_VERIFY_GET_AFTER_PUT; } /// @@ -78,7 +80,8 @@ namespace RemoteSynchronization var dry_run_opt = new Option(aliases: ["--dry-run", "-d"], description: "Do not actually write or delete files. If not set here, the global options will be checked", getDefaultValue: () => DEFAULT_DRY_RUN); var src_opts = OptionWithMultipleTokens(aliases: ["--src-options"], description: "Options for the source backend"); var dst_opts = OptionWithMultipleTokens(aliases: ["--dst-options"], description: "Options for the destination backend"); - var verify_opt = new Option(aliases: ["--verify"], description: "Verify the files after copying", getDefaultValue: () => DEFAULT_VERIFY); + var verify_contents_opt = new Option(aliases: ["--verify-contents"], description: "Verify the contents of the files to decide whether the pre-existing destination files should be overwritten", getDefaultValue: () => DEFAULT_VERIFY_CONTENTS); + var verify_get_after_put_opt = new Option(aliases: ["--verify-get-after-put"], description: "Verify the files after uploading them to ensure that they were uploaded correctly", getDefaultValue: () => DEFAULT_VERIFY_GET_AFTER_PUT); var retry_opt = new Option(aliases: ["--retry"], description: "Number of times to retry on errors", getDefaultValue: () => DEFAULT_RETRY) { Arity = ArgumentArity.ExactlyOne }; var force_opt = new Option(aliases: ["--force"], description: "Force the synchronization", getDefaultValue: () => DEFAULT_FORCE); var retention_opt = new Option(aliases: ["--retention"], description: "Toggles whether to keep old files. Any deletes will be renames instead", getDefaultValue: () => DEFAULT_RETENTION); @@ -112,7 +115,8 @@ destination will be verified before being overwritten (if they seemingly match). root_cmd.AddOption(dry_run_opt); root_cmd.AddOption(src_opts); root_cmd.AddOption(dst_opts); - root_cmd.AddOption(verify_opt); + root_cmd.AddOption(verify_contents_opt); + root_cmd.AddOption(verify_get_after_put_opt); root_cmd.AddOption(retry_opt); root_cmd.AddOption(force_opt); root_cmd.AddOption(retention_opt); @@ -148,7 +152,8 @@ destination will be verified before being overwritten (if they seemingly match). GlobalConfig.DryRun = options["dry-run"] as bool? ?? DEFAULT_DRY_RUN; GlobalConfig.LogLevel = options["log-level"] as string ?? DEFAULT_LOG_LEVEL; GlobalConfig.LogFile = options["log-file"] as string ?? DEFAULT_LOG_FILE; - GlobalConfig.Verify = options["verify"] as bool? ?? DEFAULT_VERIFY; + GlobalConfig.VerifyContents = options["verify"] as bool? ?? DEFAULT_VERIFY_CONTENTS; + GlobalConfig.VerifyGetAfterPut = options["verify-get-after-put"] as bool? ?? DEFAULT_VERIFY_GET_AFTER_PUT; var retries = options["retry"] as int? ?? DEFAULT_RETRY; var retention = options["retention"] as bool? ?? DEFAULT_RETENTION; var confirm = options["confirm"] as bool? ?? DEFAULT_CONFIRM; @@ -201,7 +206,7 @@ destination will be verified before being overwritten (if they seemingly match). // Verify the files if requested. If the files are not verified, they will be deleted and copied again. long verified = 0, failed_verify = 0; - if (GlobalConfig.Verify) + if (GlobalConfig.VerifyContents) { // As this is a potentially slow operation, ask for confirmation of the verification) if (!confirm) @@ -310,7 +315,6 @@ destination will be verified before being overwritten (if they seemingly match). // TODO Duplicati Results // TODO Profiling logging - // TODO Should the "get after put" verification be extracted out to be its own flag, as the "verify destination matches source" check is a desiredly separate operation? /// /// Copies the files from one backend to another. @@ -344,7 +348,7 @@ destination will be verified before being overwritten (if they seemingly match). else { await b_dst.PutAsync(f.Name, s_src, CancellationToken.None); - if (GlobalConfig.Verify) + if (GlobalConfig.VerifyGetAfterPut) { await b_dst.GetAsync(f.Name, s_dst, CancellationToken.None); From f2cd4f5641db4b336d09e6d9905a10baa2d7f8e5 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 31 Jan 2025 14:42:03 +0100 Subject: [PATCH 074/362] Added docstrings to the global configuration class --- Tools/RemoteSynchronization/Program.cs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 30cb717f2..1d3dafb15 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -54,12 +54,33 @@ namespace RemoteSynchronization /// private class GlobalConfig { + /// + /// Whether the tool should not actually write or delete files. Defaults to false. + /// internal static bool DryRun = DEFAULT_DRY_RUN; + /// + /// Whether the tool should force the synchronization. Defaults to false. + /// internal static bool Force = DEFAULT_FORCE; + /// + /// The log level to use. Defaults to "Information". + /// internal static string LogLevel = DEFAULT_LOG_LEVEL; + /// + /// The log file to write to. Defaults to an empty string. + /// internal static string LogFile = DEFAULT_LOG_FILE; + /// + /// Whether the tool should print progress to STDOUT. Defaults to false. + /// internal static bool Progress = DEFAULT_PROGRESS; + /// + /// Whether the tool should verify the contents of the files to decide whether the pre-existing destination files should be overwritten. Defaults to false. + /// internal static bool VerifyContents = DEFAULT_VERIFY_CONTENTS; + /// + /// Whether the tool should verify the files after uploading them to ensure that they were uploaded correctly. Defaults to false. + /// internal static bool VerifyGetAfterPut = DEFAULT_VERIFY_GET_AFTER_PUT; } From 3c39744f0e8100e71d87382376fc8994279d71a6 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 31 Jan 2025 15:12:25 +0100 Subject: [PATCH 075/362] Bump ngclient --- Duplicati/Server/webroot/ngclient/package-lock.json | 8 ++++---- Duplicati/Server/webroot/ngclient/package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Duplicati/Server/webroot/ngclient/package-lock.json b/Duplicati/Server/webroot/ngclient/package-lock.json index 5e823b2ba..01c099cab 100644 --- a/Duplicati/Server/webroot/ngclient/package-lock.json +++ b/Duplicati/Server/webroot/ngclient/package-lock.json @@ -9,13 +9,13 @@ "version": "1.0.0", "license": "MIT", "dependencies": { - "@duplicati/ngclient": "^0.0.57" + "@duplicati/ngclient": "^0.0.73" } }, "node_modules/@duplicati/ngclient": { - "version": "0.0.57", - "resolved": "https://registry.npmjs.org/@duplicati/ngclient/-/ngclient-0.0.57.tgz", - "integrity": "sha512-w38h07j1KJkqwD2VsLE8i+EQIpsjbw5uNHH+Utho/wzY0tcuRYgN44GJ6O0UEEx9rFRK5ecz+T40wpZQGIEOkw==", + "version": "0.0.73", + "resolved": "https://registry.npmjs.org/@duplicati/ngclient/-/ngclient-0.0.73.tgz", + "integrity": "sha512-ucVBNCkMW/Zc4k8ypbYc4t0y75BFTiYo9pybyaiF8UpFCJtswBqBsyr21rR6S/W9Z/qI16WHnvzvEz359KfxNQ==", "license": "MIT" } } diff --git a/Duplicati/Server/webroot/ngclient/package.json b/Duplicati/Server/webroot/ngclient/package.json index 9f5165ab8..1fdc16e31 100644 --- a/Duplicati/Server/webroot/ngclient/package.json +++ b/Duplicati/Server/webroot/ngclient/package.json @@ -4,6 +4,6 @@ "description": "Modern Angular Client for Duplicati", "license": "MIT", "dependencies": { - "@duplicati/ngclient": "^0.0.57" + "@duplicati/ngclient": "^0.0.73" } } From 77b728820853a44ac7c2d9d43ca76fc12d2001a3 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 31 Jan 2025 16:16:55 +0100 Subject: [PATCH 076/362] Version bump to v2.1.0.108-2.1.0.108_canary_2025-01-31 You can download this build from: Binaries: https://updates.duplicati.com/canary/?version=2.1.0.108 Signature file: https://updates.duplicati.com/canary/duplicati-2.1.0.108_canary_2025-01-31.signatures.zip --- ReleaseBuilder/build_version.txt | 2 +- changelog.txt | 41 ++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/ReleaseBuilder/build_version.txt b/ReleaseBuilder/build_version.txt index 202bdd1a8..81633e73f 100644 --- a/ReleaseBuilder/build_version.txt +++ b/ReleaseBuilder/build_version.txt @@ -1 +1 @@ -2.1.0.107 \ No newline at end of file +2.1.0.108 \ No newline at end of file diff --git a/changelog.txt b/changelog.txt index 752a6c20e..a1cedc446 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,44 @@ +2025-01-31 - 2.1.0.108_canary_2025-01-31 +========== +This release is a canary release intended to be used for testing. + +# Important change +This build changes the folder locations for the `Duplicati-server.sqlite` file on Windows and Linux when running as a service. +There is support for backwards compatibility, so existing installations should not be affected. + +The detailed changes are: +Windows: Avoid storing data in `C:\Windows\System32\config\systemprofile\AppData\Local\Duplicati` and prefer `{SystemRoot}\Users\LocalService\Duplicati` +Linux: Avoid storing data in `/Duplicati` and prefer `/var/lib/Duplicati`. + +If you are not running Duplicati as a service, this has no effect. +If you are using `--server-datafolder` or `DUPLICATI_HOME`, this has no effect on the database, but may cause your machineid and installid to change. + +The `machineid.txt` and `installid.txt` would previously be stored in the local app data folder, even when using portable mode or choosing a specific data folder. +This has been fixed, so the files will now follow the database. +If you are using the Duplicati console or otherwise depend on these values, you need to move them into the folder where the database is stored. + +# New UI +This release contains an updated version of the new UI. +In the url, change `ngax` to `ngclient` to try the new UI. + +## Detailed list of changes: +- Fixed encoding of exported files +- Fixed issue with restore trying to read files opened with write-only +- Improved backend tester +- Missing file detection during recreate is now a warning +- Fixed Minio backend not throwing exceptions +- Fixed hidden files display in source picker +- Added additional source folders for MacOS +- Removed SHA1 signatures on Windows signing +- Rewrote the backend manager to better control backend (re-)use +- Updated reporting to support sending OS type and backend type +- Tagging Docker images with channel, thanks @stavros-k +- Added support for CORS headers in the Webserver +- Unified help arguments across all executables +- Fixed parsing flags as multiple values for FTP SSL type +- Added guard against storing files in Windows folder +- Fixed crash in AutoUpdater + 2025-01-17 - 2.1.0.107_canary_2025-01-17 ========== This release is a canary release intended to be used for testing. From 23d304c7360aa0346a9c635b0f74d89b7b5fb00e Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sat, 1 Feb 2025 13:59:29 +0100 Subject: [PATCH 077/362] Fixed not creating releases on master when building from a branch --- ReleaseBuilder/Build/Command.Upload.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ReleaseBuilder/Build/Command.Upload.cs b/ReleaseBuilder/Build/Command.Upload.cs index 6f07cbd5c..d75cc2af8 100644 --- a/ReleaseBuilder/Build/Command.Upload.cs +++ b/ReleaseBuilder/Build/Command.Upload.cs @@ -151,6 +151,9 @@ public static partial class Command /// An awaitable task public static async Task UploadToGithub(IEnumerable files, RuntimeConfig rtcfg) { + var commithash = (await ProcessHelper.ExecuteWithOutput(["git", "rev-parse", "HEAD"]))?.Trim() + ?? throw new Exception("Failed to get the current commit hash"); + using var httpClient = new HttpClient(); var totalSize = files.Sum(f => new FileInfo(f.Path).Length); Console.WriteLine($"Uploading {files.Count()} files ({Duplicati.Library.Utility.Utility.FormatSizeString(totalSize)}) to Github..."); @@ -166,7 +169,7 @@ public static partial class Command request.Headers.Add("User-Agent", "Duplicati Release Builder v1"); request.Content = JsonContent.Create(new GhReleaseInfo( tag_name: $"v{rtcfg.ReleaseInfo.ReleaseName}", - target_commitish: "master", + target_commitish: commithash, name: $"v{rtcfg.ReleaseInfo.ReleaseName}", body: rtcfg.ChangelogNews, draft: false, From 686ab0338b35330919609ada8c6fcdd0cdb9dfc8 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 3 Feb 2025 08:38:18 +0100 Subject: [PATCH 078/362] Replaced a \n that should have been a \r --- Tools/RemoteSynchronization/Program.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 1d3dafb15..25c43dc11 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2025, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a @@ -358,7 +358,7 @@ destination will be verified before being overwritten (if they seemingly match). foreach (var f in files) { if (GlobalConfig.Progress) - Console.Write($"\nCopying: {i}/{n}"); + Console.Write($"\rCopying: {i}/{n}"); try { await b_src.GetAsync(f.Name, s_src, CancellationToken.None); @@ -427,7 +427,7 @@ destination will be verified before being overwritten (if they seemingly match). { if (n > 1) { - Console.Write($"\nDeleting: {i}/{n}"); + Console.Write($"\rDeleting: {i}/{n}"); } try { @@ -561,7 +561,7 @@ destination will be verified before being overwritten (if they seemingly match). foreach (var f in files) { if (GlobalConfig.Progress) - Console.Write($"\nRenaming: {i}/{n}"); + Console.Write($"\rRenaming: {i}/{n}"); try { @@ -600,7 +600,7 @@ destination will be verified before being overwritten (if they seemingly match). foreach (var f in files) { if (GlobalConfig.Progress) - Console.Write($"\nRenaming: {i}/{n}"); + Console.Write($"\rRenaming: {i}/{n}"); try { @@ -650,7 +650,7 @@ destination will be verified before being overwritten (if they seemingly match). foreach (var f in files) { if (GlobalConfig.Progress) - Console.Write($"\nVerifying: {i}/{n}"); + Console.Write($"\rVerifying: {i}/{n}"); try { From 7c7f58bdaf2109ef2ff0b48c07d12168e73c7f75 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 3 Feb 2025 08:38:37 +0100 Subject: [PATCH 079/362] Added profiling logging --- Tools/RemoteSynchronization/Program.cs | 89 +++++++++++++++++++++++++- 1 file changed, 86 insertions(+), 3 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 25c43dc11..ea11d62bc 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -335,8 +335,6 @@ destination will be verified before being overwritten (if they seemingly match). // TODO Save hash to minimize redownload // TODO Duplicati Results - // TODO Profiling logging - /// /// Copies the files from one backend to another. /// The files are copied one by one, and each file is verified after uploading if the verify flag is set. @@ -354,6 +352,10 @@ destination will be verified before being overwritten (if they seemingly match). using var s_src = new MemoryStream(); using var s_dst = new MemoryStream(); long i = 0, n = files.Count(); + var sw_get_src = new System.Diagnostics.Stopwatch(); + var sw_put_dst = new System.Diagnostics.Stopwatch(); + var sw_get_dst = new System.Diagnostics.Stopwatch(); + var sw_get_cmp = new System.Diagnostics.Stopwatch(); foreach (var f in files) { @@ -361,18 +363,25 @@ destination will be verified before being overwritten (if they seemingly match). Console.Write($"\rCopying: {i}/{n}"); try { + sw_get_src.Start(); await b_src.GetAsync(f.Name, s_src, CancellationToken.None); + sw_get_src.Stop(); if (GlobalConfig.DryRun) { Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", "Would write {0} bytes of {1} to {2}", s_src.Length, f.Name, b_dst.DisplayName); } else { + sw_put_dst.Start(); await b_dst.PutAsync(f.Name, s_src, CancellationToken.None); + sw_put_dst.Stop(); if (GlobalConfig.VerifyGetAfterPut) { + sw_get_dst.Start(); await b_dst.GetAsync(f.Name, s_dst, CancellationToken.None); + sw_get_dst.Stop(); + sw_get_cmp.Start(); string? err_string = null; if (s_src.Length != s_dst.Length) { @@ -383,6 +392,7 @@ destination will be verified before being overwritten (if they seemingly match). { err_string = (err_string is null ? "" : err_string + " ") + "The contents of the files do not match."; } + sw_get_cmp.Stop(); if (err_string is not null) { @@ -402,12 +412,20 @@ destination will be verified before being overwritten (if they seemingly match). s_src.SetLength(0); s_dst.SetLength(0); i++; + + // Stop any running timers + sw_get_src.Stop(); + sw_put_dst.Stop(); + sw_get_dst.Stop(); + sw_get_cmp.Stop(); } } if (GlobalConfig.Progress) Console.WriteLine($"\rCopying: {n}/{n}"); + Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", "Copy | Get source: {0} ms, Put destination: {1} ms, Get destination: {2} ms, Get compare: {3} ms", sw_get_src.ElapsedMilliseconds, sw_put_dst.ElapsedMilliseconds, sw_get_dst.ElapsedMilliseconds, sw_get_cmp.ElapsedMilliseconds); + return (successful_copies, errors); } @@ -422,6 +440,8 @@ destination will be verified before being overwritten (if they seemingly match). { long successful_deletes = 0; long i = 0, n = files.Count(); + var sw = new System.Diagnostics.Stopwatch(); + sw.Restart(); foreach (var f in files) { @@ -447,6 +467,10 @@ destination will be verified before being overwritten (if they seemingly match). } i++; } + + sw.Stop(); + Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", "Delete: {0} ms", sw.ElapsedMilliseconds); + Console.WriteLine($"\rDeleting: {n}/{n}"); return successful_deletes; } @@ -478,8 +502,16 @@ destination will be verified before being overwritten (if they seemingly match). /// A tuple of Lists each holding the files to copy, delete and verify. private static (IEnumerable, IEnumerable, IEnumerable) PrepareFileLists(IStreamingBackend b_src, IStreamingBackend b_dst) { + var sw = new System.Diagnostics.Stopwatch(); + + sw.Restart(); var files_src = b_src.List(); + sw.Stop(); + Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", "Prepare | List source: {0} ms", sw.ElapsedMilliseconds); + sw.Restart(); var files_dst = b_dst.List(); + sw.Stop(); + Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", "Prepare | List destination: {0} ms", sw.ElapsedMilliseconds); // Shortcut for force if (GlobalConfig.Force) @@ -493,14 +525,18 @@ destination will be verified before being overwritten (if they seemingly match). return (files_src, [], []); } + sw.Restart(); var lookup_src = files_src.ToDictionary(x => x.Name); var lookup_dst = files_dst.ToDictionary(x => x.Name); + sw.Stop(); + Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", "Prepare | Build lookup for source and destination: {0} ms", sw.ElapsedMilliseconds); var to_copy = new List(); var to_delete = new HashSet(); var to_verify = new List(); // Find all of the files in src that are not in dst, have a different size or have a more recent modification date + sw.Restart(); foreach (var f_src in files_src) { if (lookup_dst.TryGetValue(f_src.Name, out var f_dst)) @@ -523,8 +559,11 @@ destination will be verified before being overwritten (if they seemingly match). to_copy.Add(f_src); } } + sw.Stop(); + Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", "Prepare | Check the files that are present in source against destination: {0} ms", sw.ElapsedMilliseconds); // Find all of the files in dst that are not in src + sw.Start(); foreach (var f_dst in files_dst) { if (to_delete.Contains(f_dst.Name)) @@ -535,8 +574,15 @@ destination will be verified before being overwritten (if they seemingly match). to_delete.Add(f_dst.Name); } } + sw.Stop(); + Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", "Prepare | Check the files that are present in destination against source: {0} ms", sw.ElapsedMilliseconds); - return (to_copy, to_delete.Select(x => lookup_dst[x]), to_verify); + sw.Restart(); + var to_delete_lookedup = to_delete.Select(x => lookup_dst[x]).ToList(); + sw.Stop(); + Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", "Prepare | Lookup the files to delete: {0} ms", sw.ElapsedMilliseconds); + + return (to_copy, to_delete_lookedup, to_verify); } /// @@ -558,6 +604,10 @@ destination will be verified before being overwritten (if they seemingly match). { case IStreamingBackend sb: { + var sw_get_src = new System.Diagnostics.Stopwatch(); + var sw_put_dst = new System.Diagnostics.Stopwatch(); + var sw_del_src = new System.Diagnostics.Stopwatch(); + foreach (var f in files) { if (GlobalConfig.Progress) @@ -565,15 +615,22 @@ destination will be verified before being overwritten (if they seemingly match). try { + sw_get_src.Start(); await sb.GetAsync(f.Name, downloaded, CancellationToken.None); + sw_get_src.Stop(); + if (GlobalConfig.DryRun) { Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", "Would rename {0} to {0}.{1} by deleting and re-uploading {2} bytes to {3}", f.Name, suffix, downloaded.Length, sb.DisplayName); } else { + sw_put_dst.Start(); await sb.PutAsync($"{f.Name}.{suffix}", downloaded, CancellationToken.None); + sw_put_dst.Stop(); + sw_del_src.Start(); await sb.DeleteAsync(f.Name, CancellationToken.None); + sw_del_src.Stop(); } successful_renames++; } @@ -585,11 +642,18 @@ destination will be verified before being overwritten (if they seemingly match). { // Reset the stream downloaded.SetLength(0); + + // Stop any running timers + sw_get_src.Stop(); + sw_put_dst.Stop(); + sw_del_src.Stop(); } i++; } + Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", "Rename | Get source: {0} ms, Put destination: {1} ms, Delete source: {2} ms", sw_get_src.ElapsedMilliseconds, sw_put_dst.ElapsedMilliseconds, sw_del_src.ElapsedMilliseconds); + if (GlobalConfig.Progress) Console.WriteLine($"\rRenaming: {n}/{n}"); @@ -597,6 +661,8 @@ destination will be verified before being overwritten (if they seemingly match). } case IRenameEnabledBackend rb: { + var sw = new System.Diagnostics.Stopwatch(); + foreach (var f in files) { if (GlobalConfig.Progress) @@ -610,18 +676,23 @@ destination will be verified before being overwritten (if they seemingly match). } else { + sw.Start(); await rb.RenameAsync(f.Name, $"{f.Name}.{suffix}", CancellationToken.None); + sw.Stop(); } successful_renames++; } catch (Exception e) { Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e, "Error renaming {0}: {1}", f.Name, e.Message); + sw.Stop(); } i++; } + Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", "Rename: {0} ms", sw.ElapsedMilliseconds); + if (GlobalConfig.Progress) Console.WriteLine($"\rRenaming: {n}/{n}"); @@ -646,6 +717,8 @@ destination will be verified before being overwritten (if they seemingly match). using var s_src = new MemoryStream(); using var s_dst = new MemoryStream(); long i = 0, n = files.Count(); + var sw_get = new System.Diagnostics.Stopwatch(); + var sw_cmp = new System.Diagnostics.Stopwatch(); foreach (var f in files) { @@ -655,15 +728,19 @@ destination will be verified before being overwritten (if they seemingly match). try { // Get both files + sw_get.Start(); var fs = b_src.GetAsync(f.Name, s_src, CancellationToken.None); var ds = b_dst.GetAsync(f.Name, s_dst, CancellationToken.None); await Task.WhenAll(fs, ds); + sw_get.Stop(); // Compare the contents + sw_cmp.Start(); if (s_src.Length != s_dst.Length || !s_src.ToArray().SequenceEqual(s_dst.ToArray())) { errors.Add(f); } + sw_cmp.Stop(); } catch (Exception e) { @@ -675,11 +752,17 @@ destination will be verified before being overwritten (if they seemingly match). // Reset the streams s_src.SetLength(0); s_dst.SetLength(0); + + // Stop any running timers + sw_get.Stop(); + sw_cmp.Stop(); } i++; } + Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", "Verify | Get: {0} ms, Compare: {1} ms", sw_get.ElapsedMilliseconds, sw_cmp.ElapsedMilliseconds); + if (GlobalConfig.Progress) Console.WriteLine($"\rVerifying: {n}/{n}"); From f88c2971a34120eda2c055d9d1e3a4026ea98a02 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 3 Feb 2025 09:06:30 +0100 Subject: [PATCH 080/362] Added a test for parsing all of the commandline options --- Duplicati/UnitTest/ToolTests.cs | 48 ++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/Duplicati/UnitTest/ToolTests.cs b/Duplicati/UnitTest/ToolTests.cs index 66a9d09ba..cfd7aa2d1 100644 --- a/Duplicati/UnitTest/ToolTests.cs +++ b/Duplicati/UnitTest/ToolTests.cs @@ -37,7 +37,49 @@ namespace Duplicati.UnitTest { /// - /// Tests the remote synchronization tool. + /// Tests passing all arguments to the main method of the remote synchronization tool. + /// + [Test] + public void TestMainMethodParsesArgumentsCorrectly() + { + string[][] testCases = + { + ["source", "destination", "--parse-arguments-only"], + ["source", "destination", "--parse-arguments-only", "--dry-run"], + ["source", "destination", "--parse-arguments-only", "--force"], + ["source", "destination", "--parse-arguments-only", "--dry-run", "--force"], + ["source", "destination", "--parse-arguments-only", "--verify-contents"], + ["source", "destination", "--parse-arguments-only", "--verify-get-after-put"], + ["source", "destination", "--parse-arguments-only", "--retry", "3"], + ["source", "destination", "--parse-arguments-only", "--log-level", "Debug"], + ["source", "destination", "--parse-arguments-only", "--log-file", "somefile.log"], + ["source", "destination", "--parse-arguments-only", "--progress"], + ["source", "destination", "--parse-arguments-only", "--retention"], + ["source", "destination", "--parse-arguments-only", "--confirm"], + ["source", "destination", "--parse-arguments-only", "--global-options", "someglobalkey=someglobalvalue", "anotherglobalkey=anotherglobalvalue"], + ["source", "destination", "--parse-arguments-only", "--src-options", "somesrckey=somesrcvalue", "anothersrckey=anothersrcvalue"], + ["source", "destination", "--parse-arguments-only", "--dst-options", "somedstkey=somedstvalue", "anotherdstkey=anotherdstvalue"], + [ + "source", "destination", "--parse-arguments-only", + "--dry-run", "--force", "--verify-contents", "--retry", "3", "--log-level", "Debug", "--log-file", "somefile.log", "--progress", "--retention", "--confirm", + "--global-options", "someglobalkey=someglobalvalue", "anotherglobalkey=anotherglobalvalue", + "--src-options", "somesrckey=somesrcvalue", "anothersrckey=anothersrcvalue", + "--dst-options", "somedstkey=somedstvalue", "anotherdstkey=anotherdstvalue" + ], + }; + + foreach (var args in testCases) + { + int result = RemoteSynchronization.Program.Main(args).ConfigureAwait(false).GetAwaiter().GetResult(); + Assert.AreEqual(0, result, $"Failed for args: {string.Join(" ", args)}"); + } + + int failed_result = RemoteSynchronization.Program.Main(["source", "destination", "--bogus-option"]).ConfigureAwait(false).GetAwaiter().GetResult(); + Assert.AreEqual(1, failed_result, "Invalid option did not return 1"); + } + + /// + /// Tests the original inded use of the remote synchronization tool on an empty destination. /// [Test] [Category("Tools")] @@ -115,6 +157,10 @@ namespace Duplicati.UnitTest Assert.IsTrue(DirectoriesAndContentsAreEqual(DATAFOLDER, l2r), "Restored second level files is not equal to original files"); } + // + // Helper methods + // + /// /// Compares two directories and their contents. /// From 085648369939ba1d7d839a53005606ae50dab096 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 3 Feb 2025 09:06:44 +0100 Subject: [PATCH 081/362] Added an option to only parse the commandline options --- Tools/RemoteSynchronization/Program.cs | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index ea11d62bc..f6158b1f6 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2025, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a @@ -48,6 +48,7 @@ namespace RemoteSynchronization private const string DEFAULT_LOG_LEVEL = "Information"; private const string DEFAULT_LOG_FILE = ""; private const bool DEFAULT_PROGRESS = false; + private const bool DEFAULT_PARSE_ARGUMENTS_ONLY = false; /// /// Global configuration for the tool. Should be set after parsing the commandline arguments. @@ -82,6 +83,10 @@ namespace RemoteSynchronization /// Whether the tool should verify the files after uploading them to ensure that they were uploaded correctly. Defaults to false. /// internal static bool VerifyGetAfterPut = DEFAULT_VERIFY_GET_AFTER_PUT; + /// + /// Try to only parse the arguments and then exit. Defaults to false. + /// + internal static bool ParseArgumentsOnly = DEFAULT_PARSE_ARGUMENTS_ONLY; } /// @@ -111,6 +116,7 @@ namespace RemoteSynchronization var log_level_opt = new Option(aliases: ["--log-level"], description: "The log level to use. If not set here, global options will be checked", getDefaultValue: () => DEFAULT_LOG_LEVEL) { Arity = ArgumentArity.ExactlyOne }; var log_file_opt = new Option(aliases: ["--log-file"], description: "The log file to write to. If not set here, global options will be checked", getDefaultValue: () => DEFAULT_LOG_FILE) { Arity = ArgumentArity.ExactlyOne }; var progress_opt = new Option(aliases: ["--progress"], description: "Print progress to STDOUT", getDefaultValue: () => DEFAULT_PROGRESS); + var parse_opt = new Option(aliases: ["--parse-arguments-only"], description: "Only parse the arguments and then exit", getDefaultValue: () => DEFAULT_PARSE_ARGUMENTS_ONLY); var root_cmd = new RootCommand(@"Remote Synchronization Tool @@ -146,6 +152,7 @@ destination will be verified before being overwritten (if they seemingly match). root_cmd.AddOption(log_level_opt); root_cmd.AddOption(log_file_opt); root_cmd.AddOption(progress_opt); + root_cmd.AddOption(parse_opt); root_cmd.SetHandler((InvocationContext ctx) => { @@ -173,11 +180,12 @@ destination will be verified before being overwritten (if they seemingly match). GlobalConfig.DryRun = options["dry-run"] as bool? ?? DEFAULT_DRY_RUN; GlobalConfig.LogLevel = options["log-level"] as string ?? DEFAULT_LOG_LEVEL; GlobalConfig.LogFile = options["log-file"] as string ?? DEFAULT_LOG_FILE; - GlobalConfig.VerifyContents = options["verify"] as bool? ?? DEFAULT_VERIFY_CONTENTS; + GlobalConfig.VerifyContents = options["verify-contents"] as bool? ?? DEFAULT_VERIFY_CONTENTS; GlobalConfig.VerifyGetAfterPut = options["verify-get-after-put"] as bool? ?? DEFAULT_VERIFY_GET_AFTER_PUT; var retries = options["retry"] as int? ?? DEFAULT_RETRY; var retention = options["retention"] as bool? ?? DEFAULT_RETENTION; var confirm = options["confirm"] as bool? ?? DEFAULT_CONFIRM; + var parse_only = options["parse-arguments-only"] as bool? ?? DEFAULT_PARSE_ARGUMENTS_ONLY; // Unpack and parse the multi token options Dictionary global_options = (options["global-options"] as List) @@ -192,7 +200,7 @@ destination will be verified before being overwritten (if they seemingly match). // Parse the log file var log_file = options["log-file"] as string ?? DEFAULT_LOG_FILE; - var file_sink = (Duplicati.Library.Main.IMessageSink?)(log_file != "" ? new Duplicati.Library.Logging.StreamLogDestination(log_file) : null); + var file_sink = (Duplicati.Library.Main.IMessageSink?)(string.IsNullOrEmpty(log_file) ? new Duplicati.Library.Logging.StreamLogDestination(log_file) : null); var console_sink = new Duplicati.CommandLine.ConsoleOutput(Console.Out, global_options); Duplicati.Library.Main.IMessageSink[] sinks = file_sink is null ? [console_sink] : [file_sink, console_sink]; var multi_sink = new Duplicati.Library.Main.MultiMessageSink(sinks); @@ -213,6 +221,13 @@ destination will be verified before being overwritten (if they seemingly match). foreach (var x in global_options) src_opts[x.Key] = dst_opts[x.Key] = x.Value; + // Check if we only had to parse the arguments + if (parse_only) + { + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Arguments parsed successfully; exiting."); + return 0; + } + // Load the backends using var b1 = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(src, src_opts); var b1s = b1 as IStreamingBackend; From 3cedaead7886a7480545603fd80f3bc425c1263b Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 3 Feb 2025 10:06:02 +0100 Subject: [PATCH 082/362] Added additional help text to the multi token arguments --- Tools/RemoteSynchronization/Program.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index f6158b1f6..f30baac27 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2025, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a @@ -104,15 +104,15 @@ namespace RemoteSynchronization var src_arg = new Argument(name: "backend_src", description: "The source backend string"); var dst_arg = new Argument(name: "backend_dst", description: "The destination backend string"); var dry_run_opt = new Option(aliases: ["--dry-run", "-d"], description: "Do not actually write or delete files. If not set here, the global options will be checked", getDefaultValue: () => DEFAULT_DRY_RUN); - var src_opts = OptionWithMultipleTokens(aliases: ["--src-options"], description: "Options for the source backend"); - var dst_opts = OptionWithMultipleTokens(aliases: ["--dst-options"], description: "Options for the destination backend"); + var src_opts = OptionWithMultipleTokens(aliases: ["--src-options"], description: "Options for the source backend. Each option is a key-value pair separated by an equals sign, e.g. --src-options key1=value1 key2=value2"); + var dst_opts = OptionWithMultipleTokens(aliases: ["--dst-options"], description: "Options for the destination backend. Each option is a key-value pair separated by an equals sign, e.g. --dst-options key1=value1 key2=value2"); var verify_contents_opt = new Option(aliases: ["--verify-contents"], description: "Verify the contents of the files to decide whether the pre-existing destination files should be overwritten", getDefaultValue: () => DEFAULT_VERIFY_CONTENTS); var verify_get_after_put_opt = new Option(aliases: ["--verify-get-after-put"], description: "Verify the files after uploading them to ensure that they were uploaded correctly", getDefaultValue: () => DEFAULT_VERIFY_GET_AFTER_PUT); var retry_opt = new Option(aliases: ["--retry"], description: "Number of times to retry on errors", getDefaultValue: () => DEFAULT_RETRY) { Arity = ArgumentArity.ExactlyOne }; var force_opt = new Option(aliases: ["--force"], description: "Force the synchronization", getDefaultValue: () => DEFAULT_FORCE); var retention_opt = new Option(aliases: ["--retention"], description: "Toggles whether to keep old files. Any deletes will be renames instead", getDefaultValue: () => DEFAULT_RETENTION); var confirm_opt = new Option(aliases: ["--confirm"], description: "Automatically confirm the operation", getDefaultValue: () => DEFAULT_CONFIRM); - var global_opts = OptionWithMultipleTokens(aliases: ["--global-options"], description: "Global options all backends. May be overridden by backend specific options (src-options, dst-options)"); + var global_opts = OptionWithMultipleTokens(aliases: ["--global-options"], description: "Global options all backends. May be overridden by backend specific options (src-options, dst-options). Each option is a key-value pair separated by an equals sign, e.g. --global-options key1=value1 key2=value2"); var log_level_opt = new Option(aliases: ["--log-level"], description: "The log level to use. If not set here, global options will be checked", getDefaultValue: () => DEFAULT_LOG_LEVEL) { Arity = ArgumentArity.ExactlyOne }; var log_file_opt = new Option(aliases: ["--log-file"], description: "The log file to write to. If not set here, global options will be checked", getDefaultValue: () => DEFAULT_LOG_FILE) { Arity = ArgumentArity.ExactlyOne }; var progress_opt = new Option(aliases: ["--progress"], description: "Print progress to STDOUT", getDefaultValue: () => DEFAULT_PROGRESS); From 1d6774b5d3812a7431f63d32a19c03a3d4ab0a2e Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 3 Feb 2025 10:32:18 +0100 Subject: [PATCH 083/362] Log file was being created incorrectly --- Tools/RemoteSynchronization/Program.cs | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index f30baac27..50425d7c8 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2025, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a @@ -19,6 +19,7 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using System; using System.Collections.Generic; @@ -198,12 +199,21 @@ destination will be verified before being overwritten (if they seemingly match). var log_level_parsed = Enum.TryParse(log_level, true, out var log_level_enum); log_level_enum = log_level_parsed ? log_level_enum : Duplicati.Library.Logging.LogMessageType.Information; + using var console_sink = new Duplicati.CommandLine.ConsoleOutput(Console.Out, global_options); + using var multi_sink = new Duplicati.Library.Main.ControllerMultiLogTarget(console_sink, log_level_enum, null); + // Parse the log file var log_file = options["log-file"] as string ?? DEFAULT_LOG_FILE; - var file_sink = (Duplicati.Library.Main.IMessageSink?)(string.IsNullOrEmpty(log_file) ? new Duplicati.Library.Logging.StreamLogDestination(log_file) : null); - var console_sink = new Duplicati.CommandLine.ConsoleOutput(Console.Out, global_options); - Duplicati.Library.Main.IMessageSink[] sinks = file_sink is null ? [console_sink] : [file_sink, console_sink]; - var multi_sink = new Duplicati.Library.Main.MultiMessageSink(sinks); + // The log file sink doesn't have to be disposed, as the multi_sink will take care of it + Duplicati.Library.Logging.StreamLogDestination? log_file_sink = null; + if (!string.IsNullOrEmpty(log_file)) + { + string log_file_dir = SystemIO.IO_OS.PathGetDirectoryName(log_file); + if (!string.IsNullOrEmpty(log_file_dir) && !SystemIO.IO_OS.DirectoryExists(log_file_dir)) + SystemIO.IO_OS.DirectoryCreate(log_file_dir); + log_file_sink = new Duplicati.Library.Logging.StreamLogDestination(log_file); + } + multi_sink.AddTarget(log_file_sink, log_level_enum, null); // Start the logging scope using var _ = Duplicati.Library.Logging.Log.StartScope(multi_sink, log_level_enum); From 217c26c6638550a387e782d85ba5c0074f2561ee Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 3 Feb 2025 10:32:30 +0100 Subject: [PATCH 084/362] Added more tests of the log level enum --- Duplicati/UnitTest/ToolTests.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Duplicati/UnitTest/ToolTests.cs b/Duplicati/UnitTest/ToolTests.cs index cfd7aa2d1..448a8eedc 100644 --- a/Duplicati/UnitTest/ToolTests.cs +++ b/Duplicati/UnitTest/ToolTests.cs @@ -52,6 +52,9 @@ namespace Duplicati.UnitTest ["source", "destination", "--parse-arguments-only", "--verify-get-after-put"], ["source", "destination", "--parse-arguments-only", "--retry", "3"], ["source", "destination", "--parse-arguments-only", "--log-level", "Debug"], + ["source", "destination", "--parse-arguments-only", "--log-level", "Information"], + ["source", "destination", "--parse-arguments-only", "--log-level", "Profiling"], + ["source", "destination", "--parse-arguments-only", "--log-level", "Verbose"], ["source", "destination", "--parse-arguments-only", "--log-file", "somefile.log"], ["source", "destination", "--parse-arguments-only", "--progress"], ["source", "destination", "--parse-arguments-only", "--retention"], From a8edca231b6945320885759bd07146f4271e79e8 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 3 Feb 2025 13:12:38 +0100 Subject: [PATCH 085/362] Added a test with everything being empty --- Duplicati/UnitTest/ToolTests.cs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/Duplicati/UnitTest/ToolTests.cs b/Duplicati/UnitTest/ToolTests.cs index 448a8eedc..5ae5e8338 100644 --- a/Duplicati/UnitTest/ToolTests.cs +++ b/Duplicati/UnitTest/ToolTests.cs @@ -81,6 +81,28 @@ namespace Duplicati.UnitTest Assert.AreEqual(1, failed_result, "Invalid option did not return 1"); } + /// + /// Tests that the remote synchronization tool works with an empty source to an empty destination. + /// + [Test] + [Category("Tools")] + public void TestEmptySourceAndDestination() + { + var l1 = Path.Combine(TARGETFOLDER, "empty_src"); + var l2 = Path.Combine(TARGETFOLDER, "l2"); + + Directory.CreateDirectory(l1); + Directory.CreateDirectory(l2); + + var args = new string[] { $"file://{l1}", $"file://{l2}", "--confirm" }; + + var async_call = RemoteSynchronization.Program.Main(args); + var return_code = async_call.ConfigureAwait(false).GetAwaiter().GetResult(); + + Assert.AreEqual(0, return_code, "Remote synchronization tool did not return 0."); + Assert.IsTrue(DirectoriesAndContentsAreEqual(l1, l2), "Synchronized directories are not equal"); + } + /// /// Tests the original inded use of the remote synchronization tool on an empty destination. /// From 55aca59aeb201299242a47ab56d523b017b89128 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 3 Feb 2025 13:13:53 +0100 Subject: [PATCH 086/362] Added a test for whether an empty source deletes destination files. --- Duplicati/UnitTest/ToolTests.cs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/Duplicati/UnitTest/ToolTests.cs b/Duplicati/UnitTest/ToolTests.cs index 5ae5e8338..1beb9545c 100644 --- a/Duplicati/UnitTest/ToolTests.cs +++ b/Duplicati/UnitTest/ToolTests.cs @@ -103,6 +103,30 @@ namespace Duplicati.UnitTest Assert.IsTrue(DirectoriesAndContentsAreEqual(l1, l2), "Synchronized directories are not equal"); } + /// + /// Test that remote synchronizing an empty source to a non-empty destination deletes the destination files. + /// + [Test] + [Category("Tools")] + public void TestEmptySourceDeletesDestination() + { + var l1 = Path.Combine(TARGETFOLDER, "empty_src"); + var l2 = Path.Combine(TARGETFOLDER, "l2"); + + Directory.CreateDirectory(l1); + Directory.CreateDirectory(l2); + + GenerateTestData(l2, 5, 0, 0, 1024).Wait(); + + var args = new string[] { $"file://{l1}", $"file://{l2}", "--confirm" }; + + var async_call = RemoteSynchronization.Program.Main(args); + var return_code = async_call.ConfigureAwait(false).GetAwaiter().GetResult(); + + Assert.AreEqual(0, return_code, "Remote synchronization tool did not return 0."); + Assert.IsTrue(DirectoriesAndContentsAreEqual(l1, l2), "Synchronized directories are not equal"); + } + /// /// Tests the original inded use of the remote synchronization tool on an empty destination. /// From 2dd3aaffb3d0797d68c6828039b9e6c99f3d12e4 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 3 Feb 2025 13:21:45 +0100 Subject: [PATCH 087/362] Added a test for the retention option with an empty source directory --- Duplicati/UnitTest/ToolTests.cs | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/Duplicati/UnitTest/ToolTests.cs b/Duplicati/UnitTest/ToolTests.cs index 1beb9545c..2803dcd4b 100644 --- a/Duplicati/UnitTest/ToolTests.cs +++ b/Duplicati/UnitTest/ToolTests.cs @@ -127,6 +127,39 @@ namespace Duplicati.UnitTest Assert.IsTrue(DirectoriesAndContentsAreEqual(l1, l2), "Synchronized directories are not equal"); } + /// + /// Test that remote synchronizing an empty source to a non-empty destination renames the destination files when the `--retention` option is used. + /// + [Test] + [Category("Tools")] + public void TestEmptySourceRenamesDestination() + { + var l1 = Path.Combine(TARGETFOLDER, "empty_src"); + var l2 = Path.Combine(TARGETFOLDER, "l2"); + + Directory.CreateDirectory(l1); + Directory.CreateDirectory(l2); + + GenerateTestData(l2, 5, 0, 0, 1024).Wait(); + + var filelist = Directory.EnumerateFiles(l2).ToList(); + var files = filelist.Select(x => File.ReadAllBytes(x)).ToList(); + + var args = new string[] { $"file://{l1}", $"file://{l2}", "--confirm", "--retention" }; + + var async_call = RemoteSynchronization.Program.Main(args); + var return_code = async_call.ConfigureAwait(false).GetAwaiter().GetResult(); + + Assert.AreEqual(0, return_code, "Remote synchronization tool did not return 0."); + + var newfilelist = Directory.EnumerateFiles(l2).ToList(); + foreach (var (name, contents) in filelist.Zip(files)) + { + var newcontents = File.ReadAllBytes(newfilelist.FirstOrDefault(x => x.StartsWith(name))); + Assert.AreEqual(contents, newcontents, "File contents are not equal"); + } + } + /// /// Tests the original inded use of the remote synchronization tool on an empty destination. /// From 0e86f4459397ccd499aece083c7f0d9ad674bdcf Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 3 Feb 2025 13:41:01 +0100 Subject: [PATCH 088/362] Added a test for dry run --- Duplicati/UnitTest/ToolTests.cs | 100 +++++++++++++++++++------------- 1 file changed, 61 insertions(+), 39 deletions(-) diff --git a/Duplicati/UnitTest/ToolTests.cs b/Duplicati/UnitTest/ToolTests.cs index 2803dcd4b..36090af8b 100644 --- a/Duplicati/UnitTest/ToolTests.cs +++ b/Duplicati/UnitTest/ToolTests.cs @@ -36,49 +36,26 @@ namespace Duplicati.UnitTest public class ToolTests : BasicSetupHelper { - /// - /// Tests passing all arguments to the main method of the remote synchronization tool. - /// [Test] - public void TestMainMethodParsesArgumentsCorrectly() + [Category("Tools")] + public void TestDryRun() { - string[][] testCases = - { - ["source", "destination", "--parse-arguments-only"], - ["source", "destination", "--parse-arguments-only", "--dry-run"], - ["source", "destination", "--parse-arguments-only", "--force"], - ["source", "destination", "--parse-arguments-only", "--dry-run", "--force"], - ["source", "destination", "--parse-arguments-only", "--verify-contents"], - ["source", "destination", "--parse-arguments-only", "--verify-get-after-put"], - ["source", "destination", "--parse-arguments-only", "--retry", "3"], - ["source", "destination", "--parse-arguments-only", "--log-level", "Debug"], - ["source", "destination", "--parse-arguments-only", "--log-level", "Information"], - ["source", "destination", "--parse-arguments-only", "--log-level", "Profiling"], - ["source", "destination", "--parse-arguments-only", "--log-level", "Verbose"], - ["source", "destination", "--parse-arguments-only", "--log-file", "somefile.log"], - ["source", "destination", "--parse-arguments-only", "--progress"], - ["source", "destination", "--parse-arguments-only", "--retention"], - ["source", "destination", "--parse-arguments-only", "--confirm"], - ["source", "destination", "--parse-arguments-only", "--global-options", "someglobalkey=someglobalvalue", "anotherglobalkey=anotherglobalvalue"], - ["source", "destination", "--parse-arguments-only", "--src-options", "somesrckey=somesrcvalue", "anothersrckey=anothersrcvalue"], - ["source", "destination", "--parse-arguments-only", "--dst-options", "somedstkey=somedstvalue", "anotherdstkey=anotherdstvalue"], - [ - "source", "destination", "--parse-arguments-only", - "--dry-run", "--force", "--verify-contents", "--retry", "3", "--log-level", "Debug", "--log-file", "somefile.log", "--progress", "--retention", "--confirm", - "--global-options", "someglobalkey=someglobalvalue", "anotherglobalkey=anotherglobalvalue", - "--src-options", "somesrckey=somesrcvalue", "anothersrckey=anothersrcvalue", - "--dst-options", "somedstkey=somedstvalue", "anotherdstkey=anotherdstvalue" - ], - }; + var l1 = Path.Combine(TARGETFOLDER, "l1"); + var l2 = Path.Combine(TARGETFOLDER, "l2"); - foreach (var args in testCases) - { - int result = RemoteSynchronization.Program.Main(args).ConfigureAwait(false).GetAwaiter().GetResult(); - Assert.AreEqual(0, result, $"Failed for args: {string.Join(" ", args)}"); - } + Directory.CreateDirectory(l1); + Directory.CreateDirectory(l2); - int failed_result = RemoteSynchronization.Program.Main(["source", "destination", "--bogus-option"]).ConfigureAwait(false).GetAwaiter().GetResult(); - Assert.AreEqual(1, failed_result, "Invalid option did not return 1"); + GenerateTestData(l1, 5, 0, 0, 1024).Wait(); + + var args = new string[] { $"file://{l1}", $"file://{l2}", "--confirm", "--dry-run" }; + + var async_call = RemoteSynchronization.Program.Main(args); + var return_code = async_call.ConfigureAwait(false).GetAwaiter().GetResult(); + + Assert.AreEqual(0, return_code, "Remote synchronization tool did not return 0."); + Assert.IsFalse(DirectoriesAndContentsAreEqual(l1, l2), "Synchronized directories are equal"); + Assert.IsTrue(!Directory.EnumerateFiles(l2).Any(), "Destination directory is not empty"); } /// @@ -160,6 +137,51 @@ namespace Duplicati.UnitTest } } + /// + /// Tests passing all arguments to the main method of the remote synchronization tool. + /// + [Test] + public void TestMainMethodParsesArgumentsCorrectly() + { + string[][] testCases = + { + ["source", "destination", "--parse-arguments-only"], + ["source", "destination", "--parse-arguments-only", "--dry-run"], + ["source", "destination", "--parse-arguments-only", "--force"], + ["source", "destination", "--parse-arguments-only", "--dry-run", "--force"], + ["source", "destination", "--parse-arguments-only", "--verify-contents"], + ["source", "destination", "--parse-arguments-only", "--verify-get-after-put"], + ["source", "destination", "--parse-arguments-only", "--retry", "3"], + ["source", "destination", "--parse-arguments-only", "--log-level", "Debug"], + ["source", "destination", "--parse-arguments-only", "--log-level", "Information"], + ["source", "destination", "--parse-arguments-only", "--log-level", "Profiling"], + ["source", "destination", "--parse-arguments-only", "--log-level", "Verbose"], + ["source", "destination", "--parse-arguments-only", "--log-file", "somefile.log"], + ["source", "destination", "--parse-arguments-only", "--progress"], + ["source", "destination", "--parse-arguments-only", "--retention"], + ["source", "destination", "--parse-arguments-only", "--confirm"], + ["source", "destination", "--parse-arguments-only", "--global-options", "someglobalkey=someglobalvalue", "anotherglobalkey=anotherglobalvalue"], + ["source", "destination", "--parse-arguments-only", "--src-options", "somesrckey=somesrcvalue", "anothersrckey=anothersrcvalue"], + ["source", "destination", "--parse-arguments-only", "--dst-options", "somedstkey=somedstvalue", "anotherdstkey=anotherdstvalue"], + [ + "source", "destination", "--parse-arguments-only", + "--dry-run", "--force", "--verify-contents", "--retry", "3", "--log-level", "Debug", "--log-file", "somefile.log", "--progress", "--retention", "--confirm", + "--global-options", "someglobalkey=someglobalvalue", "anotherglobalkey=anotherglobalvalue", + "--src-options", "somesrckey=somesrcvalue", "anothersrckey=anothersrcvalue", + "--dst-options", "somedstkey=somedstvalue", "anotherdstkey=anotherdstvalue" + ], + }; + + foreach (var args in testCases) + { + int result = RemoteSynchronization.Program.Main(args).ConfigureAwait(false).GetAwaiter().GetResult(); + Assert.AreEqual(0, result, $"Failed for args: {string.Join(" ", args)}"); + } + + int failed_result = RemoteSynchronization.Program.Main(["source", "destination", "--bogus-option"]).ConfigureAwait(false).GetAwaiter().GetResult(); + Assert.AreEqual(1, failed_result, "Invalid option did not return 1"); + } + /// /// Tests the original inded use of the remote synchronization tool on an empty destination. /// From 4d007795a84ed38c4ec15401d78b1f0432631c15 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 3 Feb 2025 13:47:18 +0100 Subject: [PATCH 089/362] Added a test for the two verify flags --- Duplicati/UnitTest/ToolTests.cs | 42 +++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/Duplicati/UnitTest/ToolTests.cs b/Duplicati/UnitTest/ToolTests.cs index 36090af8b..efef72d35 100644 --- a/Duplicati/UnitTest/ToolTests.cs +++ b/Duplicati/UnitTest/ToolTests.cs @@ -36,6 +36,9 @@ namespace Duplicati.UnitTest public class ToolTests : BasicSetupHelper { + /// + /// Tests that the remote synchronization tool doesn't do anything when the dry run option is used. + /// [Test] [Category("Tools")] public void TestDryRun() @@ -261,6 +264,45 @@ namespace Duplicati.UnitTest Assert.IsTrue(DirectoriesAndContentsAreEqual(DATAFOLDER, l2r), "Restored second level files is not equal to original files"); } + /// + /// Tests that the remote synchronization tool verifies the contents of the files. + /// + [Test] + [Category("Tools")] + public void TestVerifies() + { + var l1 = Path.Combine(TARGETFOLDER, "l1"); + var l2 = Path.Combine(TARGETFOLDER, "l2"); + + Directory.CreateDirectory(l1); + Directory.CreateDirectory(l2); + + GenerateTestData(l1, 5, 0, 0, 1024).Wait(); + + var filenames = Directory.EnumerateFiles(l1).Take(2).ToList(); + var first_file = filenames.First(); + var second_file = filenames.Skip(1).First(); + + File.Copy(first_file, Path.Combine(l2, Path.GetFileName(first_file))); + File.Copy(second_file, Path.Combine(l2, Path.GetFileName(second_file))); + + // Touch the first file to give it a different timestamp + File.SetLastWriteTime(first_file, DateTime.Now.AddMinutes(1)); + + // Modify the second file to give it different contents, but the same size + var second_file_contents = File.ReadAllBytes(second_file); + second_file_contents[0] = (byte)(second_file_contents[0] + 1); + File.WriteAllBytes(second_file, second_file_contents); + + var args = new string[] { $"file://{l1}", $"file://{l2}", "--confirm", "--verify-contents", "--verify-get-after-put" }; + + var async_call = RemoteSynchronization.Program.Main(args); + var return_code = async_call.ConfigureAwait(false).GetAwaiter().GetResult(); + + Assert.AreEqual(0, return_code, "Remote synchronization tool did not return 0."); + Assert.IsTrue(DirectoriesAndContentsAreEqual(l1, l2), "Synchronized directories are not equal"); + } + // // Helper methods // From b5d53bb7415964397684f2114faa3821c2d9d7f6 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 3 Feb 2025 13:53:06 +0100 Subject: [PATCH 090/362] Found a progress print that wasn't properly guarded by the commandline option --- Tools/RemoteSynchronization/Program.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 50425d7c8..0ad3cba88 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -470,10 +470,11 @@ destination will be verified before being overwritten (if they seemingly match). foreach (var f in files) { - if (n > 1) + if (n > 1 && GlobalConfig.Progress) { Console.Write($"\rDeleting: {i}/{n}"); } + try { if (GlobalConfig.DryRun) @@ -490,13 +491,15 @@ destination will be verified before being overwritten (if they seemingly match). { Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e, "Error deleting {0}: {1}", f.Name, e.Message); } + i++; } sw.Stop(); Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", "Delete: {0} ms", sw.ElapsedMilliseconds); - Console.WriteLine($"\rDeleting: {n}/{n}"); + if (GlobalConfig.Progress) + Console.WriteLine($"\rDeleting: {n}/{n}"); return successful_deletes; } From 3b356446df360a678223a404ee221de05d620ea4 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 3 Feb 2025 13:58:30 +0100 Subject: [PATCH 091/362] Added more verbose logging --- Tools/RemoteSynchronization/Program.cs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 0ad3cba88..92b3450ab 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -278,6 +278,8 @@ destination will be verified before being overwritten (if they seemingly match). } } + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "The remote synchronization plan is to {0} {1} files from {2}, then copy {3} files from {4} to {2}.", retention ? "rename" : "delete", to_delete.Count(), b2s.DisplayName, to_copy.Count(), b1s.DisplayName); + // As this is a potentially destructive operation, ask for confirmation if (!confirm) { @@ -386,6 +388,9 @@ destination will be verified before being overwritten (if they seemingly match). { if (GlobalConfig.Progress) Console.Write($"\rCopying: {i}/{n}"); + + Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", "Copying {0} from {1} to {2}", f.Name, b_src.DisplayName, b_dst.DisplayName); + try { sw_get_src.Start(); @@ -425,6 +430,7 @@ destination will be verified before being overwritten (if they seemingly match). } } } + successful_copies++; } catch (Exception e) @@ -475,6 +481,8 @@ destination will be verified before being overwritten (if they seemingly match). Console.Write($"\rDeleting: {i}/{n}"); } + Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", "Deleting {0} from {1}", f.Name, b.DisplayName); + try { if (GlobalConfig.DryRun) @@ -641,6 +649,8 @@ destination will be verified before being overwritten (if they seemingly match). if (GlobalConfig.Progress) Console.Write($"\rRenaming: {i}/{n}"); + Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", "Renaming {0} to {0}.{1} by deleting and re-uploading {2} bytes to {3}", f.Name, suffix, downloaded.Length, sb.DisplayName); + try { sw_get_src.Start(); @@ -696,6 +706,8 @@ destination will be verified before being overwritten (if they seemingly match). if (GlobalConfig.Progress) Console.Write($"\rRenaming: {i}/{n}"); + Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", "Renaming {0} to {0}.{1} by calling Rename on {2}", f.Name, suffix, rb.DisplayName); + try { if (GlobalConfig.DryRun) @@ -753,6 +765,8 @@ destination will be verified before being overwritten (if they seemingly match). if (GlobalConfig.Progress) Console.Write($"\rVerifying: {i}/{n}"); + Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", "Verifying {0} by downloading and comparing {1} bytes from {2} and {3}", f.Name, s_src.Length, b_dst.DisplayName, b_src.DisplayName); + try { // Get both files From a203de90e1193dfe825505b7dcbf71467f9ea43a Mon Sep 17 00:00:00 2001 From: "Marcelo C." Date: Mon, 3 Feb 2025 11:05:27 -0300 Subject: [PATCH 092/362] Added support for Topic ID on Telegram This adds support for TopicID as specified in parameter "send-telegram-topid-id" which allows the messages to be organized in topics on a group. Finding out your topic ID in a group can be rather tricky the easiest way is to use curl to create it. ```bash export CHAT_ID="-99999999" export TOPIC_NAME="Your Topic" export BOT_TOKEN="7777777777:AAAAAAAAA-4WwwwwwwwwwwwwwwwwwwwwwwQ" curl -X POST \ -H "Content-Type: application/json" \ -d "{\"chat_id\":\"$CHAT_ID\",\"name\":\"$TOPIC_NAME\"}" \ https://api.telegram.org/bot$BOT_TOKEN/createForumTopic ``` Which will response with a json where message_thread_id is the Topic ID ```bash {"ok":true,"result":{"message_thread_id":33,"name":..... ``` --- .../Modules/Builtin/SendTelegramMessage.cs | 46 ++++++++---- Duplicati/Library/Modules/Builtin/Strings.cs | 72 ++++++++++--------- 2 files changed, 69 insertions(+), 49 deletions(-) diff --git a/Duplicati/Library/Modules/Builtin/SendTelegramMessage.cs b/Duplicati/Library/Modules/Builtin/SendTelegramMessage.cs index 394d41c31..b4a73948a 100644 --- a/Duplicati/Library/Modules/Builtin/SendTelegramMessage.cs +++ b/Duplicati/Library/Modules/Builtin/SendTelegramMessage.cs @@ -56,6 +56,10 @@ public class SendTelegramMessage : ReportHelper /// private const string OPTION_CHANNEL = "send-telegram-channel-id"; /// + /// Option used to specify the topic ID to be used in telegram groups. + /// + private const string OPTION_TOPICID = "send-telegram-topid-id"; + /// /// Option used to specify report body /// private const string OPTION_MESSAGE = "send-telegram-message"; @@ -133,20 +137,19 @@ public class SendTelegramMessage : ReportHelper /// public override IList SupportedCommands => [ - new CommandLineArgument(OPTION_CHANNEL, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendtelegramchannelShort, Strings.SendTelegramMessage.SendtelegramchannelLong), - new CommandLineArgument(OPTION_MESSAGE, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendtelegrammessageShort, Strings.SendTelegramMessage.SendtelegrammessageLong, DEFAULT_BODY), - new CommandLineArgument(OPTION_BOTID, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendtelegrambotidShort, Strings.SendTelegramMessage.SendtelegrambotidLong), - new CommandLineArgument(OPTION_APIKEY, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendtelegramapikeyShort, Strings.SendTelegramMessage.SendtelegramapikeyLong), - new CommandLineArgument(OPTION_SENDLEVEL, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendtelegramlevelShort, Strings.SendTelegramMessage.SendtelegramlevelLong(ParsedResultType.Success.ToString(), ParsedResultType.Warning.ToString(), ParsedResultType.Error.ToString(), ParsedResultType.Fatal.ToString(), "All"), DEFAULT_LEVEL, null, Enum.GetNames(typeof(ParsedResultType)).Union(new string[] { "All" } ).ToArray()), - new CommandLineArgument(OPTION_SENDALL, CommandLineArgument.ArgumentType.Boolean, Strings.SendTelegramMessage.SendtelegramanyoperationShort, Strings.SendTelegramMessage.SendtelegramanyoperationLong), - + new CommandLineArgument(OPTION_CHANNEL, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramChannelShort, Strings.SendTelegramMessage.SendTelegramChannelLong), + new CommandLineArgument(OPTION_MESSAGE, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramMessageShort, Strings.SendTelegramMessage.SendTelegramMessageLong, DEFAULT_BODY), + new CommandLineArgument(OPTION_BOTID, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramBotIdShort, Strings.SendTelegramMessage.SendTelegramBotIdLong), + new CommandLineArgument(OPTION_APIKEY, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramApiKeyShort, Strings.SendTelegramMessage.SendTelegramApiKeyLong), + new CommandLineArgument(OPTION_SENDLEVEL, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramLevelShort, Strings.SendTelegramMessage.SendTelegramlevelLong(ParsedResultType.Success.ToString(), ParsedResultType.Warning.ToString(), ParsedResultType.Error.ToString(), ParsedResultType.Fatal.ToString(), "All"), DEFAULT_LEVEL, null, Enum.GetNames(typeof(ParsedResultType)).Union(new string[] { "All" } ).ToArray()), + new CommandLineArgument(OPTION_SENDALL, CommandLineArgument.ArgumentType.Boolean, Strings.SendTelegramMessage.SendTelegramManyOperationShort, Strings.SendTelegramMessage.SendTelegramManyOperationLong), new CommandLineArgument(OPTION_LOG_LEVEL, CommandLineArgument.ArgumentType.Enumeration, Strings.ReportHelper.OptionLoglevelShort, Strings.ReportHelper.OptionLoglevelLong, DEFAULT_LOG_LEVEL.ToString(), null, Enum.GetNames(typeof(Logging.LogMessageType))), new CommandLineArgument(OPTION_LOG_FILTER, CommandLineArgument.ArgumentType.String, Strings.ReportHelper.OptionLogfilterShort, Strings.ReportHelper.OptionLogfilterLong), new CommandLineArgument(OPTION_MAX_LOG_LINES, CommandLineArgument.ArgumentType.Integer, Strings.ReportHelper.OptionmaxloglinesShort, Strings.ReportHelper.OptionmaxloglinesLong, DEFAULT_LOGLINES.ToString()), - new CommandLineArgument(OPTION_RESULT_FORMAT, CommandLineArgument.ArgumentType.Enumeration, Strings.ReportHelper.ResultFormatShort, Strings.ReportHelper.ResultFormatLong(Enum.GetNames(typeof(ResultExportFormat))), DEFAULT_EXPORT_FORMAT.ToString(), null, Enum.GetNames(typeof(ResultExportFormat))), + new CommandLineArgument(OPTION_TOPICID, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramTopicShort, Strings.SendTelegramMessage.SendTelegramTopicLong), ]; - + protected override string SubjectOptionName => OPTION_MESSAGE; protected override string BodyOptionName => OPTION_MESSAGE; protected override string ActionLevelOptionName => OPTION_SENDLEVEL; @@ -168,6 +171,10 @@ public class SendTelegramMessage : ReportHelper /// The Telegram ChannelID /// private string m_channelId; + /// + /// The telegram TopicID + /// + private string m_topicId; /// /// This method is the interception where the module can interact with the execution environment and modify the settings. @@ -182,12 +189,14 @@ public class SendTelegramMessage : ReportHelper commandlineOptions.TryGetValue(OPTION_BOTID, out m_botid); commandlineOptions.TryGetValue(OPTION_APIKEY, out m_apikey); + commandlineOptions.TryGetValue(OPTION_TOPICID, out m_topicId); return true; } #endregion - + + /// protected override string ReplaceTemplate(string input, object result, Exception exception, bool subjectline) { // No need to do the expansion as we throw away the result @@ -220,7 +229,7 @@ public class SendTelegramMessage : ReportHelper // Send all chunks sequentially for (var i = 0; i < messages.Length; i++) - await SendMessageChunk(messages[i], i + 1, messages.Length); + await SendMessageChunk(messages[i], i + 1, messages.Length).ConfigureAwait(false); } catch (Exception e) { @@ -239,9 +248,15 @@ public class SendTelegramMessage : ReportHelper { try { + if (string.IsNullOrWhiteSpace(m_botid)) + throw new Exception("Telegram Bot ID is required and not set"); + if (string.IsNullOrWhiteSpace(m_apikey)) + throw new Exception("Telegram API Key is required and not set"); + var p = new { chat_id = Uri.EscapeDataString(m_channelId), + message_thread_id = string.IsNullOrWhiteSpace(m_topicId) ? null : Uri.EscapeDataString(m_topicId), parse_mode = "Markdown", text = Uri.EscapeDataString(totalParts > 1 ? $"Part {partNumber}/{totalParts}:\n{message}" @@ -250,13 +265,16 @@ public class SendTelegramMessage : ReportHelper apiKey = Uri.EscapeDataString(m_apikey) }; - var url = $"https://api.telegram.org/bot{p.botId}:{p.apiKey}/sendMessage?chat_id={p.chat_id}&parse_mode={p.parse_mode}&text={p.text}"; + var baseUrl = $"https://api.telegram.org/bot{p.botId}:{p.apiKey}/sendMessage"; + var url = $"{baseUrl}?chat_id={p.chat_id}&parse_mode={p.parse_mode}&text={p.text}" + + $"{(!string.IsNullOrWhiteSpace(m_topicId) ? $"&message_thread_id={m_topicId}" : string.Empty)}"; + using var timeoutToken = new CancellationTokenSource(); timeoutToken.CancelAfter(REQUEST_TIMEOUT); using var client = HttpClientHelper.CreateClient(); - var response = await client.GetAsync(url, timeoutToken.Token); - var responseContent = await response.Content.ReadAsStringAsync(timeoutToken.Token); + var response = await client.GetAsync(url, timeoutToken.Token).ConfigureAwait(false); + var responseContent = await response.Content.ReadAsStringAsync(timeoutToken.Token).ConfigureAwait(false); if (!responseContent.Contains("\"ok\":true")) Logging.Log.WriteWarningMessage(LOGTAG, "telegramSendError", null, diff --git a/Duplicati/Library/Modules/Builtin/Strings.cs b/Duplicati/Library/Modules/Builtin/Strings.cs index e4d9cc974..5d4750cc5 100644 --- a/Duplicati/Library/Modules/Builtin/Strings.cs +++ b/Duplicati/Library/Modules/Builtin/Strings.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Localization.Short; using System; @@ -167,11 +167,12 @@ You can supply multiple options with a comma separator, e.g. ""{0},{1}"". The sp internal static class SendTelegramMessage { - public static string Description { get { return LC.L(@"This module provides support for sending status reports via Telegram messages"); } } - public static string DisplayName { get { return LC.L(@"Telegram report module"); } } - public static string SendtelegramchannelLong { get { return LC.L(@"Use this option to set the channel ID."); } } - public static string SendtelegramchannelShort { get { return LC.L(@"Telegram channel ID"); } } - public static string SendtelegrammessageLong { get { return LC.L(@"This value can be a filename. If the file exists, the file contents will be used as the message. + public static string Description => LC.L(@"This module provides support for sending status reports via Telegram messages"); + public static string DisplayName => LC.L(@"Telegram report module"); + public static string SendTelegramChannelLong => LC.L(@"Use this option to set the channel ID."); + public static string SendTelegramChannelShort => LC.L(@"Telegram channel ID"); + public static string SendTelegramMessageLong => + LC.L(@"This value can be a filename. If the file exists, the file contents will be used as the message. In the message, certain tokens are replaced: %OPERATIONNAME% - The name of the operation, normally ""Backup"" @@ -179,19 +180,20 @@ In the message, certain tokens are replaced: %LOCALPATH% - The path to the local files or folders involved in the operation (if any) %PARSEDRESULT% - The parsed result, if the operation is a backup. Possible values are: Error, Warning, Success -All command line options are also reported within %value%, e.g. %volsize%. Any unknown/unset value is removed."); } } - public static string SendtelegrammessageShort { get { return LC.L(@"The message template"); } } - public static string SendtelegrambotidLong { get { return LC.L(@"Use this option to set a bot ID for the bot that will send the message."); } } +All command line options are also reported within %value%, e.g. %volsize%. Any unknown/unset value is removed."); - public static string SendtelegrambotidShort { get { return LC.L(@"The Telegram bot ID"); } } - public static string SendtelegramapikeyLong { get { return LC.L(@"Use this option to set a API key for the bot that will send the message."); } } - public static string SendtelegramapikeyShort { get { return LC.L(@"The Telegram API key"); } } - public static string SendtelegramlevelLong(string success, string warning, string error, string fatal, string all) { return LC.L(@"You can specify one of ""{0}"", ""{1}"", ""{2}"", ""{3}"". + public static string SendTelegramMessageShort => LC.L(@"The message template"); + public static string SendTelegramBotIdLong => LC.L(@"Use this option to set a bot ID for the bot that will send the message."); + public static string SendTelegramBotIdShort => LC.L(@"The Telegram bot ID"); + public static string SendTelegramApiKeyLong => LC.L(@"Use this option to set a API key for the bot that will send the message."); + public static string SendTelegramApiKeyShort => LC.L(@"The Telegram API key"); + public static string SendTelegramTopicShort => LC.L(@"The Telegram topic ID"); + public static string SendTelegramTopicLong => LC.L(@"Topic ID for the Topic in the telegram group. For more information on Telegram setup, refer to documentation."); + public static string SendTelegramlevelLong(string success, string warning, string error, string fatal, string all) { return LC.L(@"You can specify one of ""{0}"", ""{1}"", ""{2}"", ""{3}"". You can supply multiple options with a comma separator, e.g. ""{0},{1}"". The special value ""{4}"" is a shorthand for ""{0},{1},{2},{3}"" and will cause all backup operations to send a message.", success, warning, error, fatal, all); } - public static string SendtelegramlevelShort { get { return LC.L(@"The messages to send"); } } - public static string SendtelegramanyoperationLong { get { return LC.L(@"By default, messages will only be sent after a backup operation. Use this option to send messages for all operations."); } } - public static string SendtelegramanyoperationShort { get { return LC.L(@"Send messages for all operations"); } } - public static string LoginTimeoutError { get { return LC.L(@"Timeout occurred while sending to Telegram server"); } } + public static string SendTelegramLevelShort => LC.L(@"The messages to send"); + public static string SendTelegramManyOperationLong => LC.L(@"By default, messages will only be sent after a backup operation. Use this option to send messages for all operations."); + public static string SendTelegramManyOperationShort => LC.L(@"Send messages for all operations"); } internal static class SendHttpMessage From ec4c0f80a458ab1f5b1d1b7302a498b4195c1d28 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 3 Feb 2025 19:19:53 +0100 Subject: [PATCH 093/362] Renaming should be with prefix, not suffix, as duplicati picks them up otherwise --- Tools/RemoteSynchronization/Program.cs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 92b3450ab..0dffc427f 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -632,7 +632,7 @@ destination will be verified before being overwritten (if they seemingly match). private static async Task RenameAsync(IBackend b, IEnumerable files) { long successful_renames = 0; - string suffix = $"{System.DateTime.Now:yyyyMMddHHmmss}.old"; + string prefix = $"{System.DateTime.Now:yyyyMMddHHmmss}.old"; using var downloaded = new MemoryStream(); long i = 0, n = files.Count(); @@ -649,7 +649,7 @@ destination will be verified before being overwritten (if they seemingly match). if (GlobalConfig.Progress) Console.Write($"\rRenaming: {i}/{n}"); - Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", "Renaming {0} to {0}.{1} by deleting and re-uploading {2} bytes to {3}", f.Name, suffix, downloaded.Length, sb.DisplayName); + Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", "Renaming {0} to {1}.{0} by deleting and re-uploading {2} bytes to {3}", f.Name, prefix, downloaded.Length, sb.DisplayName); try { @@ -659,12 +659,12 @@ destination will be verified before being overwritten (if they seemingly match). if (GlobalConfig.DryRun) { - Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", "Would rename {0} to {0}.{1} by deleting and re-uploading {2} bytes to {3}", f.Name, suffix, downloaded.Length, sb.DisplayName); + Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", "Would rename {0} to {1}.{0} by deleting and re-uploading {2} bytes to {3}", f.Name, prefix, downloaded.Length, sb.DisplayName); } else { sw_put_dst.Start(); - await sb.PutAsync($"{f.Name}.{suffix}", downloaded, CancellationToken.None); + await sb.PutAsync($"{prefix}.{f.Name}", downloaded, CancellationToken.None); sw_put_dst.Stop(); sw_del_src.Start(); await sb.DeleteAsync(f.Name, CancellationToken.None); @@ -706,18 +706,18 @@ destination will be verified before being overwritten (if they seemingly match). if (GlobalConfig.Progress) Console.Write($"\rRenaming: {i}/{n}"); - Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", "Renaming {0} to {0}.{1} by calling Rename on {2}", f.Name, suffix, rb.DisplayName); + Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", "Renaming {0} to {1}.{0} by calling Rename on {2}", f.Name, prefix, rb.DisplayName); try { if (GlobalConfig.DryRun) { - Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", "Would rename {0} to {0}.{1} by calling Rename on {2}", f.Name, suffix, rb.DisplayName); + Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", "Would rename {0} to {1}.{0} by calling Rename on {2}", f.Name, prefix, rb.DisplayName); } else { sw.Start(); - await rb.RenameAsync(f.Name, $"{f.Name}.{suffix}", CancellationToken.None); + await rb.RenameAsync(f.Name, $"{prefix}.{f.Name}", CancellationToken.None); sw.Stop(); } successful_renames++; From f38fc6ba189cf8bcf3b94aeb7e503328700820ae Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 3 Feb 2025 19:21:08 +0100 Subject: [PATCH 094/362] Extended the general remote synchronization test with more general usage --- Duplicati/UnitTest/ToolTests.cs | 143 ++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) diff --git a/Duplicati/UnitTest/ToolTests.cs b/Duplicati/UnitTest/ToolTests.cs index efef72d35..a34c95f9e 100644 --- a/Duplicati/UnitTest/ToolTests.cs +++ b/Duplicati/UnitTest/ToolTests.cs @@ -20,6 +20,7 @@ // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Common.IO; +using Duplicati.Library.Interface; using Duplicati.Library.Main; using NUnit.Framework; using System; @@ -262,6 +263,148 @@ namespace Duplicati.UnitTest Console.WriteLine($"Restored {results.RestoredFiles} files to {options["restore-path"]} in {DateTime.Now - now}"); } Assert.IsTrue(DirectoriesAndContentsAreEqual(DATAFOLDER, l2r), "Restored second level files is not equal to original files"); + + // Delete the l2r directory + SystemIO.IO_OS.DirectoryDelete(l2r, true); + + // Delete one file from the l2 backup + var files = Directory.EnumerateFiles(l2).ToList(); + File.Delete(files.First()); + + // Check that the restore fails + options["restore-path"] = l2r; + using (var c = new Controller($"file://{l2}", options, null)) + { + now = DateTime.Now; + try + { + var results = c.Restore([]); + } + catch (RemoteListVerificationException) + { + Console.WriteLine($"Failed (as expected) to restore files to {options["restore-path"]} in {DateTime.Now - now}"); + } + } + + // Run the tool again to copy the missing file + now = DateTime.Now; + async_call = exe(args); + async_call.Wait(); + Assert.AreEqual(0, async_call.Result, "Remote synchronization tool did not return 0."); + Console.WriteLine($"Remote synchronization tool returned 0 in {DateTime.Now - now}"); + + // Try to restore the second level again + options["restore-path"] = l2r; + using (var c = new Controller($"file://{l2}", options, null)) + { + now = DateTime.Now; + var results = c.Restore([]); + Assert.AreEqual(0, results.Errors.Count()); + Assert.AreEqual(0, results.Warnings.Count()); + Console.WriteLine($"Restored {results.RestoredFiles} files to {options["restore-path"]} in {DateTime.Now - now}"); + } + + Assert.IsTrue(DirectoriesAndContentsAreEqual(DATAFOLDER, l2r), "Restored second level files is not equal to original files"); + + // Add some more files to the source + GenerateTestData(Path.Combine(DATAFOLDER, "brand_new_files"), 5, 2, 2, 1024 * 1024).Wait(); + + // Backup the new files to l1 + using (var c = new Controller($"file://{l1}", options, null)) + { + now = DateTime.Now; + var results = c.Backup([DATAFOLDER]); + Assert.AreEqual(0, results.Errors.Count()); + Assert.AreEqual(0, results.Warnings.Count()); + Console.WriteLine($"Backed up {results.AddedFiles} files to {l1} in {DateTime.Now - now}"); + } + + // Run the tool again to copy the new files + now = DateTime.Now; + async_call = exe(args); + async_call.Wait(); + Assert.AreEqual(0, async_call.Result, "Remote synchronization tool did not return 0."); + Console.WriteLine($"Remote synchronization tool returned 0 in {DateTime.Now - now}"); + + // Try to restore the second level again + options["restore-path"] = l2r; + using (var c = new Controller($"file://{l2}", options, null)) + { + now = DateTime.Now; + var results = c.Restore([]); + Assert.AreEqual(0, results.Errors.Count()); + Assert.AreEqual(0, results.Warnings.Count()); + Console.WriteLine($"Restored {results.RestoredFiles} files to {options["restore-path"]} in {DateTime.Now - now}"); + } + + Assert.IsTrue(DirectoriesAndContentsAreEqual(DATAFOLDER, l2r), "Restored second level files is not equal to original files"); + + // Delete the l2r directory + SystemIO.IO_OS.DirectoryDelete(l2r, true); + + // Remove a directory from the source + SystemIO.IO_OS.DirectoryDelete(Path.Combine(DATAFOLDER, "dir_0"), true); + + // Backup the new files to l1 + using (var c = new Controller($"file://{l1}", options, null)) + { + now = DateTime.Now; + var results = c.Backup([DATAFOLDER]); + Assert.AreEqual(0, results.Errors.Count()); + Assert.AreEqual(0, results.Warnings.Count()); + Console.WriteLine($"Backed up {results.AddedFiles} files to {l1} in {DateTime.Now - now}"); + } + + // Compact the backup + using (var c = new Controller($"file://{l1}", options, null)) + { + now = DateTime.Now; + var results = c.Compact(); + Assert.AreEqual(0, results.Errors.Count()); + Assert.AreEqual(0, results.Warnings.Count()); + Console.WriteLine($"Compacted backup in {DateTime.Now - now}"); + } + + // Run the tool again to copy the new files + now = DateTime.Now; + async_call = exe(args); + async_call.Wait(); + Assert.AreEqual(0, async_call.Result, "Remote synchronization tool did not return 0."); + + // Try to restore the second level again + options["restore-path"] = l2r; + using (var c = new Controller($"file://{l2}", options, null)) + { + now = DateTime.Now; + var results = c.Restore([]); + Assert.AreEqual(0, results.Errors.Count()); + Assert.AreEqual(0, results.Warnings.Count()); + Console.WriteLine($"Restored {results.RestoredFiles} files to {options["restore-path"]} in {DateTime.Now - now}"); + } + + Assert.IsTrue(DirectoriesAndContentsAreEqual(DATAFOLDER, l2r), "Restored second level files is not equal to original files"); + + // Delete the l2r directory + SystemIO.IO_OS.DirectoryDelete(l2r, true); + + // Perform a forced synchronization with retention to check that retention doesn't break a restore + now = DateTime.Now; + async_call = exe([.. args, "--force", "--retention"]); + async_call.Wait(); + Assert.AreEqual(0, async_call.Result, "Remote synchronization tool did not return 0."); + + // Try to restore the second level again + options["restore-path"] = l2r; + using (var c = new Controller($"file://{l2}", options, null)) + { + now = DateTime.Now; + var results = c.Restore([]); + Assert.AreEqual(0, results.Errors.Count()); + Assert.AreEqual(0, results.Warnings.Count()); + Console.WriteLine($"Restored {results.RestoredFiles} files to {options["restore-path"]} in {DateTime.Now - now}"); + } + + Assert.IsTrue(DirectoriesAndContentsAreEqual(DATAFOLDER, l2r), "Restored second level files is not equal to original files"); } /// From 4943a0de2daacfc164ad6563d99747b6f6086c9a Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 3 Feb 2025 19:40:55 +0100 Subject: [PATCH 095/362] Found another suffix > prefix change --- Duplicati/UnitTest/ToolTests.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Duplicati/UnitTest/ToolTests.cs b/Duplicati/UnitTest/ToolTests.cs index a34c95f9e..e5ec55dad 100644 --- a/Duplicati/UnitTest/ToolTests.cs +++ b/Duplicati/UnitTest/ToolTests.cs @@ -136,7 +136,8 @@ namespace Duplicati.UnitTest var newfilelist = Directory.EnumerateFiles(l2).ToList(); foreach (var (name, contents) in filelist.Zip(files)) { - var newcontents = File.ReadAllBytes(newfilelist.FirstOrDefault(x => x.StartsWith(name))); + var filename = Path.GetFileName(name); + var newcontents = File.ReadAllBytes(newfilelist.FirstOrDefault(x => x.EndsWith(filename))); Assert.AreEqual(contents, newcontents, "File contents are not equal"); } } From 0882a89b1dd9552b6f2d066429309a0dbbca3d01 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 4 Feb 2025 08:33:48 +0100 Subject: [PATCH 096/362] Removed comment on beta releases --- Duplicati/Server/webroot/ngax/templates/settings.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Server/webroot/ngax/templates/settings.html b/Duplicati/Server/webroot/ngax/templates/settings.html index abc79760c..d3a116aab 100644 --- a/Duplicati/Server/webroot/ngax/templates/settings.html +++ b/Duplicati/Server/webroot/ngax/templates/settings.html @@ -138,7 +138,7 @@
-
Try out the new features that we are working on. Currently the most stable version available. Test Restore data before using this in production environments.
+
Try out the new features that we are working on. Test Backup & Restore before using this in production environments.
From abe79b1a15b5142922f7bb77e80399b1704bcc01 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 6 Feb 2025 11:50:38 +0100 Subject: [PATCH 097/362] Added option to disable update check This fixes #3983 --- Duplicati/Library/RestAPI/Strings.cs | 2 ++ Duplicati/Library/RestAPI/UpdatePollThread.cs | 16 +++++++++++----- Duplicati/Server/Program.cs | 4 +++- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/Duplicati/Library/RestAPI/Strings.cs b/Duplicati/Library/RestAPI/Strings.cs index 115f56e7b..a7e7d3e0c 100644 --- a/Duplicati/Library/RestAPI/Strings.cs +++ b/Duplicati/Library/RestAPI/Strings.cs @@ -59,6 +59,8 @@ Error message: {0}", error); } public static string WebserverAllowedhostnamesDescription { get { return LC.L(@"The hostnames that are accepted, separated with semicolons. If any of the hostnames are ""*"", all hostnames are allowed and the hostname checking is disabled."); } } public static string PingpongkeepaliveLong { get { return LC.L(@"When running as a server, the service daemon must verify that the process is responding. If this option is enabled, the server reads stdin and writes a reply to each line read."); } } public static string PingpongkeepaliveShort { get { return LC.L(@"Enable the ping-pong responder"); } } + public static string DisableupdatecheckShort { get { return LC.L(@"Disable the automatic update check"); } } + public static string DisableupdatecheckLong { get { return LC.L(@"Use this option to disable the automatic update check. Manual update checks can still be performed."); } } public static string LogretentionLong { get { return LC.L(@"Set the time after which log data will be purged from the database."); } } public static string LogretentionShort { get { return LC.L(@"Clean up old log data"); } } public static string ServerdatafolderLong(string envname) { return LC.L(@"Duplicati needs to store a small database with all settings. Use this option to choose where the settings are stored. This option can also be set with the environment variable {0}.", envname); } diff --git a/Duplicati/Library/RestAPI/UpdatePollThread.cs b/Duplicati/Library/RestAPI/UpdatePollThread.cs index 5996c46e6..ee251f244 100644 --- a/Duplicati/Library/RestAPI/UpdatePollThread.cs +++ b/Duplicati/Library/RestAPI/UpdatePollThread.cs @@ -39,6 +39,7 @@ namespace Duplicati.Server private readonly object m_lock = new object(); private AutoResetEvent m_waitSignal; private double m_downloadProgress; + private bool m_disableChecks = false; public bool IsUpdateRequested { get; private set; } = false; @@ -56,8 +57,9 @@ namespace Duplicati.Server } } - public void Init() + public void Init(bool disableChecks) { + m_disableChecks = disableChecks; m_waitSignal = new AutoResetEvent(false); ThreadState = UpdatePollerStates.Waiting; m_thread = new Thread(Run) @@ -115,16 +117,20 @@ namespace Duplicati.Server if (nextCheck < DateTime.UtcNow || m_forceCheck) { + DateTime started = DateTime.UtcNow; + connection.ApplicationSettings.LastUpdateCheck = started; + nextCheck = connection.ApplicationSettings.NextUpdateCheck; + + // If this is not forced, and we have disabled checks, we just update the next check time + if (m_disableChecks && !m_forceCheck) + continue; + lock (m_lock) m_forceCheck = false; ThreadState = UpdatePollerStates.Checking; eventPollNotify.SignalNewEvent(); - DateTime started = DateTime.UtcNow; - connection.ApplicationSettings.LastUpdateCheck = started; - nextCheck = connection.ApplicationSettings.NextUpdateCheck; - Library.AutoUpdater.ReleaseType rt; if (!Enum.TryParse(connection.ApplicationSettings.UpdateChannel, true, out rt)) rt = Duplicati.Library.AutoUpdater.ReleaseType.Unknown; diff --git a/Duplicati/Server/Program.cs b/Duplicati/Server/Program.cs index b43feb0f0..25b37a954 100644 --- a/Duplicati/Server/Program.cs +++ b/Duplicati/Server/Program.cs @@ -51,6 +51,7 @@ namespace Duplicati.Server private const string DISABLE_DB_ENCRYPTION_OPTION = "disable-db-encryption"; private const string REQUIRE_DB_ENCRYPTION_KEY_OPTION = "require-db-encryption-key"; private const string SETTINGS_ENCRYPTION_KEY_OPTION = "settings-encryption-key"; + private const string DISABLE_UPDATE_CHECK_OPTION = "disable-update-check"; #if DEBUG private const bool DEBUG_MODE = true; @@ -266,7 +267,7 @@ namespace Duplicati.Server DuplicatiWebserver = StartWebServer(commandlineOptions, DataConnection).ConfigureAwait(false).GetAwaiter().GetResult(); - UpdatePoller.Init(); + UpdatePoller.Init(Library.Utility.Utility.ParseBoolOption(commandlineOptions, DISABLE_UPDATE_CHECK_OPTION)); SetPurgeTempFilesTimer(commandlineOptions); @@ -961,6 +962,7 @@ namespace Duplicati.Server new Duplicati.Library.Interface.CommandLineArgument(WebServerLoader.OPTION_WEBSERVICE_TIMEZONE, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.String, Strings.Program.WebserverTimezoneDescription, Strings.Program.WebserverTimezoneDescription, TimeZoneHelper.GetLocalTimeZone(), null, TimeZoneHelper.GetTimeZones().Select(x => x.Id).ToArray()), new Duplicati.Library.Interface.CommandLineArgument(WebServerLoader.OPTION_WEBSERVICE_CORS_ORIGINS, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Path, Strings.Program.WebserverCorsOriginsDescription, Strings.Program.WebserverCorsOriginsDescription, WebServerLoader.DEFAULT_OPTION_SPAPATHS), new Duplicati.Library.Interface.CommandLineArgument(PING_PONG_KEEPALIVE_OPTION, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.Program.PingpongkeepaliveShort, Strings.Program.PingpongkeepaliveLong), + new Duplicati.Library.Interface.CommandLineArgument(DISABLE_UPDATE_CHECK_OPTION, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.Program.DisableupdatecheckShort, Strings.Program.DisableupdatecheckLong), new Duplicati.Library.Interface.CommandLineArgument("log-retention", Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Timespan, Strings.Program.LogretentionShort, Strings.Program.LogretentionLong, DEFAULT_LOG_RETENTION), new Duplicati.Library.Interface.CommandLineArgument(DataFolderManager.SERVER_DATAFOLDER_OPTION, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Path, Strings.Program.ServerdatafolderShort, Strings.Program.ServerdatafolderLong(DataFolderManager.DATAFOLDER_ENV_NAME), DataFolderManager.DATAFOLDER), new Duplicati.Library.Interface.CommandLineArgument(DISABLE_DB_ENCRYPTION_OPTION, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.Program.DisabledbencryptionShort, Strings.Program.DisabledbencryptionLong), From bae82feeae9bc4327ef3cb568151237de26b42b3 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 6 Feb 2025 13:20:32 +0100 Subject: [PATCH 098/362] Add a check for stale database This commit adds a check to verify that the remote database is up-to-date before proceeding to repair the remote storage. Without this check it is possible to start repairs with an old database, and this will cause the remote data to be deleted. This fixes #4579 --- Duplicati/Library/Main/Operation/RepairHandler.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index be4083989..821e75381 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -141,6 +141,13 @@ namespace Duplicati.Library.Main.Operation var progress = 0; var targetProgess = tp.ExtraVolumes.Count() + tp.MissingVolumes.Count() + tp.VerificationRequiredVolumes.Count(); + var mostRecentLocal = db.FilesetTimes.Select(x => x.Value).Append(DateTime.MinValue).Max(); + var mostRecentRemote = tp.ParsedVolumes.Select(x => x.Time).Append(DateTime.MinValue).Max(); + if (mostRecentLocal < DateTime.UnixEpoch) + throw new UserInformationException("The local database has no fileset times. Consider deleting the local database and run the repair operation again.", "LocalDatabaseHasNoFilesetTimes"); + if (mostRecentRemote > mostRecentLocal) + throw new UserInformationException($"The remote files are newer ({mostRecentRemote}) than the local database ({mostRecentLocal}), this is likely because the database is outdated. Consider deleting the local database and run the repair operation again.", "RemoteFilesNewerThanLocalDatabase"); + if (m_options.Dryrun) { if (!tp.ParsedVolumes.Any() && tp.OtherVolumes.Any()) From 11643c2413f64294dceabe020d7a1cc925de59ac Mon Sep 17 00:00:00 2001 From: "Marcelo C." Date: Thu, 6 Feb 2025 11:07:17 -0300 Subject: [PATCH 099/362] Removing TimeoutObservingStream from backend manager --- .../Main/Backend/BackendManager.GetOperation.cs | 6 ++---- .../Main/Backend/BackendManager.PutOperation.cs | 10 ++++------ 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/Duplicati/Library/Main/Backend/BackendManager.GetOperation.cs b/Duplicati/Library/Main/Backend/BackendManager.GetOperation.cs index 14bd9a2bf..3d3855ef4 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.GetOperation.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.GetOperation.cs @@ -123,16 +123,14 @@ partial class BackendManager { // extended to use stacked streams using (var fs = System.IO.File.OpenWrite(dlTarget)) - using (var act = new StreamUtil.TimeoutObservingStream(fs) { WriteTimeout = backend is ITimeoutExemptBackend ? Timeout.Infinite : Context.Options.ReadWriteTimeout }) using (var hasher = HashFactory.CreateHasher(Context.Options.FileHashAlgorithm)) - using (var hs = new HashCalculatingStream(act, hasher)) + using (var hs = new HashCalculatingStream(fs, hasher)) using (var ss = new ShaderStream(hs, true)) - using (var linkedToken = CancellationTokenSource.CreateLinkedTokenSource(cancelToken, act.TimeoutToken)) { using (var ts = new ThrottledStream(ss, 0, Context.Options.MaxDownloadPrSecond)) using (var pgs = new ProgressReportingStream(ts, pg => Context.HandleProgress(ts, pg, RemoteFilename))) { - await streamingBackend.GetAsync(RemoteFilename, pgs, linkedToken.Token).ConfigureAwait(false); + await streamingBackend.GetAsync(RemoteFilename, pgs, cancelToken).ConfigureAwait(false); } ss.Flush(); retDownloadSize = ss.TotalBytesWritten; diff --git a/Duplicati/Library/Main/Backend/BackendManager.PutOperation.cs b/Duplicati/Library/Main/Backend/BackendManager.PutOperation.cs index f9733c775..cea59320d 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.PutOperation.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.PutOperation.cs @@ -268,12 +268,10 @@ partial class BackendManager if (backend is IStreamingBackend streamingBackend && !Context.Options.DisableStreamingTransfers) { - using (var fs = System.IO.File.OpenRead(LocalFilename)) - using (var act = new StreamUtil.TimeoutObservingStream(fs) { ReadTimeout = backend is ITimeoutExemptBackend ? Timeout.Infinite : Context.Options.ReadWriteTimeout }) - using (var ts = new ThrottledStream(act, Context.Options.MaxUploadPrSecond, 0)) - using (var pgs = new ProgressReportingStream(ts, pg => Context.HandleProgress(ts, pg, RemoteFilename))) - using (var linkedToken = CancellationTokenSource.CreateLinkedTokenSource(cancelToken, act.TimeoutToken)) - await streamingBackend.PutAsync(RemoteFilename, pgs, linkedToken.Token).ConfigureAwait(false); + using var fs = System.IO.File.OpenRead(LocalFilename); + using var ts = new ThrottledStream(fs, Context.Options.MaxUploadPrSecond, 0); + using var pgs = new ProgressReportingStream(ts,pg => Context.HandleProgress(ts, pg, RemoteFilename)); + await streamingBackend.PutAsync(RemoteFilename, pgs, cancelToken).ConfigureAwait(false); } else await backend.PutAsync(RemoteFilename, LocalFilename, cancelToken).ConfigureAwait(false); From 74d5b757e8f5735585f7f6b401a94449a2d36707 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 6 Feb 2025 15:30:28 +0100 Subject: [PATCH 100/362] Added test that replicates issue #4312 --- Duplicati/UnitTest/Issue4312.cs | 133 ++++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 Duplicati/UnitTest/Issue4312.cs diff --git a/Duplicati/UnitTest/Issue4312.cs b/Duplicati/UnitTest/Issue4312.cs new file mode 100644 index 000000000..dea2f18a5 --- /dev/null +++ b/Duplicati/UnitTest/Issue4312.cs @@ -0,0 +1,133 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using System; +using System.IO; +using System.Linq; +using NUnit.Framework; + +namespace Duplicati.UnitTest; + +public class Issue4312 : BasicSetupHelper +{ + [Test] + [Category("Targeted")] + public void ChangeTimestampShouldCreateExtraBackup() + { + var testopts = TestOptions; + // Make sure we detect changes from metadata + testopts["upload-unchanged-backups"] = "false"; + testopts["blocksize"] = "50kb"; + + var data = new byte[1024 * 1024 * 10]; + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + var r = c.Backup(new string[] { DATAFOLDER }); + Assert.AreEqual(0, r.Errors.Count()); + Assert.AreEqual(0, r.Warnings.Count()); + Assert.AreEqual(1, r.AddedFiles); + var pr = (Library.Interface.IParsedBackendStatistics)r.BackendStatistics; + if (pr.KnownFileSize == 0 || pr.KnownFileCount != 3 || pr.BackupListCount != 1) + throw new Exception(string.Format("Failed to get stats from remote backend: {0}, {1}, {2}", pr.KnownFileSize, pr.KnownFileCount, pr.BackupListCount)); + + System.Threading.Thread.Sleep(3000); + + r = c.Backup(new string[] { DATAFOLDER }); + Assert.AreEqual(0, r.Errors.Count()); + Assert.AreEqual(0, r.Warnings.Count()); + Assert.AreEqual(0, r.AddedFiles); + Assert.AreEqual(0, r.ModifiedFiles); + pr = (Library.Interface.IParsedBackendStatistics)r.BackendStatistics; + if (pr.KnownFileSize == 0 || pr.KnownFileCount != 3 || pr.BackupListCount != 1) + throw new Exception(string.Format("Failed to get stats from remote backend: {0}, {1}, {2}", pr.KnownFileSize, pr.KnownFileCount, pr.BackupListCount)); + + System.Threading.Thread.Sleep(3000); + + // Force a change in the file metadata + File.SetLastWriteTimeUtc(Path.Combine(DATAFOLDER, "a"), DateTime.Now); + + r = c.Backup(new string[] { DATAFOLDER }); + Assert.AreEqual(0, r.Errors.Count()); + Assert.AreEqual(0, r.Warnings.Count()); + Assert.AreEqual(0, r.AddedFiles); + Assert.AreEqual(1, r.ModifiedFiles); + pr = (Library.Interface.IParsedBackendStatistics)r.BackendStatistics; + if (pr.KnownFileSize == 0 || pr.KnownFileCount != 6 || pr.BackupListCount != 2) + throw new Exception(string.Format("Failed to get stats from remote backend: {0}, {1}, {2}", pr.KnownFileSize, pr.KnownFileCount, pr.BackupListCount)); + } + } + + [Test] + [Category("Targeted")] + public void BackupFromRecreatedDatabaseShouldUpdateMetadata() + { + var testopts = TestOptions; + // Make sure we detect changes from metadata + testopts["upload-unchanged-backups"] = "false"; + testopts["blocksize"] = "50kb"; + + var data = new byte[1024 * 1024 * 10]; + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + var r = c.Backup(new string[] { DATAFOLDER }); + Assert.AreEqual(0, r.Errors.Count()); + Assert.AreEqual(0, r.Warnings.Count()); + Assert.AreEqual(1, r.AddedFiles); + var pr = (Library.Interface.IParsedBackendStatistics)r.BackendStatistics; + if (pr.KnownFileSize == 0 || pr.KnownFileCount != 3 || pr.BackupListCount != 1) + throw new Exception(string.Format("Failed to get stats from remote backend: {0}, {1}, {2}", pr.KnownFileSize, pr.KnownFileCount, pr.BackupListCount)); + + // Remove the database + File.Delete(testopts["dbpath"]); + + // Recreate + var rr = c.Repair(); + Assert.AreEqual(0, rr.Errors.Count()); + Assert.AreEqual(0, rr.Warnings.Count()); + + // Because the timestamps are restored with lower precision + // this will trigger a rescan of the files + r = c.Backup(new string[] { DATAFOLDER }); + Assert.AreEqual(0, r.Errors.Count()); + Assert.AreEqual(0, r.Warnings.Count()); + Assert.AreEqual(0, r.AddedFiles); + Assert.AreEqual(0, r.ModifiedFiles); + Assert.AreEqual(1, r.OpenedFiles); + pr = (Library.Interface.IParsedBackendStatistics)r.BackendStatistics; + if (pr.KnownFileSize == 0 || pr.KnownFileCount != 3 || pr.BackupListCount != 1) + throw new Exception(string.Format("Looks like the metadata scan failed: {0}, {1}, {2}", pr.KnownFileSize, pr.KnownFileCount, pr.BackupListCount)); + + // Make a backup again, and ensure that no files are opened + r = c.Backup(new string[] { DATAFOLDER }); + Assert.AreEqual(0, r.Errors.Count()); + Assert.AreEqual(0, r.Warnings.Count()); + Assert.AreEqual(0, r.AddedFiles); + Assert.AreEqual(0, r.ModifiedFiles); + Assert.AreEqual(0, r.OpenedFiles); + pr = (Library.Interface.IParsedBackendStatistics)r.BackendStatistics; + if (pr.KnownFileSize == 0 || pr.KnownFileCount != 3 || pr.BackupListCount != 1) + throw new Exception(string.Format("Failed to get stats from remote backend: {0}, {1}, {2}", pr.KnownFileSize, pr.KnownFileCount, pr.BackupListCount)); + } + + } +} From 1690871d105c43b29ece9fd87f26e634b2316359 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 6 Feb 2025 15:31:20 +0100 Subject: [PATCH 101/362] Implemented fix that pushes updated timestamps to previous version, in the case that no files were changed, and no new backup is recorded in remote storage. --- .../Library/Main/Database/LocalDatabase.cs | 17 ++++++ .../Main/Operation/Backup/BackupDatabase.cs | 5 ++ .../Operation/Backup/BackupStatsCollector.cs | 58 +++++++++++-------- .../Operation/Backup/FileBlockProcessor.cs | 44 ++++++++------ .../Operation/Backup/FilePreFilterProcess.cs | 48 +++++++-------- .../Operation/Backup/MetadataPreProcess.cs | 39 +++++++------ .../Operation/Backup/UploadRealFilelist.cs | 6 ++ Duplicati/Library/Main/ResultClasses.cs | 39 +++++++------ 8 files changed, 152 insertions(+), 104 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 18003cc06..84effc3c0 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -1211,6 +1211,23 @@ ORDER BY } } + public void PushTimestampChangesToPreviousVersion(long filesetId, System.Data.IDbTransaction transaction) + { + using (var cmd = m_connection.CreateCommand()) + { + cmd.Transaction = transaction; + var query = @" +UPDATE FilesetEntry AS oldVersion +SET Lastmodified = tempVersion.Lastmodified +FROM FilesetEntry AS tempVersion +WHERE oldVersion.FileID = tempVersion.FileID +AND tempVersion.FilesetID = ? +AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != ? ORDER BY Timestamp DESC LIMIT 1)"; + + cmd.ExecuteNonQuery(query, filesetId, filesetId); + } + } + /// /// Keeps a list of filenames in a temporary table with a single column Path /// diff --git a/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs b/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs index 372bc2cf5..cc7eb1b39 100644 --- a/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs +++ b/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs @@ -255,6 +255,11 @@ namespace Duplicati.Library.Main.Operation.Backup return RunOnMain(() => m_database.WriteFileset(fsw, filesetid, GetTransaction())); } + public Task PushTimestampChangesToPreviousVersionAsync(long filesetid) + { + return RunOnMain(() => m_database.PushTimestampChangesToPreviousVersion(filesetid, GetTransaction())); + } + public Task UpdateFilesetAndMarkAsFullBackupAsync(long filesetid) { return RunOnMain(() => m_database.UpdateFullBackupStateInFileset(filesetid, true, GetTransaction())); diff --git a/Duplicati/Library/Main/Operation/Backup/BackupStatsCollector.cs b/Duplicati/Library/Main/Operation/Backup/BackupStatsCollector.cs index ad898a891..03572ab17 100644 --- a/Duplicati/Library/Main/Operation/Backup/BackupStatsCollector.cs +++ b/Duplicati/Library/Main/Operation/Backup/BackupStatsCollector.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -41,15 +41,25 @@ namespace Duplicati.Library.Main.Operation.Backup public Task AddOpenedFile(long size) { - return RunOnMain(() => { + return RunOnMain(() => + { m_res.SizeOfOpenedFiles += size; m_res.OpenedFiles++; }); } + public Task AddTimestampChangedFile() + { + return RunOnMain(() => + { + m_res.TimestampChangedFiles++; + }); + } + public Task AddAddedFile(long size) { - return RunOnMain(() => { + return RunOnMain(() => + { m_res.SizeOfAddedFiles += size; m_res.AddedFiles++; }); @@ -57,7 +67,8 @@ namespace Duplicati.Library.Main.Operation.Backup public Task AddModifiedFile(long size) { - return RunOnMain(() => { + return RunOnMain(() => + { m_res.SizeOfModifiedFiles += size; m_res.ModifiedFiles++; }); @@ -65,7 +76,8 @@ namespace Duplicati.Library.Main.Operation.Backup public Task AddExaminedFile(long size) { - return RunOnMain(() => { + return RunOnMain(() => + { m_res.SizeOfExaminedFiles += size; m_res.ExaminedFiles++; }); diff --git a/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs b/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs index 7dcfead03..99ef8ad76 100644 --- a/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs +++ b/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -118,6 +118,12 @@ namespace Duplicati.Library.Main.Operation.Backup Logging.Log.WriteVerboseMessage(FILELOGTAG, "FileMetadataChanged", "File has only metadata changes {0}", e.Path); await database.AddFileAsync(e.PathPrefixID, e.Filename, e.LastWrite, filestreamdata.Blocksetid, metadataid); } + else if (e.TimestampChanged) + { + await stats.AddTimestampChangedFile(); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "FileTimestampChanged", "File has only timestamp changes {0}", e.Path); + await database.AddUnmodifiedAsync(e.OldId, e.LastWrite); + } else /*if (e.OldId >= 0)*/ { // When we write the file to output, update the last modified time diff --git a/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs b/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs index 4276fdefc..a6eba3f8d 100644 --- a/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -75,7 +75,7 @@ namespace Duplicati.Library.Main.Operation.Backup { filestatsize = snapshot.GetFileSize(e.Path); } - catch(Exception ex) + catch (Exception ex) { Logging.Log.WriteExplicitMessage(FILELOGTAG, "FailedToReadSize", ex, "Failed to read size of file: {0}", e.Path); } @@ -96,11 +96,11 @@ namespace Duplicati.Library.Main.Operation.Backup // If we disable the filetime check, we always assume that the file has changed // Otherwise we check that the timestamps are different or if any of them are empty - var timestampChanged = DISABLEFILETIMECHECK || e.LastWrite != e.OldModified || e.LastWrite.Ticks == 0 || e.OldModified.Ticks == 0; + e.TimestampChanged = DISABLEFILETIMECHECK || e.LastWrite != e.OldModified || e.LastWrite.Ticks == 0 || e.OldModified.Ticks == 0; // Avoid generating a new metadata blob if timestamp has not changed // and we only check for timestamp changes - if (CHECKFILETIMEONLY && !timestampChanged && !isNewFile) + if (CHECKFILETIMEONLY && !e.TimestampChanged && !isNewFile) { Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoTimestampChange", "Skipped checking file, because timestamp was not updated {0}", e.Path); try @@ -135,9 +135,9 @@ namespace Duplicati.Library.Main.Operation.Backup // Check if the file is new, or something indicates a change var filesizeChanged = filestatsize < 0 || e.LastFileSize < 0 || filestatsize != e.LastFileSize; - if (isNewFile || timestampChanged || filesizeChanged || e.MetadataChanged) + if (isNewFile || e.TimestampChanged || filesizeChanged || e.MetadataChanged) { - Logging.Log.WriteVerboseMessage(FILELOGTAG, "CheckFileForChanges", "Checking file for changes {0}, new: {1}, timestamp changed: {2}, size changed: {3}, metadatachanged: {4}, {5} vs {6}", e.Path, isNewFile, timestampChanged, filesizeChanged, e.MetadataChanged, e.LastWrite, e.OldModified); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "CheckFileForChanges", "Checking file for changes {0}, new: {1}, timestamp changed: {2}, size changed: {3}, metadatachanged: {4}, {5} vs {6}", e.Path, isNewFile, e.TimestampChanged, filesizeChanged, e.MetadataChanged, e.LastWrite, e.OldModified); await self.Output.WriteAsync(e); } else diff --git a/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs b/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs index b8554d45f..4f990c380 100644 --- a/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -63,6 +63,7 @@ namespace Duplicati.Library.Main.Operation.Backup // After processing metadata public IMetahash MetaHashAndSize; public bool MetadataChanged; + public bool TimestampChanged; } public static Task Run(Channels channels, Snapshots.ISnapshotService snapshot, Options options, BackupDatabase database, long lastfilesetid, ITaskReader taskReader) diff --git a/Duplicati/Library/Main/Operation/Backup/UploadRealFilelist.cs b/Duplicati/Library/Main/Operation/Backup/UploadRealFilelist.cs index 284884cf4..44df554ba 100644 --- a/Duplicati/Library/Main/Operation/Backup/UploadRealFilelist.cs +++ b/Duplicati/Library/Main/Operation/Backup/UploadRealFilelist.cs @@ -72,6 +72,12 @@ internal static class UploadRealFilelist } else { + if (result.TimestampChangedFiles != 0) + { + Logging.Log.WriteInformationMessage(LOGTAG, "DetectedTimestampChanges", "Detected timestamp changes, but no data needs to be uploaded, pushing timestamp changes to the latest fileset"); + await db.PushTimestampChangesToPreviousVersionAsync(filesetid).ConfigureAwait(false); + } + Logging.Log.WriteVerboseMessage(LOGTAG, "RemovingLeftoverTempFile", "removing temp files, as no data needs to be uploaded"); await db.RemoveRemoteVolumeAsync(filesetvolume.RemoteFilename); } diff --git a/Duplicati/Library/Main/ResultClasses.cs b/Duplicati/Library/Main/ResultClasses.cs index 0765ed458..846332274 100644 --- a/Duplicati/Library/Main/ResultClasses.cs +++ b/Duplicati/Library/Main/ResultClasses.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -449,6 +449,7 @@ namespace Duplicati.Library.Main public long AddedFolders { get; internal set; } public long TooLargeFiles { get; internal set; } public long FilesWithError { get; internal set; } + public long TimestampChangedFiles { get; internal set; } public long ModifiedFolders { get; internal set; } public long ModifiedSymlinks { get; internal set; } public long AddedSymlinks { get; internal set; } From aab91eb0dfb86a8051993a58894b4219308897cc Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 6 Feb 2025 16:53:23 +0100 Subject: [PATCH 102/362] Added protection to newly create data folder and databases --- Duplicati/Agent/Settings.cs | 5 +- Duplicati/CommandLine/ServerUtil/Settings.cs | 8 +- .../Library/AutoUpdater/DataFolderLocator.cs | 21 ++++- .../Library/AutoUpdater/DataFolderManager.cs | 3 + Duplicati/Library/Common/IO/ISystemIO.cs | 48 ++++++---- Duplicati/Library/Common/IO/SystemIOLinux.cs | 75 +++++++++++----- .../Library/Common/IO/SystemIOWindows.cs | 88 +++++++++++++++---- Duplicati/Library/Main/CLIDatabaseLocator.cs | 7 +- .../Library/SQLiteHelper/SQLiteLoader.cs | 71 +++++---------- 9 files changed, 199 insertions(+), 127 deletions(-) diff --git a/Duplicati/Agent/Settings.cs b/Duplicati/Agent/Settings.cs index 3788eefa1..079fd626f 100644 --- a/Duplicati/Agent/Settings.cs +++ b/Duplicati/Agent/Settings.cs @@ -149,10 +149,7 @@ public sealed record Settings( private static string GetSettingsPath() { // Ideally, this should use DataFolderManager.DATAFOLDER, but we cannot due to backwards compatibility - var folder = Library.AutoUpdater.DataFolderLocator.GetDefaultStorageFolder(DefaultSettingsFilename); - if (!Directory.Exists(folder)) - Directory.CreateDirectory(folder); - + var folder = Library.AutoUpdater.DataFolderLocator.GetDefaultStorageFolder(DefaultSettingsFilename, true); return Path.Combine(folder, DefaultSettingsFilename); } } diff --git a/Duplicati/CommandLine/ServerUtil/Settings.cs b/Duplicati/CommandLine/ServerUtil/Settings.cs index 1f4528663..5743a4c27 100644 --- a/Duplicati/CommandLine/ServerUtil/Settings.cs +++ b/Duplicati/CommandLine/ServerUtil/Settings.cs @@ -70,14 +70,8 @@ public sealed record Settings( /// The filename to use /// The default storage folder private static string GetDefaultStorageFolder(string filename) - { // Ideally, this should use DataFolderManager.DATAFOLDER, but we cannot due to backwards compatibility - var folder = DataFolderLocator.GetDefaultStorageFolder(filename); - if (!Directory.Exists(folder)) - Directory.CreateDirectory(folder); - - return folder; - } + => DataFolderLocator.GetDefaultStorageFolder(filename, true); /// /// Loads the settings from the settings file diff --git a/Duplicati/Library/AutoUpdater/DataFolderLocator.cs b/Duplicati/Library/AutoUpdater/DataFolderLocator.cs index 1176ee530..ff5e0f2e9 100644 --- a/Duplicati/Library/AutoUpdater/DataFolderLocator.cs +++ b/Duplicati/Library/AutoUpdater/DataFolderLocator.cs @@ -22,6 +22,7 @@ using System; using System.Linq; using System.Collections.Generic; +using Duplicati.Library.Common.IO; #nullable enable @@ -32,7 +33,6 @@ namespace Duplicati.Library.AutoUpdater; /// public static class DataFolderLocator { - /// /// Finds a default storage folder, using the operating system specific locations. /// The targetfilename is used to detect locations that are used in previous versions. @@ -41,12 +41,27 @@ public static class DataFolderLocator /// If the data folder is overriden, the overriden folder is used, and no search is performed. /// /// The filename to look for + /// Whether to create the folder if it does not exist /// The default storage folder - public static string GetDefaultStorageFolder(string targetfilename) - => DataFolderManager.OVERRIDEN_DATAFOLDER + public static string GetDefaultStorageFolder(string targetfilename, bool autoCreate) + { + var folder = DataFolderManager.OVERRIDEN_DATAFOLDER ? DataFolderManager.DATAFOLDER : GetDefaultStorageFolderInternal(targetfilename, AutoUpdateSettings.AppName); + // If the folder does not exist, and we are allowed to create it, we do so + if (autoCreate && !SystemIO.IO_OS.DirectoryExists(folder)) + { + // Create the folder + SystemIO.IO_OS.DirectoryCreate(folder); + + // Make sure the folder is only accessible by the current user + SystemIO.IO_OS.DirectorySetPermissionUserRWOnly(folder); + } + + return folder; + } + /// /// Finds a default storage folder, using the operating system specific locations. /// The targetfilename is used to detect locations that are used in previous versions. diff --git a/Duplicati/Library/AutoUpdater/DataFolderManager.cs b/Duplicati/Library/AutoUpdater/DataFolderManager.cs index db64ba9d2..edbaa32d7 100644 --- a/Duplicati/Library/AutoUpdater/DataFolderManager.cs +++ b/Duplicati/Library/AutoUpdater/DataFolderManager.cs @@ -182,7 +182,10 @@ public static class DataFolderManager } if (!Directory.Exists(DATAFOLDER)) + { Directory.CreateDirectory(DATAFOLDER); + SystemIO.IO_OS.DirectorySetPermissionUserRWOnly(DATAFOLDER); + } if (!File.Exists(Path.Combine(DATAFOLDER, INSTALL_FILE))) { diff --git a/Duplicati/Library/Common/IO/ISystemIO.cs b/Duplicati/Library/Common/IO/ISystemIO.cs index 62479855d..c291d2200 100644 --- a/Duplicati/Library/Common/IO/ISystemIO.cs +++ b/Duplicati/Library/Common/IO/ISystemIO.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -82,6 +82,16 @@ namespace Duplicati.Library.Common.IO void SetMetadata(string path, Dictionary metdata, bool restorePermissions); Dictionary GetMetadata(string path, bool isSymlink, bool followSymlink); + /// + /// Sets the permission to read-write only for the current user. + /// + /// The file to set permissions on. + void FileSetPermissionUserRWOnly(string path); + /// + /// Sets the permission to read-write only for the current user. + /// + /// The directory to set permissions on. + void DirectorySetPermissionUserRWOnly(string path); } } diff --git a/Duplicati/Library/Common/IO/SystemIOLinux.cs b/Duplicati/Library/Common/IO/SystemIOLinux.cs index ef43cd719..1bce31daf 100644 --- a/Duplicati/Library/Common/IO/SystemIOLinux.cs +++ b/Duplicati/Library/Common/IO/SystemIOLinux.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -64,7 +64,8 @@ namespace Duplicati.Library.Common.IO File.SetLastWriteTimeUtc(path, time); var gtime = FileGetLastWriteTimeUtc(path); - if(gtime != time) { + if (gtime != time) + { Console.Error.WriteLine($"DISS: {path} {gtime.ToFileTimeUtc() - time.ToFileTimeUtc()}"); } } @@ -214,7 +215,7 @@ namespace Duplicati.Library.Common.IO var n = PosixFile.GetExtendedAttributes(f, isSymlink, followSymlink); if (n != null) - foreach(var x in n) + foreach (var x in n) dict["unix-ext:" + x.Key] = Convert.ToBase64String(x.Value); var fse = PosixFile.GetUserGroupAndPermissions(f); @@ -238,7 +239,7 @@ namespace Duplicati.Library.Common.IO var f = NormalizePath(file); - foreach(var x in data.Where(x => x.Key.StartsWith("unix-ext:", StringComparison.Ordinal)).Select(x => new KeyValuePair(x.Key.Substring("unix-ext:".Length), Convert.FromBase64String(x.Value)))) + foreach (var x in data.Where(x => x.Key.StartsWith("unix-ext:", StringComparison.Ordinal)).Select(x => new KeyValuePair(x.Key.Substring("unix-ext:".Length), Convert.FromBase64String(x.Value)))) PosixFile.SetExtendedAttribute(f, x.Key, x.Value); if (restorePermissions && data.ContainsKey("unix:uid-gid-perm")) @@ -346,6 +347,36 @@ namespace Duplicati.Library.Common.IO { return new FileEntry(fileInfo.Name, fileInfo.Length, fileInfo.LastAccessTime, fileInfo.LastWriteTime); } + + /// + /// Sets the unix permission user read-write Only. + /// + /// The file to set permissions on. + public void FileSetPermissionUserRWOnly(string path) + { + var fi = PosixFile.GetUserGroupAndPermissions(path); + PosixFile.SetUserGroupAndPermissions( + path, + fi.UID, + fi.GID, + Convert.ToInt32("600", 8) /* FilePermissions.S_IRUSR | FilePermissions.S_IWUSR*/ + ); + } + + /// + /// Sets the permission to read-write only for the current user. + /// + /// The directory to set permissions on. + public void DirectorySetPermissionUserRWOnly(string path) + { + var fi = PosixFile.GetUserGroupAndPermissions(path); + PosixFile.SetUserGroupAndPermissions( + path, + fi.UID, + fi.GID, + Convert.ToInt32("700", 8) /* FilePermissions.S_IRUSR | FilePermissions.S_IWUSR | FilePermissions.S_IXUSR */ + ); + } } } diff --git a/Duplicati/Library/Common/IO/SystemIOWindows.cs b/Duplicati/Library/Common/IO/SystemIOWindows.cs index 32d80495b..528cb8cfc 100644 --- a/Duplicati/Library/Common/IO/SystemIOWindows.cs +++ b/Duplicati/Library/Common/IO/SystemIOWindows.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -28,9 +28,11 @@ using System.Linq; using Duplicati.Library.Interface; using Newtonsoft.Json; using System.Runtime.Versioning; +using System.Security.Principal; namespace Duplicati.Library.Common.IO { + [SupportedOSPlatform("windows")] public struct SystemIOWindows : ISystemIO { // Based on the constant names used in @@ -42,6 +44,8 @@ namespace Duplicati.Library.Common.IO private static readonly string DIRSEP = Util.DirectorySeparatorString; + private static readonly string CURRENT_USERNAME = WindowsIdentity.GetCurrent().Name; + /// /// Prefix path with one of the extended device path prefixes /// (@"\\?\" or @"\\?\UNC\") but only if it's a fully qualified @@ -671,6 +675,52 @@ namespace Duplicati.Library.Common.IO if ((attr & System.IO.FileAttributes.ReparsePoint) == 0) throw new System.IO.IOException(string.Format("Unable to create symlink, check account permissions: {0}", symlinkfile)); } + + /// + /// Sets the unix permission user read-write Only. + /// + /// The file to set permissions on. + public void FileSetPermissionUserRWOnly(string path) + { + // Create directory security settings + var security = new FileSecurity(); + + // Remove inherited permissions to ensure only the current user has access + security.SetAccessRuleProtection(true, false); + + // Grant the current user read access + security.AddAccessRule(new FileSystemAccessRule( + CURRENT_USERNAME, + FileSystemRights.FullControl, + AccessControlType.Allow + )); + + // Adjust with the new security settings + new FileInfo(path).SetAccessControl(security); + } + + /// + /// Sets the permission to read-write only for the current user. + /// + /// The directory to set permissions on. + public void DirectorySetPermissionUserRWOnly(string path) + { + // Create directory security settings + var security = new DirectorySecurity(); + + // Remove inherited permissions to ensure only the current user has access + security.SetAccessRuleProtection(true, false); + + // Grant the current user read access + security.AddAccessRule(new FileSystemAccessRule( + CURRENT_USERNAME, + FileSystemRights.FullControl, + AccessControlType.Allow + )); + + // Adjust with the new security settings + new DirectoryInfo(path).SetAccessControl(security); + } #endregion } } diff --git a/Duplicati/Library/Main/CLIDatabaseLocator.cs b/Duplicati/Library/Main/CLIDatabaseLocator.cs index fe3ab3304..0cd7f93df 100644 --- a/Duplicati/Library/Main/CLIDatabaseLocator.cs +++ b/Duplicati/Library/Main/CLIDatabaseLocator.cs @@ -97,7 +97,7 @@ namespace Duplicati.Library.Main return options.Dbpath; // Ideally, this should use DataFolderManager.DATAFOLDER, but we cannot due to backwards compatibility - var folder = AutoUpdater.DataFolderLocator.GetDefaultStorageFolder(CONFIG_FILE); + var folder = AutoUpdater.DataFolderLocator.GetDefaultStorageFolder(CONFIG_FILE, true); // Emit a warning if the database is stored in the Windows folder if (Util.IsPathUnderWindowsFolder(folder)) Logging.Log.WriteWarningMessage(LOGTAG, "DatabaseInWindowsFolder", null, "The database config is stored in the Windows folder, this is not recommended as it will be deleted on Windows upgrades."); @@ -209,9 +209,6 @@ namespace Duplicati.Library.Main ParameterFile = null }); - if (!System.IO.Directory.Exists(folder)) - System.IO.Directory.CreateDirectory(folder); - var settings = new Newtonsoft.Json.JsonSerializerSettings(); settings.Formatting = Newtonsoft.Json.Formatting.Indented; System.IO.File.WriteAllText(file, Newtonsoft.Json.JsonConvert.SerializeObject(configs, settings), System.Text.Encoding.UTF8); @@ -249,7 +246,7 @@ namespace Duplicati.Library.Main public static bool IsDatabasePathInUse(string path) { // Ideally, this should use DataFolderManager.DATAFOLDER, but we cannot due to backwards compatibility - var file = System.IO.Path.Combine(AutoUpdater.DataFolderLocator.GetDefaultStorageFolder(CONFIG_FILE), CONFIG_FILE); + var file = System.IO.Path.Combine(AutoUpdater.DataFolderLocator.GetDefaultStorageFolder(CONFIG_FILE, false), CONFIG_FILE); if (!System.IO.File.Exists(file)) return false; diff --git a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs index 339422991..754a49b3b 100644 --- a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs +++ b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs @@ -1,30 +1,27 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable using System; -using System.IO; -using System.Runtime.Versioning; -using Duplicati.Library.Common; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; @@ -217,9 +214,7 @@ namespace Duplicati.Library.SQLiteHelper { // Check if SQLite database exists before opening a connection to it. // This information is used to 'fix' permissions on a newly created file. - var fileExists = false; - if (!OperatingSystem.IsWindows()) - fileExists = File.Exists(path); + var fileExists = SystemIO.IO_OS.FileExists(path); con.ConnectionString = "Data Source=" + path; con.Open(); @@ -230,29 +225,9 @@ namespace Duplicati.Library.SQLiteHelper sqlitecon.SetConfigurationOption(System.Data.SQLite.SQLiteConfigDbOpsEnum.SQLITE_DBCONFIG_DQS_DML, false); } - - // If we are non-Windows, make the file only accessible by the current user - if ((OperatingSystem.IsMacOS() || OperatingSystem.IsLinux()) && !fileExists) - SetUnixPermissionUserRWOnly(path); - } - - /// - /// Sets the unix permission user read-write Only. - /// - /// The file to set permissions on. - /// Make sure we do not inline this, as we might eventually load Mono.Posix, which is not present on Windows - [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] - [SupportedOSPlatform("linux")] - [SupportedOSPlatform("macOS")] - private static void SetUnixPermissionUserRWOnly(string path) - { - var fi = PosixFile.GetUserGroupAndPermissions(path); - PosixFile.SetUserGroupAndPermissions( - path, - fi.UID, - fi.GID, - 0x180 /* FilePermissions.S_IRUSR | FilePermissions.S_IWUSR*/ - ); + // Make the file only accessible by the current user if we just created it + if (!fileExists) + SystemIO.IO_OS.FileSetPermissionUserRWOnly(path); } /// From ebdfc774c28aef16876c04bcaa52d261ae28b476 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 6 Feb 2025 16:57:12 +0100 Subject: [PATCH 103/362] Removed static `IO_WIN` and `IO_SYS` as cross-platform calls are not possible. --- Duplicati/Library/Common/IO/SystemIO.cs | 67 ++++++---------- .../Library/Common/IO/VssBackupComponents.cs | 42 +++++----- Duplicati/Library/Snapshots/HyperVUtility.cs | 58 +++++++------- Duplicati/Library/Snapshots/LinuxSnapshot.cs | 62 +++++++-------- .../Library/Snapshots/NoSnapshotLinux.cs | 42 +++++----- .../Library/Snapshots/NoSnapshotWindows.cs | 79 ++++++++++--------- Duplicati/Library/Snapshots/USNJournal.cs | 42 +++++----- .../Library/Snapshots/WindowsSnapshot.cs | 73 +++++++++-------- Duplicati/UnitTest/IOTests.cs | 48 ++++++----- 9 files changed, 254 insertions(+), 259 deletions(-) diff --git a/Duplicati/Library/Common/IO/SystemIO.cs b/Duplicati/Library/Common/IO/SystemIO.cs index 8b3bcc027..268940aba 100644 --- a/Duplicati/Library/Common/IO/SystemIO.cs +++ b/Duplicati/Library/Common/IO/SystemIO.cs @@ -1,61 +1,44 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; -using System.Runtime.Versioning; namespace Duplicati.Library.Common.IO { public static class SystemIO { - - /// - /// A cached lookup for windows methods for dealing with long filenames - /// - [SupportedOSPlatform("windows")] - public static readonly ISystemIO IO_WIN; - - [SupportedOSPlatform("linux")] - [SupportedOSPlatform("macOS")] - public static readonly ISystemIO IO_SYS; - public static readonly ISystemIO IO_OS; static SystemIO() { - // TODO: These interfaces cannot be properly guarded by the supported platform attribute in this form. - // They are used in static methods of USNJournal on all platforms. - - // Since that is the case, the warnings will be suppressed with pragma. - -#pragma warning disable CA1416 - IO_WIN = new SystemIOWindows(); - IO_SYS = new SystemIOLinux(); -#pragma warning restore CA1416 if (OperatingSystem.IsWindows()) { - IO_OS = IO_WIN; + IO_OS = new SystemIOWindows(); } else if (OperatingSystem.IsMacOS() || OperatingSystem.IsLinux()) { - IO_OS = IO_SYS; + IO_OS = new SystemIOLinux(); + } + else + { + throw new PlatformNotSupportedException("The current platform is not supported"); } } } diff --git a/Duplicati/Library/Common/IO/VssBackupComponents.cs b/Duplicati/Library/Common/IO/VssBackupComponents.cs index 8b87aa044..98de80439 100644 --- a/Duplicati/Library/Common/IO/VssBackupComponents.cs +++ b/Duplicati/Library/Common/IO/VssBackupComponents.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -160,7 +160,7 @@ namespace Duplicati.Library.Common.IO } else { - var fileWithSpec = SystemIO.IO_WIN.PathCombine(file.Path, file.FileSpecification); + var fileWithSpec = SystemIO.IO_OS.PathCombine(file.Path, file.FileSpecification); if (File.Exists(fileWithSpec)) paths.Add(fileWithSpec); } @@ -222,7 +222,7 @@ namespace Duplicati.Library.Common.IO _volumes = new Dictionary(StringComparer.OrdinalIgnoreCase); foreach (var s in sources) { - var drive = SystemIO.IO_WIN.GetPathRoot(s); + var drive = SystemIO.IO_OS.GetPathRoot(s); if (!_volumes.ContainsKey(drive)) { if (!_vssBackupComponents.IsVolumeSupported(drive)) diff --git a/Duplicati/Library/Snapshots/HyperVUtility.cs b/Duplicati/Library/Snapshots/HyperVUtility.cs index c45ecab9d..4e7b268a6 100644 --- a/Duplicati/Library/Snapshots/HyperVUtility.cs +++ b/Duplicati/Library/Snapshots/HyperVUtility.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; @@ -24,7 +24,6 @@ using System.IO; using System.Linq; using System.Management; using System.Runtime.Versioning; -using Duplicati.Library.Common; using Duplicati.Library.Common.IO; namespace Duplicati.Library.Snapshots @@ -90,6 +89,11 @@ namespace Duplicati.Library.Snapshots /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(); + /// + /// The System.IO abstraction for the Windows platform + /// + private static readonly ISystemIO IO_WIN = SystemIO.IO_OS; + private readonly ManagementScope _wmiScope; private readonly string _vmIdField; private readonly string _wmiHost = "localhost"; @@ -251,7 +255,7 @@ namespace Duplicati.Library.Snapshots using (var mObject1 = new ManagementObjectSearcher(_wmiScope, new ObjectQuery(wmiQuery)).Get().Cast().First()) if (_wmiv2Namespace) { - path = SystemIO.IO_WIN.PathCombine((string)mObject1["ConfigurationDataRoot"], (string)mObject1["ConfigurationFile"]); + path = IO_WIN.PathCombine((string)mObject1["ConfigurationDataRoot"], (string)mObject1["ConfigurationFile"]); if (File.Exists(path)) result.Add(path); @@ -261,10 +265,10 @@ namespace Duplicati.Library.Snapshots { foreach (var snap in snaps) { - path = SystemIO.IO_WIN.PathCombine((string)snap["ConfigurationDataRoot"], (string)snap["ConfigurationFile"]); + path = IO_WIN.PathCombine((string)snap["ConfigurationDataRoot"], (string)snap["ConfigurationFile"]); if (File.Exists(path)) result.Add(path); - path = Util.AppendDirSeparator(SystemIO.IO_WIN.PathCombine((string)snap["ConfigurationDataRoot"], (string)snap["SuspendDataRoot"])); + path = Util.AppendDirSeparator(IO_WIN.PathCombine((string)snap["ConfigurationDataRoot"], (string)snap["SuspendDataRoot"])); if (Directory.Exists(path)) result.Add(path); } @@ -272,10 +276,10 @@ namespace Duplicati.Library.Snapshots } else { - path = SystemIO.IO_WIN.PathCombine((string)mObject1["ExternalDataRoot"], "Virtual Machines", vmID + ".xml"); + path = IO_WIN.PathCombine((string)mObject1["ExternalDataRoot"], "Virtual Machines", vmID + ".xml"); if (File.Exists(path)) result.Add(path); - path = Util.AppendDirSeparator(SystemIO.IO_WIN.PathCombine((string)mObject1["ExternalDataRoot"], "Virtual Machines", vmID)); + path = Util.AppendDirSeparator(IO_WIN.PathCombine((string)mObject1["ExternalDataRoot"], "Virtual Machines", vmID)); if (Directory.Exists(path)) result.Add(path); @@ -285,10 +289,10 @@ namespace Duplicati.Library.Snapshots foreach (var snapID in snapsIDs) { - path = SystemIO.IO_WIN.PathCombine((string)mObject1["SnapshotDataRoot"], "Snapshots", snapID.Replace("Microsoft:", "") + ".xml"); + path = IO_WIN.PathCombine((string)mObject1["SnapshotDataRoot"], "Snapshots", snapID.Replace("Microsoft:", "") + ".xml"); if (File.Exists(path)) result.Add(path); - path = Util.AppendDirSeparator(SystemIO.IO_WIN.PathCombine((string)mObject1["SnapshotDataRoot"], "Snapshots", snapID.Replace("Microsoft:", ""))); + path = Util.AppendDirSeparator(IO_WIN.PathCombine((string)mObject1["SnapshotDataRoot"], "Snapshots", snapID.Replace("Microsoft:", ""))); if (Directory.Exists(path)) result.Add(path); } diff --git a/Duplicati/Library/Snapshots/LinuxSnapshot.cs b/Duplicati/Library/Snapshots/LinuxSnapshot.cs index f49044f04..c7b62d1c3 100644 --- a/Duplicati/Library/Snapshots/LinuxSnapshot.cs +++ b/Duplicati/Library/Snapshots/LinuxSnapshot.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. @@ -40,7 +40,7 @@ namespace Duplicati.Library.Snapshots [SupportedOSPlatform("macOS")] public sealed class LinuxSnapshot : SnapshotBase { - /// + /// /// The tag used for logging messages /// public static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(WindowsSnapshot)); @@ -94,9 +94,9 @@ namespace Duplicati.Library.Snapshots { Dispose(); } - catch (Exception ex) + catch (Exception ex) { - Logging.Log.WriteVerboseMessage(LOGTAG, "SnapshotCleanupError", ex, "Failed to clean up after error"); + Logging.Log.WriteVerboseMessage(LOGTAG, "SnapshotCleanupError", ex, "Failed to clean up after error"); } throw; @@ -111,10 +111,10 @@ namespace Duplicati.Library.Snapshots if (disposing) { // Attempt to clean out as many as possible - foreach(var s in m_snapShots) + foreach (var s in m_snapShots) { try { s.Dispose(); } - catch (Exception ex) { Logging.Log.WriteVerboseMessage(LOGTAG, "SnapshotCloseError", ex, "Failed to close a snapshot"); } + catch (Exception ex) { Logging.Log.WriteVerboseMessage(LOGTAG, "SnapshotCloseError", ex, "Failed to close a snapshot"); } } } @@ -466,7 +466,7 @@ namespace Duplicati.Library.Snapshots /// The symlink target public override string GetSymlinkTarget(string localPath) { - return SystemIO.IO_SYS.GetSymlinkTarget(ConvertToSnapshotPath(localPath)); + return SystemIO.IO_OS.GetSymlinkTarget(ConvertToSnapshotPath(localPath)); } /// @@ -478,7 +478,7 @@ namespace Duplicati.Library.Snapshots /// A flag indicating if a symlink should be followed public override Dictionary GetMetadata(string localPath, bool isSymlink, bool followSymlink) { - return SystemIO.IO_SYS.GetMetadata(ConvertToSnapshotPath(localPath), isSymlink, followSymlink); + return SystemIO.IO_OS.GetMetadata(ConvertToSnapshotPath(localPath), isSymlink, followSymlink); } /// @@ -500,8 +500,8 @@ namespace Duplicati.Library.Snapshots default: return true; } - } - catch + } + catch { if (!System.IO.File.Exists(SystemIOLinux.NormalizePath(localPath))) return false; @@ -509,7 +509,7 @@ namespace Duplicati.Library.Snapshots throw; } } - + /// /// Gets a unique hardlink target ID /// @@ -518,10 +518,10 @@ namespace Duplicati.Library.Snapshots public override string HardlinkTargetID(string localPath) { var snapshotPath = ConvertToSnapshotPath(localPath); - + if (PosixFile.GetHardlinkCount(snapshotPath) <= 1) return null; - + return PosixFile.GetInodeTargetID(snapshotPath); } diff --git a/Duplicati/Library/Snapshots/NoSnapshotLinux.cs b/Duplicati/Library/Snapshots/NoSnapshotLinux.cs index f1601169b..53d7070f9 100644 --- a/Duplicati/Library/Snapshots/NoSnapshotLinux.cs +++ b/Duplicati/Library/Snapshots/NoSnapshotLinux.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -82,7 +82,7 @@ namespace Duplicati.Library.Snapshots /// The symlink target public override string GetSymlinkTarget(string localPath) { - return SystemIO.IO_SYS.GetSymlinkTarget(localPath); + return SystemIO.IO_OS.GetSymlinkTarget(localPath); } /// @@ -94,7 +94,7 @@ namespace Duplicati.Library.Snapshots /// A flag indicating if a symlink should be followed public override Dictionary GetMetadata(string localPath, bool isSymlink, bool followSymlink) { - return SystemIO.IO_SYS.GetMetadata(localPath, isSymlink, followSymlink); + return SystemIO.IO_OS.GetMetadata(localPath, isSymlink, followSymlink); } /// diff --git a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs index f3a3c7184..882978056 100644 --- a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs +++ b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -33,6 +33,11 @@ namespace Duplicati.Library.Snapshots [SupportedOSPlatform("windows")] public sealed class NoSnapshotWindows : SnapshotBase { + /// + /// The system IO implementation for Windows + /// + private static readonly ISystemIO IO_WIN = SystemIO.IO_OS; + /// /// Returns the symlink target if the entry is a symlink, and null otherwise /// @@ -40,7 +45,7 @@ namespace Duplicati.Library.Snapshots /// The symlink target public override string GetSymlinkTarget(string localPath) { - return SystemIO.IO_WIN.GetSymlinkTarget(localPath); + return IO_WIN.GetSymlinkTarget(localPath); } /// @@ -48,9 +53,9 @@ namespace Duplicati.Library.Snapshots /// /// The file attributes /// The file or folder to examine - public override System.IO.FileAttributes GetAttributes (string localPath) + public override System.IO.FileAttributes GetAttributes(string localPath) { - return SystemIO.IO_WIN.GetFileAttributes(localPath); + return IO_WIN.GetFileAttributes(localPath); } /// @@ -58,9 +63,9 @@ namespace Duplicati.Library.Snapshots /// /// The full path to the file in non-snapshot format /// The length of the file - public override long GetFileSize (string localPath) + public override long GetFileSize(string localPath) { - return SystemIO.IO_WIN.FileLength(localPath); + return IO_WIN.FileLength(localPath); } /// @@ -80,9 +85,9 @@ namespace Duplicati.Library.Snapshots /// /// The full path to the file in non-snapshot format /// The last write time of the file - public override DateTime GetLastWriteTimeUtc (string localPath) + public override DateTime GetLastWriteTimeUtc(string localPath) { - return SystemIO.IO_WIN.FileGetLastWriteTimeUtc(localPath); + return IO_WIN.FileGetLastWriteTimeUtc(localPath); } /// @@ -90,9 +95,9 @@ namespace Duplicati.Library.Snapshots /// /// The full path to the file in non-snapshot format /// The last write time of the file - public override DateTime GetCreationTimeUtc (string localPath) + public override DateTime GetCreationTimeUtc(string localPath) { - return SystemIO.IO_WIN.FileGetCreationTimeUtc(localPath); + return IO_WIN.FileGetCreationTimeUtc(localPath); } /// @@ -100,9 +105,9 @@ namespace Duplicati.Library.Snapshots /// /// The full path to the file in non-snapshot format /// An open filestream that can be read - public override System.IO.Stream OpenRead (string localPath) + public override System.IO.Stream OpenRead(string localPath) { - return SystemIO.IO_WIN.FileOpenRead(localPath); + return IO_WIN.FileOpenRead(localPath); } /// @@ -110,25 +115,25 @@ namespace Duplicati.Library.Snapshots /// /// All folders found in the folder /// The folder to examinate - protected override string[] ListFiles (string localFolderPath) + protected override string[] ListFiles(string localFolderPath) { - string[] tmp = SystemIO.IO_WIN.GetFiles(localFolderPath); + string[] tmp = IO_WIN.GetFiles(localFolderPath); string[] res = new string[tmp.Length]; - for(int i = 0; i < tmp.Length; i++) + for (int i = 0; i < tmp.Length; i++) res[i] = SystemIOWindows.RemoveExtendedDevicePathPrefix(tmp[i]); return res; } - + /// /// Lists all folders in the given folder /// /// All folders found in the folder /// The folder to examinate - protected override string[] ListFolders (string localFolderPath) + protected override string[] ListFolders(string localFolderPath) { - string[] tmp = SystemIO.IO_WIN.GetDirectories(SystemIOWindows.AddExtendedDevicePathPrefix(localFolderPath)); + string[] tmp = IO_WIN.GetDirectories(SystemIOWindows.AddExtendedDevicePathPrefix(localFolderPath)); string[] res = new string[tmp.Length]; for (int i = 0; i < tmp.Length; i++) res[i] = SystemIOWindows.RemoveExtendedDevicePathPrefix(tmp[i]); @@ -145,7 +150,7 @@ namespace Duplicati.Library.Snapshots /// A flag indicating if a symlink should be followed public override Dictionary GetMetadata(string localPath, bool isSymlink, bool followSymlink) { - return SystemIO.IO_WIN.GetMetadata(localPath, isSymlink, followSymlink); + return IO_WIN.GetMetadata(localPath, isSymlink, followSymlink); } /// diff --git a/Duplicati/Library/Snapshots/USNJournal.cs b/Duplicati/Library/Snapshots/USNJournal.cs index 199a4d745..2608395d5 100644 --- a/Duplicati/Library/Snapshots/USNJournal.cs +++ b/Duplicati/Library/Snapshots/USNJournal.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. @@ -178,7 +178,7 @@ namespace Duplicati.Library.Snapshots if (path == null) throw new Exception(Strings.USNHelper.UnexpectedPathFormat); - return SystemIO.IO_WIN.GetPathRoot(path); + return SystemIO.IO_OS.GetPathRoot(path); } public static string GetDeviceNameFromPath(string path) @@ -509,7 +509,7 @@ namespace Duplicati.Library.Snapshots var path = m_volume; foreach (var r in pathList) { - path = SystemIO.IO_WIN.PathCombine(path, r.FileName); + path = SystemIO.IO_OS.PathCombine(path, r.FileName); } if (rec.UsnRecord.FileAttributes.HasFlag(Win32USN.FileAttributes.Directory)) diff --git a/Duplicati/Library/Snapshots/WindowsSnapshot.cs b/Duplicati/Library/Snapshots/WindowsSnapshot.cs index 65df03a2c..61e8479d4 100644 --- a/Duplicati/Library/Snapshots/WindowsSnapshot.cs +++ b/Duplicati/Library/Snapshots/WindowsSnapshot.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. @@ -44,6 +44,11 @@ namespace Duplicati.Library.Snapshots /// public static readonly string LOGTAG = Logging.Log.LogTagFromType(); + /// + /// The system IO for the current platform + /// + private static readonly ISystemIO IO_WIN = SystemIO.IO_OS; + /// /// The main reference to the backup controller /// @@ -115,8 +120,8 @@ namespace Duplicati.Library.Snapshots { string[] tmp = null; var spath = ConvertToSnapshotPath(localFolderPath); - tmp = SystemIO.IO_WIN.GetDirectories(spath); - var root = Util.AppendDirSeparator(SystemIO.IO_WIN.GetPathRoot(localFolderPath)); + tmp = IO_WIN.GetDirectories(spath); + var root = Util.AppendDirSeparator(IO_WIN.GetPathRoot(localFolderPath)); var volumePath = Util.AppendDirSeparator(ConvertToSnapshotPath(root)); volumePath = SystemIOWindows.AddExtendedDevicePathPrefix(volumePath); @@ -140,10 +145,10 @@ namespace Duplicati.Library.Snapshots string[] files = null; var spath = ConvertToSnapshotPath(localFolderPath); - files = SystemIO.IO_WIN.GetFiles(spath); + files = IO_WIN.GetFiles(spath); // convert back to non-shadow, i.e., non-vss version - var root = Util.AppendDirSeparator(SystemIO.IO_WIN.GetPathRoot(localFolderPath)); + var root = Util.AppendDirSeparator(IO_WIN.GetPathRoot(localFolderPath)); var volumePath = Util.AppendDirSeparator(ConvertToSnapshotPath(root)); volumePath = SystemIOWindows.AddExtendedDevicePathPrefix(volumePath); @@ -179,7 +184,7 @@ namespace Duplicati.Library.Snapshots { var spath = ConvertToSnapshotPath(localPath); - return SystemIO.IO_WIN.GetLastWriteTimeUtc(SystemIOWindows.AddExtendedDevicePathPrefix(spath)); + return IO_WIN.GetLastWriteTimeUtc(SystemIOWindows.AddExtendedDevicePathPrefix(spath)); } /// @@ -191,7 +196,7 @@ namespace Duplicati.Library.Snapshots { var spath = ConvertToSnapshotPath(localPath); - return SystemIO.IO_WIN.GetCreationTimeUtc(SystemIOWindows.AddExtendedDevicePathPrefix(spath)); + return IO_WIN.GetCreationTimeUtc(SystemIOWindows.AddExtendedDevicePathPrefix(spath)); } /// @@ -201,7 +206,7 @@ namespace Duplicati.Library.Snapshots /// An open filestream that can be read public override Stream OpenRead(string localPath) { - return SystemIO.IO_WIN.FileOpenRead(ConvertToSnapshotPath(localPath)); + return IO_WIN.FileOpenRead(ConvertToSnapshotPath(localPath)); } /// @@ -211,7 +216,7 @@ namespace Duplicati.Library.Snapshots /// The length of the file public override long GetFileSize(string localPath) { - return SystemIO.IO_WIN.FileLength(ConvertToSnapshotPath(localPath)); + return IO_WIN.FileLength(ConvertToSnapshotPath(localPath)); } /// @@ -221,7 +226,7 @@ namespace Duplicati.Library.Snapshots /// The file or folder to examine public override FileAttributes GetAttributes(string localPath) { - return SystemIO.IO_WIN.GetFileAttributes(ConvertToSnapshotPath(localPath)); + return IO_WIN.GetFileAttributes(ConvertToSnapshotPath(localPath)); } /// @@ -232,7 +237,7 @@ namespace Duplicati.Library.Snapshots public override string GetSymlinkTarget(string localPath) { var spath = ConvertToSnapshotPath(localPath); - return SystemIO.IO_WIN.GetSymlinkTarget(spath); + return IO_WIN.GetSymlinkTarget(spath); } /// @@ -244,7 +249,7 @@ namespace Duplicati.Library.Snapshots /// A flag indicating if a symlink should be followed public override Dictionary GetMetadata(string localPath, bool isSymlink, bool followSymlink) { - return SystemIO.IO_WIN.GetMetadata(ConvertToSnapshotPath(localPath), isSymlink, followSymlink); + return IO_WIN.GetMetadata(ConvertToSnapshotPath(localPath), isSymlink, followSymlink); } /// @@ -268,7 +273,7 @@ namespace Duplicati.Library.Snapshots foreach (var kvp in _vssBackupComponents.SnapshotDeviceAndVolumes) { if (snapshotPath.StartsWith(kvp.Key, Utility.Utility.ClientFilenameStringComparison)) - return SystemIO.IO_WIN.PathCombine(kvp.Value, snapshotPath.Substring(kvp.Key.Length)); + return IO_WIN.PathCombine(kvp.Value, snapshotPath.Substring(kvp.Key.Length)); } throw new InvalidOperationException(); @@ -283,7 +288,7 @@ namespace Duplicati.Library.Snapshots if (!Path.IsPathRooted(localPath)) throw new InvalidOperationException(); - var root = SystemIO.IO_WIN.GetPathRoot(localPath); + var root = IO_WIN.GetPathRoot(localPath); var volumePath = _vssBackupComponents.GetVolumeFromCache(root); // Note that using a simple Path.Combine() for the following code @@ -303,13 +308,13 @@ namespace Duplicati.Library.Snapshots /// public override bool FileExists(string localFilePath) { - return SystemIO.IO_WIN.FileExists(ConvertToSnapshotPath(localFilePath)); + return IO_WIN.FileExists(ConvertToSnapshotPath(localFilePath)); } /// public override bool DirectoryExists(string localFolderPath) { - return SystemIO.IO_WIN.DirectoryExists(ConvertToSnapshotPath(localFolderPath)); + return IO_WIN.DirectoryExists(ConvertToSnapshotPath(localFolderPath)); } /// diff --git a/Duplicati/UnitTest/IOTests.cs b/Duplicati/UnitTest/IOTests.cs index bc0021511..0be6d3de7 100644 --- a/Duplicati/UnitTest/IOTests.cs +++ b/Duplicati/UnitTest/IOTests.cs @@ -1,30 +1,28 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System.Text; using NUnit.Framework; using Duplicati.Library.Common.IO; -using Duplicati.Library.Utility; -using Duplicati.Library.Common; using System.Collections.Generic; using System; @@ -62,13 +60,13 @@ namespace Duplicati.UnitTest var filePath = root + filename; var filePathWithExtendedDevicePathPrefix = SystemIOWindows.AddExtendedDevicePathPrefix(filePath); - var filePathWithExtendedDevicePathPrefixRoot = SystemIO.IO_WIN.GetPathRoot(filePathWithExtendedDevicePathPrefix); + var filePathWithExtendedDevicePathPrefixRoot = SystemIO.IO_OS.GetPathRoot(filePathWithExtendedDevicePathPrefix); //Prefixed with extended device path prefix remains prefixed Assert.AreEqual(SystemIOWindows.AddExtendedDevicePathPrefix(root), filePathWithExtendedDevicePathPrefixRoot); //Without extended device path prefix, no prefix. - var filePathRoot = SystemIO.IO_WIN.GetPathRoot(filePath); + var filePathRoot = SystemIO.IO_OS.GetPathRoot(filePath); Assert.AreEqual(root, filePathRoot); } @@ -282,7 +280,7 @@ namespace Duplicati.UnitTest }; foreach (var path in testCasesWherePathGetFullGivesSameResultsAsDotNet) { - var actual = SystemIO.IO_WIN.PathGetFullPath(path); + var actual = SystemIO.IO_OS.PathGetFullPath(path); var expected = System.IO.Path.GetFullPath(path); Assert.AreEqual(expected, actual, $"Path: {path}"); } @@ -313,7 +311,7 @@ namespace Duplicati.UnitTest foreach (var keyValuePair in testCasesWherePathGetFullGivesDifferentResultsThanDotNet) { var path = keyValuePair.Key; - var actual = SystemIO.IO_WIN.PathGetFullPath(path); + var actual = SystemIO.IO_OS.PathGetFullPath(path); var expected = keyValuePair.Value; Assert.AreEqual(expected, actual, $"Path: {path}"); } From 1af6e2b1d6bca19f4cc55f67857f2a45d23a825c Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 6 Feb 2025 16:57:44 +0100 Subject: [PATCH 104/362] Experiment with CommonProgramData on Windows in favor of users folder. --- Duplicati/Library/AutoUpdater/DataFolderLocator.cs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Duplicati/Library/AutoUpdater/DataFolderLocator.cs b/Duplicati/Library/AutoUpdater/DataFolderLocator.cs index ff5e0f2e9..d88805514 100644 --- a/Duplicati/Library/AutoUpdater/DataFolderLocator.cs +++ b/Duplicati/Library/AutoUpdater/DataFolderLocator.cs @@ -92,11 +92,7 @@ public static class DataFolderLocator // If %LOCALAPPDATA% is inside the Windows folder, prefer a LocalService folder instead if (Common.IO.Util.IsPathUnderWindowsFolder(newlocation)) - { - var userProfilesFolder = Library.Utility.SHGetFolder.UserProfilesFolder; - if (!string.IsNullOrWhiteSpace(userProfilesFolder)) - folderOrder.Add(System.IO.Path.Combine(userProfilesFolder, "LocalService", appName)); - } + folderOrder.Add(System.IO.Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), appName)); // Prefer the most recent location var matches = folderOrder.AsEnumerable() From f0f79418d8289b156177fe4bfafc06274a4a128b Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 6 Feb 2025 17:09:18 +0100 Subject: [PATCH 105/362] Fixed logic so the permissions are always applied, unless the user specifically opts out by placing a file named `insecure-permissions.txt` in the data folder. --- Duplicati/Library/AutoUpdater/DataFolderLocator.cs | 8 ++++++-- Duplicati/Library/AutoUpdater/DataFolderManager.cs | 7 ++++++- Duplicati/Library/Common/IO/Util.cs | 8 ++++++++ Duplicati/Library/SQLiteHelper/SQLiteLoader.cs | 11 +++++++++-- 4 files changed, 29 insertions(+), 5 deletions(-) diff --git a/Duplicati/Library/AutoUpdater/DataFolderLocator.cs b/Duplicati/Library/AutoUpdater/DataFolderLocator.cs index d88805514..1aafa1d16 100644 --- a/Duplicati/Library/AutoUpdater/DataFolderLocator.cs +++ b/Duplicati/Library/AutoUpdater/DataFolderLocator.cs @@ -49,8 +49,12 @@ public static class DataFolderLocator ? DataFolderManager.DATAFOLDER : GetDefaultStorageFolderInternal(targetfilename, AutoUpdateSettings.AppName); - // If the folder does not exist, and we are allowed to create it, we do so - if (autoCreate && !SystemIO.IO_OS.DirectoryExists(folder)) + if (SystemIO.IO_OS.DirectoryExists(folder)) + { + if (!SystemIO.IO_OS.FileExists(System.IO.Path.Combine(folder, Util.InsecurePermissionsMarkerFile))) + SystemIO.IO_OS.DirectorySetPermissionUserRWOnly(folder); + } + else if (autoCreate) { // Create the folder SystemIO.IO_OS.DirectoryCreate(folder); diff --git a/Duplicati/Library/AutoUpdater/DataFolderManager.cs b/Duplicati/Library/AutoUpdater/DataFolderManager.cs index edbaa32d7..36c1f79e8 100644 --- a/Duplicati/Library/AutoUpdater/DataFolderManager.cs +++ b/Duplicati/Library/AutoUpdater/DataFolderManager.cs @@ -181,7 +181,12 @@ public static class DataFolderManager DATAFOLDER = Util.AppendDirSeparator(DataFolderLocator.GetDefaultStorageFolderInternal(SERVER_DATABASE_FILENAME, APPNAME)); } - if (!Directory.Exists(DATAFOLDER)) + if (Directory.Exists(DATAFOLDER)) + { + if (!File.Exists(Path.Combine(DATAFOLDER, Util.InsecurePermissionsMarkerFile))) + SystemIO.IO_OS.DirectorySetPermissionUserRWOnly(DATAFOLDER); + } + else { Directory.CreateDirectory(DATAFOLDER); SystemIO.IO_OS.DirectorySetPermissionUserRWOnly(DATAFOLDER); diff --git a/Duplicati/Library/Common/IO/Util.cs b/Duplicati/Library/Common/IO/Util.cs index 381648a15..80b247cf6 100644 --- a/Duplicati/Library/Common/IO/Util.cs +++ b/Duplicati/Library/Common/IO/Util.cs @@ -31,8 +31,16 @@ namespace Duplicati.Library.Common.IO /// public static readonly string DirectorySeparatorString = Path.DirectorySeparatorChar.ToString(); + /// + /// A cached instance of the alternate directory separator as a string + /// public static readonly string AltDirectorySeparatorString = Path.AltDirectorySeparatorChar.ToString(); + /// + /// Filename of a marker file that can be put inside the data folder to prevent Duplicati from fixing lax permissions + /// + public const string InsecurePermissionsMarkerFile = "insecure-permissions.txt"; + /// /// Appends the appropriate directory separator to paths, depending on OS. /// Does not append the separator if the path already ends with it. diff --git a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs index 754a49b3b..43f8d11be 100644 --- a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs +++ b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs @@ -225,9 +225,16 @@ namespace Duplicati.Library.SQLiteHelper sqlitecon.SetConfigurationOption(System.Data.SQLite.SQLiteConfigDbOpsEnum.SQLITE_DBCONFIG_DQS_DML, false); } - // Make the file only accessible by the current user if we just created it - if (!fileExists) + // Make the file only accessible by the current user + if (fileExists) + { + if (!SystemIO.IO_OS.FileExists(SystemIO.IO_OS.PathCombine(SystemIO.IO_OS.PathGetDirectoryName(path), Util.InsecurePermissionsMarkerFile))) + SystemIO.IO_OS.FileSetPermissionUserRWOnly(path); + } + else + { SystemIO.IO_OS.FileSetPermissionUserRWOnly(path); + } } /// From a68dba607e1381fcf9ae05e2fa763cedfd66e797 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 6 Feb 2025 17:21:17 +0100 Subject: [PATCH 106/362] Added support for using the current uid/gid instead of using the file uid/gid --- Duplicati/Library/Common/IO/SystemIOLinux.cs | 37 ++++++++++++++++---- 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/Duplicati/Library/Common/IO/SystemIOLinux.cs b/Duplicati/Library/Common/IO/SystemIOLinux.cs index 1bce31daf..fea6e8845 100644 --- a/Duplicati/Library/Common/IO/SystemIOLinux.cs +++ b/Duplicati/Library/Common/IO/SystemIOLinux.cs @@ -25,6 +25,7 @@ using System.Linq; using System.Collections.Generic; using Duplicati.Library.Interface; using System.Runtime.Versioning; +using System.Runtime.InteropServices; namespace Duplicati.Library.Common.IO { @@ -32,6 +33,26 @@ namespace Duplicati.Library.Common.IO [SupportedOSPlatform("macOS")] public struct SystemIOLinux : ISystemIO { + /// + /// PInvoke methods + /// + private static class PInvoke + { + /// + /// Gets the user ID of the current user + /// + /// The user ID + [DllImport("libc")] + public static extern uint getuid(); + + /// + /// Gets the group ID of the current user + /// + /// + [DllImport("libc")] + public static extern uint getgid(); + } + #region ISystemIO implementation public void DirectoryCreate(string path) @@ -348,17 +369,22 @@ namespace Duplicati.Library.Common.IO return new FileEntry(fileInfo.Name, fileInfo.Length, fileInfo.LastAccessTime, fileInfo.LastWriteTime); } + private static (long uid, long gid) GetOwnerAndGroup(string path) + { + var fi = PosixFile.GetUserGroupAndPermissions(path); + return (fi.UID, fi.GID); + } + /// /// Sets the unix permission user read-write Only. /// /// The file to set permissions on. public void FileSetPermissionUserRWOnly(string path) { - var fi = PosixFile.GetUserGroupAndPermissions(path); PosixFile.SetUserGroupAndPermissions( path, - fi.UID, - fi.GID, + PInvoke.getuid(), + PInvoke.getgid(), Convert.ToInt32("600", 8) /* FilePermissions.S_IRUSR | FilePermissions.S_IWUSR*/ ); } @@ -369,11 +395,10 @@ namespace Duplicati.Library.Common.IO /// The directory to set permissions on. public void DirectorySetPermissionUserRWOnly(string path) { - var fi = PosixFile.GetUserGroupAndPermissions(path); PosixFile.SetUserGroupAndPermissions( path, - fi.UID, - fi.GID, + PInvoke.getuid(), + PInvoke.getgid(), Convert.ToInt32("700", 8) /* FilePermissions.S_IRUSR | FilePermissions.S_IWUSR | FilePermissions.S_IXUSR */ ); } From 536e62275f774516a0cb8ec129dd6c6e2bda5146 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 6 Feb 2025 21:23:07 +0100 Subject: [PATCH 107/362] Changed the logic in recreate to avoid opening the same zip file multiple times. --- .../Main/Operation/RecreateDatabaseHandler.cs | 12 +++- .../Library/Main/Volumes/VolumeReaderBase.cs | 59 ++++++++++++------- 2 files changed, 47 insertions(+), 24 deletions(-) diff --git a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs index ebfc4f2e8..1cbbfabcc 100644 --- a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs +++ b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs @@ -20,6 +20,7 @@ // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -239,9 +240,14 @@ namespace Duplicati.Library.Main.Operation var parsed = VolumeBase.ParseFilename(entry.Name); + using var stream = new FileStream(tmpfile, FileMode.Open, FileAccess.Read, FileShare.Read); + using var compressor = DynamicLoader.CompressionLoader.GetModule(parsed.CompressionModule, stream, ArchiveMode.Read, m_options.RawOptions); + if (compressor == null) + throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule"); + if (!hasUpdatedOptions) { - VolumeReaderBase.UpdateOptionsFromManifest(parsed.CompressionModule, tmpfile, m_options); + VolumeReaderBase.UpdateOptionsFromManifest(compressor, m_options); hasUpdatedOptions = true; // Recompute the cached sizes blocksize = m_options.Blocksize; @@ -252,12 +258,12 @@ namespace Duplicati.Library.Main.Operation var filesetid = restoredb.CreateFileset(volumeIds[entry.Name], parsed.Time, tr); // retrieve fileset data from dlist - var filesetData = VolumeReaderBase.GetFilesetData(parsed.CompressionModule, tmpfile, m_options); + var filesetData = VolumeReaderBase.GetFilesetData(compressor, m_options); // update fileset using filesetData restoredb.UpdateFullBackupStateInFileset(filesetid, filesetData.IsFullBackup); - using (var filelistreader = new FilesetVolumeReader(parsed.CompressionModule, tmpfile, m_options)) + using (var filelistreader = new FilesetVolumeReader(compressor, m_options)) foreach (var fe in filelistreader.Files.Where(x => Library.Utility.FilterExpression.Matches(filter, x.Path))) { try diff --git a/Duplicati/Library/Main/Volumes/VolumeReaderBase.cs b/Duplicati/Library/Main/Volumes/VolumeReaderBase.cs index fa2b8dffd..fffab0150 100644 --- a/Duplicati/Library/Main/Volumes/VolumeReaderBase.cs +++ b/Duplicati/Library/Main/Volumes/VolumeReaderBase.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Interface; using Duplicati.Library.Utility; @@ -116,7 +116,12 @@ namespace Duplicati.Library.Main.Volumes { using (var stream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read)) using (var c = LoadCompressor(compressor, stream, options)) - using (var s = c.OpenRead(FILESET_FILENAME)) + return GetFilesetData(c, options); + } + + public static FilesetData GetFilesetData(ICompression compressor, Options options) + { + using (var s = compressor.OpenRead(FILESET_FILENAME)) { if (s == null) { @@ -140,7 +145,19 @@ namespace Duplicati.Library.Main.Volumes { using (var stream = new System.IO.FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read)) using (var c = LoadCompressor(compressor, stream, options)) - using (var s = c.OpenRead(MANIFEST_FILENAME)) + UpdateOptionsFromManifest(c, options); + + } + + /// + /// Updates the options with data from the manifest file, but does not overwrite existing values + /// + /// The compressor to use + /// The file to read the manifest from + /// The options to update + public static void UpdateOptionsFromManifest(ICompression compressor, Options options) + { + using (var s = compressor.OpenRead(MANIFEST_FILENAME)) using (var fs = new StreamReader(s, ENCODING)) { var d = JsonConvert.DeserializeObject(fs.ReadToEnd()); From adbd6b6a7c0c87b97201d5def4d98b80cec329fb Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 6 Feb 2025 21:35:49 +0100 Subject: [PATCH 108/362] Added support for fallback when reading zip files with unsupported compression methods --- .../ZipCompression/FileArchiveZip.cs | 49 ++++++++++- Duplicati/UnitTest/ZipFallbackTest.cs | 82 +++++++++++++++++++ 2 files changed, 129 insertions(+), 2 deletions(-) create mode 100644 Duplicati/UnitTest/ZipFallbackTest.cs diff --git a/Duplicati/Library/Compression/ZipCompression/FileArchiveZip.cs b/Duplicati/Library/Compression/ZipCompression/FileArchiveZip.cs index c59967dc3..e2b8a465d 100644 --- a/Duplicati/Library/Compression/ZipCompression/FileArchiveZip.cs +++ b/Duplicati/Library/Compression/ZipCompression/FileArchiveZip.cs @@ -103,6 +103,16 @@ namespace Duplicati.Library.Compression.ZipCompression /// private readonly IZipArchive m_archive; + /// + /// The fallback archive to use, if any + /// + private IZipArchive? m_fallbackArchive = null; + + /// + /// The parameters used to create the archive + /// + private readonly (Stream Stream, ParsedZipOptions Options, CompressionLibrary Library) m_creationParams; + /// /// Default constructor, used to read file extension and supported commands /// @@ -142,6 +152,7 @@ namespace Duplicati.Library.Compression.ZipCompression var unittestMode = Utility.Utility.ParseBoolOption(options.AsReadOnly(), "unittest-mode"); var parsedOptions = new ParsedZipOptions(compressionLevel, compressionType, usingZip64, unittestMode); + var userCompressionLibrary = compressionLibrary; if (compressionLibrary == CompressionLibrary.Auto) { if (compressionType != CompressionType.Deflate || usingZip64) @@ -165,6 +176,7 @@ namespace Duplicati.Library.Compression.ZipCompression } m_archive = archive ?? new SharpCompressZipArchive(stream, mode, parsedOptions); + m_creationParams = (stream, parsedOptions, userCompressionLibrary); } #region IFileArchive Members @@ -219,7 +231,9 @@ namespace Duplicati.Library.Compression.ZipCompression public void Dispose() { - m_archive.Dispose(); + if (m_fallbackArchive != null) + m_archive.Dispose(); + m_fallbackArchive?.Dispose(); } public string[] ListFiles(string prefix) @@ -229,7 +243,38 @@ namespace Duplicati.Library.Compression.ZipCompression => m_archive.ListFilesWithSize(prefix); public Stream? OpenRead(string file) - => m_archive.OpenRead(file); + { + // If we have started the fallback archive, we should continue using it + if (m_fallbackArchive != null) + return m_fallbackArchive.OpenRead(file); + + try + { + return m_archive.OpenRead(file); + } + catch (Exception ex) + { + if (m_creationParams.Library == CompressionLibrary.Auto && m_archive is not SharpCompressZipArchive && m_creationParams.Stream.CanSeek) + { + Log.WriteWarningMessage(LOGTAG, "CompressionReadErrorFallback", ex, "Failed to open file with built-in ZIP archive, falling back to SharpCompress"); + + try + { + m_creationParams.Item1.Seek(0, SeekOrigin.Begin); + m_archive.Dispose(); + } + catch + { + + } + + m_fallbackArchive = new SharpCompressZipArchive(m_creationParams.Stream, ArchiveMode.Read, m_creationParams.Options); + return m_fallbackArchive.OpenRead(file); + } + + throw; + } + } public DateTime GetLastWriteTime(string file) => m_archive.GetLastWriteTime(file); diff --git a/Duplicati/UnitTest/ZipFallbackTest.cs b/Duplicati/UnitTest/ZipFallbackTest.cs new file mode 100644 index 000000000..8607d7192 --- /dev/null +++ b/Duplicati/UnitTest/ZipFallbackTest.cs @@ -0,0 +1,82 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using System; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using Duplicati.Library.Interface; +using NUnit.Framework; + +namespace Duplicati.UnitTest +{ + public class ZipFallbackTest : BasicSetupHelper + { + [Test] + [Category("Targeted")] + public void RunCommands() + { + var testopts = TestOptions; + testopts["zip-compression-method"] = "lzma"; + testopts["zip-compression-library"] = "SharpCompress"; + testopts["blocksize"] = "50kb"; + + var data = new byte[1024 * 1024 * 10]; + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + var r = c.Backup(new string[] { DATAFOLDER }); + Assert.AreEqual(0, r.Errors.Count()); + Assert.AreEqual(0, r.Warnings.Count()); + } + + // Delete the local database + File.Delete(DBFILE); + + // Switch back to built-in compression + testopts.Remove("zip-compression-method"); + testopts["zip-compression-library"] = "Auto"; + + // Recreate the database + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + var r = c.Repair(); + Assert.AreEqual(0, r.Errors.Count()); + // Ensure that we get warnings for the fallback, one for the dlist and one for the dindex + Assert.AreEqual(2, r.Warnings.Count()); + Assert.That(r.Warnings.All(x => x.Contains("lzma", StringComparison.OrdinalIgnoreCase))); + } + + // Restore files + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { restore_path = RESTOREFOLDER }), null)) + { + var r = c.Restore(null); + Assert.AreEqual(0, r.Errors.Count()); + // Ensure that we get warnings for the fallback, one for the dblock file + Assert.AreEqual(1, r.Warnings.Count()); + Assert.That(r.Warnings.All(x => x.Contains("lzma", StringComparison.OrdinalIgnoreCase))); + } + + TestUtils.AssertDirectoryTreesAreEquivalent(DATAFOLDER, RESTOREFOLDER, true, "Restore"); + + } + } +} From f7536df9c28832bdb60f6fd5bd3c8f5f33596092 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 6 Feb 2025 21:47:28 +0100 Subject: [PATCH 109/362] Added explicit access for the `LocalSystem` user as suggested. --- .../Library/Common/IO/SystemIOWindows.cs | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/Duplicati/Library/Common/IO/SystemIOWindows.cs b/Duplicati/Library/Common/IO/SystemIOWindows.cs index 528cb8cfc..3566de45a 100644 --- a/Duplicati/Library/Common/IO/SystemIOWindows.cs +++ b/Duplicati/Library/Common/IO/SystemIOWindows.cs @@ -44,8 +44,16 @@ namespace Duplicati.Library.Common.IO private static readonly string DIRSEP = Util.DirectorySeparatorString; + /// + /// The current user name + /// private static readonly string CURRENT_USERNAME = WindowsIdentity.GetCurrent().Name; + /// + /// The LocalSystem user name + /// + private static readonly string LOCAL_SYSTEM_NAME = new SecurityIdentifier(WellKnownSidType.LocalSystemSid, null).Translate(typeof(NTAccount)).Value; + /// /// Prefix path with one of the extended device path prefixes /// (@"\\?\" or @"\\?\UNC\") but only if it's a fully qualified @@ -695,6 +703,12 @@ namespace Duplicati.Library.Common.IO AccessControlType.Allow )); + security.AddAccessRule(new FileSystemAccessRule( + LOCAL_SYSTEM_NAME, + FileSystemRights.FullControl, + AccessControlType.Allow + )); + // Adjust with the new security settings new FileInfo(path).SetAccessControl(security); } @@ -718,6 +732,12 @@ namespace Duplicati.Library.Common.IO AccessControlType.Allow )); + security.AddAccessRule(new FileSystemAccessRule( + LOCAL_SYSTEM_NAME, + FileSystemRights.FullControl, + AccessControlType.Allow + )); + // Adjust with the new security settings new DirectoryInfo(path).SetAccessControl(security); } From 3e72768eeca3cba12646e5d018e2425d041b6848 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 6 Feb 2025 21:53:51 +0100 Subject: [PATCH 110/362] Added warning when using the Mega backend --- Duplicati/Library/Main/Controller.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Duplicati/Library/Main/Controller.cs b/Duplicati/Library/Main/Controller.cs index 7883ddfc8..d391b7fb3 100644 --- a/Duplicati/Library/Main/Controller.cs +++ b/Duplicati/Library/Main/Controller.cs @@ -865,6 +865,10 @@ namespace Duplicati.Library.Main if (string.Equals(new Library.Utility.Uri(m_backendUrl).Scheme, "tardigrade", StringComparison.OrdinalIgnoreCase)) Logging.Log.WriteWarningMessage(LOGTAG, "TardigradeRename", null, "The Tardigrade-backend got renamed to Storj DCS - please migrate your backups to the new configuration by changing the destination storage type to Storj DCS."); + //Inform the user about the unmaintained Mega support library + if (string.Equals(new Library.Utility.Uri(m_backendUrl).Scheme, "mega", StringComparison.OrdinalIgnoreCase)) + Logging.Log.WriteWarningMessage(LOGTAG, "MegaUnmaintained", null, "The Mega support library is currently unmaintained and may not work as expected. Mega has not published an official API so it may break at any moment. Please consider migrating to another backend."); + //TODO: Based on the action, see if all options are relevant } From 91d0f08cb2e36c91a1b16871a7dc299f729fd9e4 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 6 Feb 2025 22:54:30 +0100 Subject: [PATCH 111/362] Added option to propagate releases. When building a release, it is now possible to also offer that release in "lower" channels, meaning the stable release will show as an update to instances running beta or experimental. --- ReleaseBuilder/Build/Command.Upload.cs | 23 ++++++++++++++++++++--- ReleaseBuilder/Build/Command.cs | 24 ++++++++++++++++++++---- 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/ReleaseBuilder/Build/Command.Upload.cs b/ReleaseBuilder/Build/Command.Upload.cs index d75cc2af8..c311f5bbb 100644 --- a/ReleaseBuilder/Build/Command.Upload.cs +++ b/ReleaseBuilder/Build/Command.Upload.cs @@ -106,8 +106,9 @@ public static partial class Command /// /// The files to upload /// The runtime configuration + /// The release channels to propagate to /// An awaitable task - public static async Task UploadToS3(IEnumerable files, RuntimeConfig rtcfg) + public static async Task UploadToS3(IEnumerable files, RuntimeConfig rtcfg, IEnumerable propagateTo) { var totalSize = files.Sum(f => new FileInfo(f.Path).Length); Console.WriteLine($"Uploading {files.Count()} files ({Duplicati.Library.Utility.Utility.FormatSizeString(totalSize)}) to S3..."); @@ -141,6 +142,22 @@ public static partial class Command size += filesize; Console.WriteLine($"{size / (double)totalSize * 100:F1}% - Uploaded {file.Name} ({Duplicati.Library.Utility.Utility.FormatSizeString(filesize)})"); } + + foreach (var channel in propagateTo) + { + if (channel == rtcfg.ReleaseInfo.Channel) + continue; + + var target = $"{channel.ToString().ToLowerInvariant()}/latest-v2.manifest"; + var source = $"{rtcfg.ReleaseInfo.Channel.ToString().ToLowerInvariant()}/latest-v2.manifest"; + await Duplicati.Library.Utility.RetryHelper.Retry(() => + client.CopyObjectAsync( + Program.Configuration.ConfigFiles.AwsUploadBucket, + source, + Program.Configuration.ConfigFiles.AwsUploadBucket, + target + ), 3, TimeSpan.FromSeconds(1), CancellationToken.None); + } } /// @@ -214,7 +231,7 @@ public static partial class Command /// /// The runtime configuration /// An awaitable task - public static async Task ReloadUpdateServer(RuntimeConfig rtcfg) + public static async Task ReloadUpdateServer(RuntimeConfig rtcfg, IEnumerable propagateTo) { using var client = new HttpClient(); var req = new HttpRequestMessage(HttpMethod.Post, "https://updates.duplicati.com/reload"); @@ -225,7 +242,7 @@ public static partial class Command $"{rtcfg.ReleaseInfo.Channel.ToString().ToLowerInvariant()}/latest-v2.json", $"{rtcfg.ReleaseInfo.Channel.ToString().ToLowerInvariant()}/latest-v2.js", $"{rtcfg.ReleaseInfo.Channel.ToString().ToLowerInvariant()}/latest-v2.manifest", - }); + }.Concat(propagateTo.Select(c => $"{c.ToString().ToLowerInvariant()}/latest-v2.manifest"))); var response = await client.SendAsync(req); response.EnsureSuccessStatusCode(); diff --git a/ReleaseBuilder/Build/Command.cs b/ReleaseBuilder/Build/Command.cs index 84c25fb60..d0de3a4b9 100644 --- a/ReleaseBuilder/Build/Command.cs +++ b/ReleaseBuilder/Build/Command.cs @@ -299,6 +299,12 @@ public static partial class Command getDefaultValue: () => false ); + var propagateReleaseOption = new Option( + name: "--propagate-release", + description: "Propagate the release to the next channel", + getDefaultValue: () => false + ); + var command = new System.CommandLine.Command("build", "Builds the packages for a release") { gitStashPushOption, releaseChannelArgument, @@ -322,7 +328,8 @@ public static partial class Command disableUpdateServerReloadOption, disableDiscordAnnounceOption, useHostedBuildsOption, - resumeFromUploadOption + resumeFromUploadOption, + propagateReleaseOption }; command.Handler = CommandHandler.Create(DoBuild); @@ -355,6 +362,7 @@ public static partial class Command /// If forum posting should be disabled /// If hosted builds should be used /// If the process should resume from the upload step + /// If the release should be propagated to the next channel record CommandInput( PackageTarget[] Targets, DirectoryInfo BuildPath, @@ -378,7 +386,8 @@ public static partial class Command bool DisableUpdateServerReload, bool DisableDiscordAnnounce, bool UseHostedBuilds, - bool ResumeFromUpload + bool ResumeFromUpload, + bool PropagateRelease ); static async Task DoBuild(CommandInput input) @@ -619,6 +628,13 @@ public static partial class Command if (input.GitStashPush && !input.ResumeFromUpload) await GitPush.TagAndPush(baseDir, releaseInfo); + // Propagate the release to the next channel, if selected + var nextChannels = new[] { ReleaseChannel.Experimental, ReleaseChannel.Beta, ReleaseChannel.Stable } + .Where(x => x > releaseInfo.Channel) + .Where(x => rtcfg.ReleaseInfo.Channel != ReleaseChannel.Canary) + .Where(x => input.PropagateRelease) + .ToArray(); + if (rtcfg.UseS3Upload || rtcfg.UseGithubUpload) { Console.WriteLine("Build completed, uploading packages ..."); @@ -636,7 +652,7 @@ public static partial class Command .Append(new Upload.UploadFile(packageJson, Path.GetFileName(packageJson))) .Append(new Upload.UploadFile(packageJson, $"latest-v2-{releaseInfo.Version}.json")); - await Upload.UploadToS3(uploads, rtcfg); + await Upload.UploadToS3(uploads, rtcfg, nextChannels); } if (rtcfg.UseGithubUpload) @@ -657,7 +673,7 @@ public static partial class Command if (rtcfg.UseUpdateServerReload) { Console.WriteLine("Release completed, reloading update server ..."); - await Upload.ReloadUpdateServer(rtcfg); + await Upload.ReloadUpdateServer(rtcfg, nextChannels); } if (rtcfg.UseForumPosting) From e1b9d8ec9711fe7aa4b61ddf7e6896d1f9f2e687 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 7 Feb 2025 08:51:41 +0100 Subject: [PATCH 112/362] Changed argument parsing to use NamingConventionBinder for the optional arguments --- Tools/RemoteSynchronization/Program.cs | 253 +++++++----------- .../RemoteSynchronization.csproj | 1 + 2 files changed, 99 insertions(+), 155 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 0dffc427f..b8eb0d14c 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -24,7 +24,7 @@ using Duplicati.Library.Interface; using System; using System.Collections.Generic; using System.CommandLine; -using System.CommandLine.Invocation; +using System.CommandLine.NamingConventionBinder; using System.IO; using System.Linq; using System.Threading; @@ -38,57 +38,28 @@ namespace RemoteSynchronization /// public class Program { - // Default values for the options - private const bool DEFAULT_DRY_RUN = false; - private const bool DEFAULT_VERIFY_CONTENTS = false; - private const bool DEFAULT_VERIFY_GET_AFTER_PUT = false; - private const int DEFAULT_RETRY = 3; - private const bool DEFAULT_FORCE = false; - private const bool DEFAULT_RETENTION = false; - private const bool DEFAULT_CONFIRM = false; - private const string DEFAULT_LOG_LEVEL = "Information"; - private const string DEFAULT_LOG_FILE = ""; - private const bool DEFAULT_PROGRESS = false; - private const bool DEFAULT_PARSE_ARGUMENTS_ONLY = false; - /// /// Global configuration for the tool. Should be set after parsing the commandline arguments. /// - private class GlobalConfig - { - /// - /// Whether the tool should not actually write or delete files. Defaults to false. - /// - internal static bool DryRun = DEFAULT_DRY_RUN; - /// - /// Whether the tool should force the synchronization. Defaults to false. - /// - internal static bool Force = DEFAULT_FORCE; - /// - /// The log level to use. Defaults to "Information". - /// - internal static string LogLevel = DEFAULT_LOG_LEVEL; - /// - /// The log file to write to. Defaults to an empty string. - /// - internal static string LogFile = DEFAULT_LOG_FILE; - /// - /// Whether the tool should print progress to STDOUT. Defaults to false. - /// - internal static bool Progress = DEFAULT_PROGRESS; - /// - /// Whether the tool should verify the contents of the files to decide whether the pre-existing destination files should be overwritten. Defaults to false. - /// - internal static bool VerifyContents = DEFAULT_VERIFY_CONTENTS; - /// - /// Whether the tool should verify the files after uploading them to ensure that they were uploaded correctly. Defaults to false. - /// - internal static bool VerifyGetAfterPut = DEFAULT_VERIFY_GET_AFTER_PUT; - /// - /// Try to only parse the arguments and then exit. Defaults to false. - /// - internal static bool ParseArgumentsOnly = DEFAULT_PARSE_ARGUMENTS_ONLY; - } + private sealed record Config + ( + string Src, + string Dst, + bool DryRun, + List DstOptions, + List GlobalOptions, + List SrcOptions, + bool Force, + string LogLevel, + string LogFile, + bool Progress, + int Retry, + bool Retention, + bool Confirm, + bool VerifyContents, + bool VerifyGetAfterPut, + bool ParseArgumentsOnly + ); /// /// The log tag for this tool. @@ -102,22 +73,8 @@ namespace RemoteSynchronization /// 0 on success, -1 on abort, and the number of errors encountered otherwise. public static async Task Main(string[] args) { - var src_arg = new Argument(name: "backend_src", description: "The source backend string"); - var dst_arg = new Argument(name: "backend_dst", description: "The destination backend string"); - var dry_run_opt = new Option(aliases: ["--dry-run", "-d"], description: "Do not actually write or delete files. If not set here, the global options will be checked", getDefaultValue: () => DEFAULT_DRY_RUN); - var src_opts = OptionWithMultipleTokens(aliases: ["--src-options"], description: "Options for the source backend. Each option is a key-value pair separated by an equals sign, e.g. --src-options key1=value1 key2=value2"); - var dst_opts = OptionWithMultipleTokens(aliases: ["--dst-options"], description: "Options for the destination backend. Each option is a key-value pair separated by an equals sign, e.g. --dst-options key1=value1 key2=value2"); - var verify_contents_opt = new Option(aliases: ["--verify-contents"], description: "Verify the contents of the files to decide whether the pre-existing destination files should be overwritten", getDefaultValue: () => DEFAULT_VERIFY_CONTENTS); - var verify_get_after_put_opt = new Option(aliases: ["--verify-get-after-put"], description: "Verify the files after uploading them to ensure that they were uploaded correctly", getDefaultValue: () => DEFAULT_VERIFY_GET_AFTER_PUT); - var retry_opt = new Option(aliases: ["--retry"], description: "Number of times to retry on errors", getDefaultValue: () => DEFAULT_RETRY) { Arity = ArgumentArity.ExactlyOne }; - var force_opt = new Option(aliases: ["--force"], description: "Force the synchronization", getDefaultValue: () => DEFAULT_FORCE); - var retention_opt = new Option(aliases: ["--retention"], description: "Toggles whether to keep old files. Any deletes will be renames instead", getDefaultValue: () => DEFAULT_RETENTION); - var confirm_opt = new Option(aliases: ["--confirm"], description: "Automatically confirm the operation", getDefaultValue: () => DEFAULT_CONFIRM); - var global_opts = OptionWithMultipleTokens(aliases: ["--global-options"], description: "Global options all backends. May be overridden by backend specific options (src-options, dst-options). Each option is a key-value pair separated by an equals sign, e.g. --global-options key1=value1 key2=value2"); - var log_level_opt = new Option(aliases: ["--log-level"], description: "The log level to use. If not set here, global options will be checked", getDefaultValue: () => DEFAULT_LOG_LEVEL) { Arity = ArgumentArity.ExactlyOne }; - var log_file_opt = new Option(aliases: ["--log-file"], description: "The log file to write to. If not set here, global options will be checked", getDefaultValue: () => DEFAULT_LOG_FILE) { Arity = ArgumentArity.ExactlyOne }; - var progress_opt = new Option(aliases: ["--progress"], description: "Print progress to STDOUT", getDefaultValue: () => DEFAULT_PROGRESS); - var parse_opt = new Option(aliases: ["--parse-arguments-only"], description: "Only parse the arguments and then exit", getDefaultValue: () => DEFAULT_PARSE_ARGUMENTS_ONLY); + var arg_src = new Argument(name: "backend_src", description: "The source backend string"); + var arg_dst = new Argument(name: "backend_dst", description: "The destination backend string"); var root_cmd = new RootCommand(@"Remote Synchronization Tool @@ -137,31 +94,31 @@ and delete (or rename) all files in the destination. If the verify option is set, the files will be downloaded and compared after uploading to ensure that the files are correct. Files that already exist in the destination will be verified before being overwritten (if they seemingly match). - "); - root_cmd.AddArgument(src_arg); - root_cmd.AddArgument(dst_arg); - root_cmd.AddOption(dry_run_opt); - root_cmd.AddOption(src_opts); - root_cmd.AddOption(dst_opts); - root_cmd.AddOption(verify_contents_opt); - root_cmd.AddOption(verify_get_after_put_opt); - root_cmd.AddOption(retry_opt); - root_cmd.AddOption(force_opt); - root_cmd.AddOption(retention_opt); - root_cmd.AddOption(confirm_opt); - root_cmd.AddOption(global_opts); - root_cmd.AddOption(log_level_opt); - root_cmd.AddOption(log_file_opt); - root_cmd.AddOption(progress_opt); - root_cmd.AddOption(parse_opt); - - root_cmd.SetHandler((InvocationContext ctx) => + ") { - var parsed = ctx.ParseResult; + arg_src, + arg_dst, + new Option(aliases: ["--dry-run", "-d"], description: "Do not actually write or delete files. If not set here, the global options will be checked", getDefaultValue: () => false), + OptionWithMultipleTokens(aliases: ["--src-options"], description: "Options for the source backend. Each option is a key-value pair separated by an equals sign, e.g. --src-options key1=value1 key2=value2"), + OptionWithMultipleTokens(aliases: ["--dst-options"], description: "Options for the destination backend. Each option is a key-value pair separated by an equals sign, e.g. --dst-options key1=value1 key2=value2"), + new Option(aliases: ["--verify-contents"], description: "Verify the contents of the files to decide whether the pre-existing destination files should be overwritten", getDefaultValue: () => false), + new Option(aliases: ["--verify-get-after-put"], description: "Verify the files after uploading them to ensure that they were uploaded correctly", getDefaultValue: () => false), + new Option(aliases: ["--retry"], description: "Number of times to retry on errors", getDefaultValue: () => 3) { Arity = ArgumentArity.ExactlyOne }, + new Option(aliases: ["--force"], description: "Force the synchronization", getDefaultValue: () => false), + new Option(aliases: ["--retention"], description: "Toggles whether to keep old files. Any deletes will be renames instead", getDefaultValue: () => false), + new Option(aliases: ["--confirm"], description: "Automatically confirm the operation", getDefaultValue: () => false), + OptionWithMultipleTokens(aliases: ["--global-options"], description: "Global options all backends. May be overridden by backend specific options (src-options, dst-options). Each option is a key-value pair separated by an equals sign, e.g. --global-options key1=value1 key2=value2"), + new Option(aliases: ["--log-level"], description: "The log level to use. If not set here, global options will be checked", getDefaultValue: () => "Information") { Arity = ArgumentArity.ExactlyOne }, + new Option(aliases: ["--log-file"], description: "The log file to write to. If not set here, global options will be checked", getDefaultValue: () => "") { Arity = ArgumentArity.ExactlyOne }, + new Option(aliases: ["--progress"], description: "Print progress to STDOUT", getDefaultValue: () => false), + new Option(aliases: ["--parse-arguments-only"], description: "Only parse the arguments and then exit", getDefaultValue: () => false), + }; - Dictionary options = parsed.CommandResult.Command.Options.ToDictionary(x => x.Name, x => parsed.GetValueForOption(x)); + root_cmd.Handler = CommandHandler.Create((string backend_src, string backend_dst, Config config) => + { + var config_with_args = config with { Dst = backend_dst, Src = backend_src }; - Run(parsed.GetValueForArgument(src_arg), parsed.GetValueForArgument(dst_arg), options).Wait(); + return Run(config_with_args); }); return await root_cmd.InvokeAsync(args); @@ -174,55 +131,41 @@ destination will be verified before being overwritten (if they seemingly match). /// The connection string for the destination backend. /// Various options for the tool /// The return code for the main entry; 0 on success. - private static async Task Run(string src, string dst, Dictionary options) + private static async Task Run(Config config) { - // Parse the known options for this tool - GlobalConfig.Force = options["force"] as bool? ?? DEFAULT_FORCE; - GlobalConfig.DryRun = options["dry-run"] as bool? ?? DEFAULT_DRY_RUN; - GlobalConfig.LogLevel = options["log-level"] as string ?? DEFAULT_LOG_LEVEL; - GlobalConfig.LogFile = options["log-file"] as string ?? DEFAULT_LOG_FILE; - GlobalConfig.VerifyContents = options["verify-contents"] as bool? ?? DEFAULT_VERIFY_CONTENTS; - GlobalConfig.VerifyGetAfterPut = options["verify-get-after-put"] as bool? ?? DEFAULT_VERIFY_GET_AFTER_PUT; - var retries = options["retry"] as int? ?? DEFAULT_RETRY; - var retention = options["retention"] as bool? ?? DEFAULT_RETENTION; - var confirm = options["confirm"] as bool? ?? DEFAULT_CONFIRM; - var parse_only = options["parse-arguments-only"] as bool? ?? DEFAULT_PARSE_ARGUMENTS_ONLY; - // Unpack and parse the multi token options - Dictionary global_options = (options["global-options"] as List) + Dictionary global_options = config.GlobalOptions ?.Select(x => x.Split("=")) .ToDictionary(x => x[0], x => x[1]) ?? []; // Parse the log level - var log_level = options["log-level"] as string ?? DEFAULT_LOG_LEVEL; - var log_level_parsed = Enum.TryParse(log_level, true, out var log_level_enum); + var log_level_parsed = Enum.TryParse(config.LogLevel, true, out var log_level_enum); log_level_enum = log_level_parsed ? log_level_enum : Duplicati.Library.Logging.LogMessageType.Information; using var console_sink = new Duplicati.CommandLine.ConsoleOutput(Console.Out, global_options); using var multi_sink = new Duplicati.Library.Main.ControllerMultiLogTarget(console_sink, log_level_enum, null); // Parse the log file - var log_file = options["log-file"] as string ?? DEFAULT_LOG_FILE; // The log file sink doesn't have to be disposed, as the multi_sink will take care of it Duplicati.Library.Logging.StreamLogDestination? log_file_sink = null; - if (!string.IsNullOrEmpty(log_file)) + if (!string.IsNullOrEmpty(config.LogFile)) { - string log_file_dir = SystemIO.IO_OS.PathGetDirectoryName(log_file); + string log_file_dir = SystemIO.IO_OS.PathGetDirectoryName(config.LogFile); if (!string.IsNullOrEmpty(log_file_dir) && !SystemIO.IO_OS.DirectoryExists(log_file_dir)) SystemIO.IO_OS.DirectoryCreate(log_file_dir); - log_file_sink = new Duplicati.Library.Logging.StreamLogDestination(log_file); + log_file_sink = new Duplicati.Library.Logging.StreamLogDestination(config.LogFile); } multi_sink.AddTarget(log_file_sink, log_level_enum, null); // Start the logging scope using var _ = Duplicati.Library.Logging.Log.StartScope(multi_sink, log_level_enum); - Dictionary src_opts = (options["src-options"] as List) + Dictionary src_opts = config.SrcOptions ?.Select(x => x.Split("=")) .ToDictionary(x => x[0], x => x[1]) ?? []; - Dictionary dst_opts = (options["dst-options"] as List) + Dictionary dst_opts = config.DstOptions ?.Select(x => x.Split("=")) .ToDictionary(x => x[0], x => x[1]) ?? []; @@ -232,30 +175,30 @@ destination will be verified before being overwritten (if they seemingly match). src_opts[x.Key] = dst_opts[x.Key] = x.Value; // Check if we only had to parse the arguments - if (parse_only) + if (config.ParseArgumentsOnly) { Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Arguments parsed successfully; exiting."); return 0; } // Load the backends - using var b1 = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(src, src_opts); + using var b1 = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(config.Src, src_opts); var b1s = b1 as IStreamingBackend; System.Diagnostics.Debug.Assert(b1s != null); - using var b2 = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(dst, dst_opts); + using var b2 = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(config.Dst, dst_opts); var b2s = b2 as IStreamingBackend; System.Diagnostics.Debug.Assert(b2s != null); // Prepare the operations - var (to_copy, to_delete, to_verify) = PrepareFileLists(b1s, b2s); + var (to_copy, to_delete, to_verify) = PrepareFileLists(b1s, b2s, config); // Verify the files if requested. If the files are not verified, they will be deleted and copied again. long verified = 0, failed_verify = 0; - if (GlobalConfig.VerifyContents) + if (config.VerifyContents) { // As this is a potentially slow operation, ask for confirmation of the verification) - if (!confirm) + if (!config.Confirm) { Console.WriteLine($"This will verify {to_verify.Count()} files before copying them. Do you want to continue? [y/N]"); var response = Console.ReadLine(); @@ -266,7 +209,7 @@ destination will be verified before being overwritten (if they seemingly match). } } - var not_verified = await VerifyAsync(b1s, b2s, to_verify); + var not_verified = await VerifyAsync(b1s, b2s, to_verify, config); failed_verify = not_verified.Count(); verified = to_verify.Count() - failed_verify; @@ -278,15 +221,15 @@ destination will be verified before being overwritten (if they seemingly match). } } - Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "The remote synchronization plan is to {0} {1} files from {2}, then copy {3} files from {4} to {2}.", retention ? "rename" : "delete", to_delete.Count(), b2s.DisplayName, to_copy.Count(), b1s.DisplayName); + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "The remote synchronization plan is to {0} {1} files from {2}, then copy {3} files from {4} to {2}.", config.Retention ? "rename" : "delete", to_delete.Count(), b2s.DisplayName, to_copy.Count(), b1s.DisplayName); // As this is a potentially destructive operation, ask for confirmation - if (!confirm) + if (!config.Confirm) { - var delete_rename = retention ? "Rename" : "Delete"; + var delete_rename = config.Retention ? "Rename" : "Delete"; Console.WriteLine($"This will perform the following actions (in order):"); - Console.WriteLine($" - {delete_rename} {to_delete.Count()} files from {dst}"); - Console.WriteLine($" - Copy {to_copy.Count()} files from {src} to {dst}"); + Console.WriteLine($" - {delete_rename} {to_delete.Count()} files from {config.Dst}"); + Console.WriteLine($" - Copy {to_copy.Count()} files from {config.Src} to {config.Dst}"); Console.WriteLine(); Console.WriteLine("Do you want to continue? [y/N]"); @@ -300,32 +243,32 @@ destination will be verified before being overwritten (if they seemingly match). // Delete or rename the files that are not needed long renamed = 0, deleted = 0; - if (retention) + if (config.Retention) { - renamed = await RenameAsync(b2, to_delete); + renamed = await RenameAsync(b2, to_delete, config); Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Renamed {0} files in {1}", renamed, b2s.DisplayName); } else { - deleted = await DeleteAsync(b2s, to_delete); + deleted = await DeleteAsync(b2s, to_delete, config); Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Deleted {0} files from {1}", deleted, b2s.DisplayName); } // Copy the files - var (copied, copy_errors) = await CopyAsync(b1s, b2s, to_copy); + var (copied, copy_errors) = await CopyAsync(b1s, b2s, to_copy, config); Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Copied {0} files from {1} to {2}", copied, b1s, b2s); // If there are still errors, retry a few times if (copy_errors.Any()) { Duplicati.Library.Logging.Log.WriteWarningMessage(LOGTAG, "rsync", null, "Could not copy {0} files.", copy_errors.Count()); - if (retries > 0) + if (config.Retry > 0) { - Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Retrying {0} more times to copy the {1} files that failed", retries, copy_errors.Count()); - for (int i = 0; i < retries; i++) + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Retrying {0} more times to copy the {1} files that failed", config.Retry, copy_errors.Count()); + for (int i = 0; i < config.Retry; i++) { Thread.Sleep(5000); // Wait 5 seconds before retrying - (copied, copy_errors) = await CopyAsync(b1s, b2s, copy_errors); + (copied, copy_errors) = await CopyAsync(b1s, b2s, copy_errors, config); Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Copied {0} files from {1} to {2}", copied, b1s, b2s); if (!copy_errors.Any()) break; @@ -369,10 +312,9 @@ destination will be verified before being overwritten (if they seemingly match). /// The source backend. /// The destination backend. /// The files that will be copied. - /// Flag for whether destructive actions (writes) should be printed rather than performed. Downloads (reads) are still being performed. - /// Flag for whether to verify each upload to the destination backend. + /// The parsed configuration for the tool. /// A tuple holding the number of succesful copies and a List of the files that failed. - private static async Task<(long, IEnumerable)> CopyAsync(IStreamingBackend b_src, IStreamingBackend b_dst, IEnumerable files) + private static async Task<(long, IEnumerable)> CopyAsync(IStreamingBackend b_src, IStreamingBackend b_dst, IEnumerable files, Config config) { long successful_copies = 0; List errors = []; @@ -386,7 +328,7 @@ destination will be verified before being overwritten (if they seemingly match). foreach (var f in files) { - if (GlobalConfig.Progress) + if (config.Progress) Console.Write($"\rCopying: {i}/{n}"); Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", "Copying {0} from {1} to {2}", f.Name, b_src.DisplayName, b_dst.DisplayName); @@ -396,7 +338,7 @@ destination will be verified before being overwritten (if they seemingly match). sw_get_src.Start(); await b_src.GetAsync(f.Name, s_src, CancellationToken.None); sw_get_src.Stop(); - if (GlobalConfig.DryRun) + if (config.DryRun) { Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", "Would write {0} bytes of {1} to {2}", s_src.Length, f.Name, b_dst.DisplayName); } @@ -405,7 +347,7 @@ destination will be verified before being overwritten (if they seemingly match). sw_put_dst.Start(); await b_dst.PutAsync(f.Name, s_src, CancellationToken.None); sw_put_dst.Stop(); - if (GlobalConfig.VerifyGetAfterPut) + if (config.VerifyGetAfterPut) { sw_get_dst.Start(); await b_dst.GetAsync(f.Name, s_dst, CancellationToken.None); @@ -452,7 +394,7 @@ destination will be verified before being overwritten (if they seemingly match). } } - if (GlobalConfig.Progress) + if (config.Progress) Console.WriteLine($"\rCopying: {n}/{n}"); Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", "Copy | Get source: {0} ms, Put destination: {1} ms, Get destination: {2} ms, Get compare: {3} ms", sw_get_src.ElapsedMilliseconds, sw_put_dst.ElapsedMilliseconds, sw_get_dst.ElapsedMilliseconds, sw_get_cmp.ElapsedMilliseconds); @@ -465,9 +407,9 @@ destination will be verified before being overwritten (if they seemingly match). /// /// The backend to delete the files from. /// The files to delete. - /// Flag for whether the deletion should be printed rather than performed. + /// The parsed configuration for the tool. /// The number of successful deletions. - private static async Task DeleteAsync(IStreamingBackend b, IEnumerable files) + private static async Task DeleteAsync(IStreamingBackend b, IEnumerable files, Config config) { long successful_deletes = 0; long i = 0, n = files.Count(); @@ -476,7 +418,7 @@ destination will be verified before being overwritten (if they seemingly match). foreach (var f in files) { - if (n > 1 && GlobalConfig.Progress) + if (n > 1 && config.Progress) { Console.Write($"\rDeleting: {i}/{n}"); } @@ -485,7 +427,7 @@ destination will be verified before being overwritten (if they seemingly match). try { - if (GlobalConfig.DryRun) + if (config.DryRun) { Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", "Would delete {0} from {1}", f.Name, b.DisplayName); } @@ -506,7 +448,7 @@ destination will be verified before being overwritten (if they seemingly match). sw.Stop(); Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", "Delete: {0} ms", sw.ElapsedMilliseconds); - if (GlobalConfig.Progress) + if (config.Progress) Console.WriteLine($"\rDeleting: {n}/{n}"); return successful_deletes; } @@ -534,9 +476,9 @@ destination will be verified before being overwritten (if they seemingly match). /// /// The source backend. /// The destination backend. - /// Flag for whether to force the synchronization. + /// The parsed configuration for the tool. /// A tuple of Lists each holding the files to copy, delete and verify. - private static (IEnumerable, IEnumerable, IEnumerable) PrepareFileLists(IStreamingBackend b_src, IStreamingBackend b_dst) + private static (IEnumerable, IEnumerable, IEnumerable) PrepareFileLists(IStreamingBackend b_src, IStreamingBackend b_dst, Config config) { var sw = new System.Diagnostics.Stopwatch(); @@ -550,7 +492,7 @@ destination will be verified before being overwritten (if they seemingly match). Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", "Prepare | List destination: {0} ms", sw.ElapsedMilliseconds); // Shortcut for force - if (GlobalConfig.Force) + if (config.Force) { return (files_src, files_dst, []); } @@ -627,9 +569,9 @@ destination will be verified before being overwritten (if they seemingly match). /// /// The backend to rename the files in. /// The files to rename. - /// Flag for whether the renaming should be printed rather than performed. + /// The parsed configuration for the tool. /// The number of successful renames. - private static async Task RenameAsync(IBackend b, IEnumerable files) + private static async Task RenameAsync(IBackend b, IEnumerable files, Config config) { long successful_renames = 0; string prefix = $"{System.DateTime.Now:yyyyMMddHHmmss}.old"; @@ -646,7 +588,7 @@ destination will be verified before being overwritten (if they seemingly match). foreach (var f in files) { - if (GlobalConfig.Progress) + if (config.Progress) Console.Write($"\rRenaming: {i}/{n}"); Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", "Renaming {0} to {1}.{0} by deleting and re-uploading {2} bytes to {3}", f.Name, prefix, downloaded.Length, sb.DisplayName); @@ -657,7 +599,7 @@ destination will be verified before being overwritten (if they seemingly match). await sb.GetAsync(f.Name, downloaded, CancellationToken.None); sw_get_src.Stop(); - if (GlobalConfig.DryRun) + if (config.DryRun) { Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", "Would rename {0} to {1}.{0} by deleting and re-uploading {2} bytes to {3}", f.Name, prefix, downloaded.Length, sb.DisplayName); } @@ -692,7 +634,7 @@ destination will be verified before being overwritten (if they seemingly match). Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", "Rename | Get source: {0} ms, Put destination: {1} ms, Delete source: {2} ms", sw_get_src.ElapsedMilliseconds, sw_put_dst.ElapsedMilliseconds, sw_del_src.ElapsedMilliseconds); - if (GlobalConfig.Progress) + if (config.Progress) Console.WriteLine($"\rRenaming: {n}/{n}"); return successful_renames; @@ -703,14 +645,14 @@ destination will be verified before being overwritten (if they seemingly match). foreach (var f in files) { - if (GlobalConfig.Progress) + if (config.Progress) Console.Write($"\rRenaming: {i}/{n}"); Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", "Renaming {0} to {1}.{0} by calling Rename on {2}", f.Name, prefix, rb.DisplayName); try { - if (GlobalConfig.DryRun) + if (config.DryRun) { Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", "Would rename {0} to {1}.{0} by calling Rename on {2}", f.Name, prefix, rb.DisplayName); } @@ -733,7 +675,7 @@ destination will be verified before being overwritten (if they seemingly match). Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", "Rename: {0} ms", sw.ElapsedMilliseconds); - if (GlobalConfig.Progress) + if (config.Progress) Console.WriteLine($"\rRenaming: {n}/{n}"); return successful_renames; @@ -750,8 +692,9 @@ destination will be verified before being overwritten (if they seemingly match). /// The source backend. /// The destination backend. /// The files to verify. + /// The parsed configuration for the tool. /// A list of the files that failed verification. - private static async Task> VerifyAsync(IStreamingBackend b_src, IStreamingBackend b_dst, IEnumerable files) + private static async Task> VerifyAsync(IStreamingBackend b_src, IStreamingBackend b_dst, IEnumerable files, Config config) { var errors = new List(); using var s_src = new MemoryStream(); @@ -762,7 +705,7 @@ destination will be verified before being overwritten (if they seemingly match). foreach (var f in files) { - if (GlobalConfig.Progress) + if (config.Progress) Console.Write($"\rVerifying: {i}/{n}"); Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", "Verifying {0} by downloading and comparing {1} bytes from {2} and {3}", f.Name, s_src.Length, b_dst.DisplayName, b_src.DisplayName); @@ -805,7 +748,7 @@ destination will be verified before being overwritten (if they seemingly match). Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", "Verify | Get: {0} ms, Compare: {1} ms", sw_get.ElapsedMilliseconds, sw_cmp.ElapsedMilliseconds); - if (GlobalConfig.Progress) + if (config.Progress) Console.WriteLine($"\rVerifying: {n}/{n}"); return errors; diff --git a/Tools/RemoteSynchronization/RemoteSynchronization.csproj b/Tools/RemoteSynchronization/RemoteSynchronization.csproj index 7e044266f..6cd7b2704 100644 --- a/Tools/RemoteSynchronization/RemoteSynchronization.csproj +++ b/Tools/RemoteSynchronization/RemoteSynchronization.csproj @@ -11,5 +11,6 @@ + From 53a3ca19962fa8214763ad04346f4bb5494e01a5 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 7 Feb 2025 09:06:54 +0100 Subject: [PATCH 113/362] Reordered options alphabetically --- Tools/RemoteSynchronization/Program.cs | 42 ++++++++++++++------------ 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index b8eb0d14c..b662d4cfd 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2025, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a @@ -43,22 +43,25 @@ namespace RemoteSynchronization /// private sealed record Config ( + // Arguments string Src, string Dst, + + // Options + bool Confirm, bool DryRun, List DstOptions, - List GlobalOptions, - List SrcOptions, bool Force, - string LogLevel, + List GlobalOptions, string LogFile, + string LogLevel, + bool ParseArgumentsOnly, bool Progress, - int Retry, bool Retention, - bool Confirm, + int Retry, + List SrcOptions, bool VerifyContents, - bool VerifyGetAfterPut, - bool ParseArgumentsOnly + bool VerifyGetAfterPut ); /// @@ -98,20 +101,21 @@ destination will be verified before being overwritten (if they seemingly match). { arg_src, arg_dst, + + new Option(aliases: ["--confirm"], description: "Automatically confirm the operation", getDefaultValue: () => false), new Option(aliases: ["--dry-run", "-d"], description: "Do not actually write or delete files. If not set here, the global options will be checked", getDefaultValue: () => false), - OptionWithMultipleTokens(aliases: ["--src-options"], description: "Options for the source backend. Each option is a key-value pair separated by an equals sign, e.g. --src-options key1=value1 key2=value2"), - OptionWithMultipleTokens(aliases: ["--dst-options"], description: "Options for the destination backend. Each option is a key-value pair separated by an equals sign, e.g. --dst-options key1=value1 key2=value2"), + OptionWithMultipleTokens(aliases: ["--dst-options"], description: "Options for the destination backend. Each option is a key-value pair separated by an equals sign, e.g. --dst-options key1=value1 key2=value2", getDefaultValue: () => []), + new Option(aliases: ["--force"], description: "Force the synchronization", getDefaultValue: () => false), + OptionWithMultipleTokens(aliases: ["--global-options"], description: "Global options all backends. May be overridden by backend specific options (src-options, dst-options). Each option is a key-value pair separated by an equals sign, e.g. --global-options key1=value1 key2=value2", getDefaultValue: () => []), + new Option(aliases: ["--log-file"], description: "The log file to write to. If not set here, global options will be checked", getDefaultValue: () => "") { Arity = ArgumentArity.ExactlyOne }, + new Option(aliases: ["--log-level"], description: "The log level to use. If not set here, global options will be checked", getDefaultValue: () => "Information") { Arity = ArgumentArity.ExactlyOne }, + new Option(aliases: ["--parse-arguments-only"], description: "Only parse the arguments and then exit", getDefaultValue: () => false), + new Option(aliases: ["--progress"], description: "Print progress to STDOUT", getDefaultValue: () => false), + new Option(aliases: ["--retention"], description: "Toggles whether to keep old files. Any deletes will be renames instead", getDefaultValue: () => false), + new Option(aliases: ["--retry"], description: "Number of times to retry on errors", getDefaultValue: () => 3) { Arity = ArgumentArity.ExactlyOne }, + OptionWithMultipleTokens(aliases: ["--src-options"], description: "Options for the source backend. Each option is a key-value pair separated by an equals sign, e.g. --src-options key1=value1 key2=value2", getDefaultValue: () => []), new Option(aliases: ["--verify-contents"], description: "Verify the contents of the files to decide whether the pre-existing destination files should be overwritten", getDefaultValue: () => false), new Option(aliases: ["--verify-get-after-put"], description: "Verify the files after uploading them to ensure that they were uploaded correctly", getDefaultValue: () => false), - new Option(aliases: ["--retry"], description: "Number of times to retry on errors", getDefaultValue: () => 3) { Arity = ArgumentArity.ExactlyOne }, - new Option(aliases: ["--force"], description: "Force the synchronization", getDefaultValue: () => false), - new Option(aliases: ["--retention"], description: "Toggles whether to keep old files. Any deletes will be renames instead", getDefaultValue: () => false), - new Option(aliases: ["--confirm"], description: "Automatically confirm the operation", getDefaultValue: () => false), - OptionWithMultipleTokens(aliases: ["--global-options"], description: "Global options all backends. May be overridden by backend specific options (src-options, dst-options). Each option is a key-value pair separated by an equals sign, e.g. --global-options key1=value1 key2=value2"), - new Option(aliases: ["--log-level"], description: "The log level to use. If not set here, global options will be checked", getDefaultValue: () => "Information") { Arity = ArgumentArity.ExactlyOne }, - new Option(aliases: ["--log-file"], description: "The log file to write to. If not set here, global options will be checked", getDefaultValue: () => "") { Arity = ArgumentArity.ExactlyOne }, - new Option(aliases: ["--progress"], description: "Print progress to STDOUT", getDefaultValue: () => false), - new Option(aliases: ["--parse-arguments-only"], description: "Only parse the arguments and then exit", getDefaultValue: () => false), }; root_cmd.Handler = CommandHandler.Create((string backend_src, string backend_dst, Config config) => From edb9f53ea82066dbcfb0b0ad764ab98d4ef535f7 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 7 Feb 2025 09:07:11 +0100 Subject: [PATCH 114/362] Added default value to the multitoken options --- Tools/RemoteSynchronization/Program.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index b662d4cfd..fd9cf00fc 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -463,9 +463,9 @@ destination will be verified before being overwritten (if they seemingly match). /// The aliases for the option. /// The description for the option. /// The created option. - private static Option> OptionWithMultipleTokens(string[] aliases, string description) + private static Option> OptionWithMultipleTokens(string[] aliases, string description, Func> getDefaultValue) { - return new Option>(aliases: aliases, description: description) + return new Option>(aliases: aliases, description: description, getDefaultValue: getDefaultValue) { Arity = ArgumentArity.OneOrMore, AllowMultipleArgumentsPerToken = true From 83ddef691f1e028769ba97df1d2ea1e0e83270e2 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 7 Feb 2025 09:07:49 +0100 Subject: [PATCH 115/362] Added more aliases to some of the options to mimic other commandline tools --- Tools/RemoteSynchronization/Program.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index fd9cf00fc..9a5b8e4af 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -102,10 +102,10 @@ destination will be verified before being overwritten (if they seemingly match). arg_src, arg_dst, - new Option(aliases: ["--confirm"], description: "Automatically confirm the operation", getDefaultValue: () => false), + new Option(aliases: ["--confirm", "--yes", "-y"], description: "Automatically confirm the operation", getDefaultValue: () => false), new Option(aliases: ["--dry-run", "-d"], description: "Do not actually write or delete files. If not set here, the global options will be checked", getDefaultValue: () => false), OptionWithMultipleTokens(aliases: ["--dst-options"], description: "Options for the destination backend. Each option is a key-value pair separated by an equals sign, e.g. --dst-options key1=value1 key2=value2", getDefaultValue: () => []), - new Option(aliases: ["--force"], description: "Force the synchronization", getDefaultValue: () => false), + new Option(aliases: ["--force", "-f"], description: "Force the synchronization", getDefaultValue: () => false), OptionWithMultipleTokens(aliases: ["--global-options"], description: "Global options all backends. May be overridden by backend specific options (src-options, dst-options). Each option is a key-value pair separated by an equals sign, e.g. --global-options key1=value1 key2=value2", getDefaultValue: () => []), new Option(aliases: ["--log-file"], description: "The log file to write to. If not set here, global options will be checked", getDefaultValue: () => "") { Arity = ArgumentArity.ExactlyOne }, new Option(aliases: ["--log-level"], description: "The log level to use. If not set here, global options will be checked", getDefaultValue: () => "Information") { Arity = ArgumentArity.ExactlyOne }, From d9d427e27656b6f4b97dcf8d518f7c37ada42e52 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 7 Feb 2025 09:08:38 +0100 Subject: [PATCH 116/362] With the new parser, the options are no longer nullable --- Tools/RemoteSynchronization/Program.cs | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 9a5b8e4af..0586d8daf 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -139,9 +139,8 @@ destination will be verified before being overwritten (if they seemingly match). { // Unpack and parse the multi token options Dictionary global_options = config.GlobalOptions - ?.Select(x => x.Split("=")) - .ToDictionary(x => x[0], x => x[1]) - ?? []; + .Select(x => x.Split("=")) + .ToDictionary(x => x[0], x => x[1]); // Parse the log level var log_level_parsed = Enum.TryParse(config.LogLevel, true, out var log_level_enum); @@ -166,13 +165,12 @@ destination will be verified before being overwritten (if they seemingly match). using var _ = Duplicati.Library.Logging.Log.StartScope(multi_sink, log_level_enum); Dictionary src_opts = config.SrcOptions - ?.Select(x => x.Split("=")) - .ToDictionary(x => x[0], x => x[1]) - ?? []; + .Select(x => x.Split("=")) + .ToDictionary(x => x[0], x => x[1]); + Dictionary dst_opts = config.DstOptions - ?.Select(x => x.Split("=")) - .ToDictionary(x => x[0], x => x[1]) - ?? []; + .Select(x => x.Split("=")) + .ToDictionary(x => x[0], x => x[1]); // Merge the global options into the source and destination options foreach (var x in global_options) From ed7d9e18362ec11028b28bffe2af8dd4449dc8b9 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 7 Feb 2025 09:09:16 +0100 Subject: [PATCH 117/362] Fixed incorrect options precedence. [src|dst] options override global options --- Tools/RemoteSynchronization/Program.cs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 0586d8daf..73d6d805c 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2025, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a @@ -172,9 +172,14 @@ destination will be verified before being overwritten (if they seemingly match). .Select(x => x.Split("=")) .ToDictionary(x => x[0], x => x[1]); - // Merge the global options into the source and destination options + // Merge the global options into the source and destination options. The global options will be overridden by the source and destination options. foreach (var x in global_options) - src_opts[x.Key] = dst_opts[x.Key] = x.Value; + { + if (!src_opts.ContainsKey(x.Key)) + src_opts[x.Key] = x.Value; + if (!dst_opts.ContainsKey(x.Key)) + dst_opts[x.Key] = x.Value; + } // Check if we only had to parse the arguments if (config.ParseArgumentsOnly) From 7f9d5c83725393a97bb28a2fe3794b8f14962ba1 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 7 Feb 2025 09:14:05 +0100 Subject: [PATCH 118/362] Added default values to the descriptions of the options where it isn't printed in the help text. --- Tools/RemoteSynchronization/Program.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 73d6d805c..f22f69899 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -104,16 +104,16 @@ destination will be verified before being overwritten (if they seemingly match). new Option(aliases: ["--confirm", "--yes", "-y"], description: "Automatically confirm the operation", getDefaultValue: () => false), new Option(aliases: ["--dry-run", "-d"], description: "Do not actually write or delete files. If not set here, the global options will be checked", getDefaultValue: () => false), - OptionWithMultipleTokens(aliases: ["--dst-options"], description: "Options for the destination backend. Each option is a key-value pair separated by an equals sign, e.g. --dst-options key1=value1 key2=value2", getDefaultValue: () => []), + OptionWithMultipleTokens(aliases: ["--dst-options"], description: "Options for the destination backend. Each option is a key-value pair separated by an equals sign, e.g. --dst-options key1=value1 key2=value2 [default: empty]", getDefaultValue: () => []), new Option(aliases: ["--force", "-f"], description: "Force the synchronization", getDefaultValue: () => false), - OptionWithMultipleTokens(aliases: ["--global-options"], description: "Global options all backends. May be overridden by backend specific options (src-options, dst-options). Each option is a key-value pair separated by an equals sign, e.g. --global-options key1=value1 key2=value2", getDefaultValue: () => []), - new Option(aliases: ["--log-file"], description: "The log file to write to. If not set here, global options will be checked", getDefaultValue: () => "") { Arity = ArgumentArity.ExactlyOne }, + OptionWithMultipleTokens(aliases: ["--global-options"], description: "Global options all backends. May be overridden by backend specific options (src-options, dst-options). Each option is a key-value pair separated by an equals sign, e.g. --global-options key1=value1 key2=value2 [default: empty]", getDefaultValue: () => []), + new Option(aliases: ["--log-file"], description: "The log file to write to. If not set here, global options will be checked [default: \"\"]", getDefaultValue: () => "") { Arity = ArgumentArity.ExactlyOne }, new Option(aliases: ["--log-level"], description: "The log level to use. If not set here, global options will be checked", getDefaultValue: () => "Information") { Arity = ArgumentArity.ExactlyOne }, new Option(aliases: ["--parse-arguments-only"], description: "Only parse the arguments and then exit", getDefaultValue: () => false), new Option(aliases: ["--progress"], description: "Print progress to STDOUT", getDefaultValue: () => false), new Option(aliases: ["--retention"], description: "Toggles whether to keep old files. Any deletes will be renames instead", getDefaultValue: () => false), new Option(aliases: ["--retry"], description: "Number of times to retry on errors", getDefaultValue: () => 3) { Arity = ArgumentArity.ExactlyOne }, - OptionWithMultipleTokens(aliases: ["--src-options"], description: "Options for the source backend. Each option is a key-value pair separated by an equals sign, e.g. --src-options key1=value1 key2=value2", getDefaultValue: () => []), + OptionWithMultipleTokens(aliases: ["--src-options"], description: "Options for the source backend. Each option is a key-value pair separated by an equals sign, e.g. --src-options key1=value1 key2=value2 [default: empty]", getDefaultValue: () => []), new Option(aliases: ["--verify-contents"], description: "Verify the contents of the files to decide whether the pre-existing destination files should be overwritten", getDefaultValue: () => false), new Option(aliases: ["--verify-get-after-put"], description: "Verify the files after uploading them to ensure that they were uploaded correctly", getDefaultValue: () => false), }; From 8254a71d785ffcee761f24e0bae1f23b06bc5f96 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 7 Feb 2025 09:40:08 +0100 Subject: [PATCH 119/362] Unsupported backends now throw a more meaningful error message --- Tools/RemoteSynchronization/Program.cs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index f22f69899..d773e4c0d 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -189,13 +189,19 @@ destination will be verified before being overwritten (if they seemingly match). } // Load the backends + static Exception throw_message(string target) + { + var message = $"The {target} backend does not support streaming operations."; + var ex = new Exception(message); + Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", ex, message); + return ex; + } + using var b1 = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(config.Src, src_opts); - var b1s = b1 as IStreamingBackend; - System.Diagnostics.Debug.Assert(b1s != null); + var b1s = b1 as IStreamingBackend ?? throw throw_message("source"); using var b2 = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(config.Dst, dst_opts); - var b2s = b2 as IStreamingBackend; - System.Diagnostics.Debug.Assert(b2s != null); + var b2s = b2 as IStreamingBackend ?? throw throw_message("destination"); // Prepare the operations var (to_copy, to_delete, to_verify) = PrepareFileLists(b1s, b2s, config); From c54056ca96663b2d42eb348605abeeb7f1a6e21a Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 7 Feb 2025 09:41:26 +0100 Subject: [PATCH 120/362] Await Task.Delay instead of Thread.Sleep --- Tools/RemoteSynchronization/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index d773e4c0d..08dec5e5e 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -280,7 +280,7 @@ destination will be verified before being overwritten (if they seemingly match). Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Retrying {0} more times to copy the {1} files that failed", config.Retry, copy_errors.Count()); for (int i = 0; i < config.Retry; i++) { - Thread.Sleep(5000); // Wait 5 seconds before retrying + await Task.Delay(5000); // Wait 5 seconds before retrying (copied, copy_errors) = await CopyAsync(b1s, b2s, copy_errors, config); Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Copied {0} files from {1} to {2}", copied, b1s, b2s); if (!copy_errors.Any()) From 2887fa7058abe93948af9761d2ca934b75151bf1 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 7 Feb 2025 09:46:22 +0100 Subject: [PATCH 121/362] Made logged byte sizes human readable using duplicati utility function --- Tools/RemoteSynchronization/Program.cs | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 08dec5e5e..1e0e6f9eb 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -353,7 +353,10 @@ destination will be verified before being overwritten (if they seemingly match). sw_get_src.Stop(); if (config.DryRun) { - Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", "Would write {0} bytes of {1} to {2}", s_src.Length, f.Name, b_dst.DisplayName); + Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", + "Would write {0} bytes of {1} to {2}", + Duplicati.Library.Utility.Utility.FormatSizeString(s_src.Length), + f.Name, b_dst.DisplayName); } else { @@ -604,7 +607,11 @@ destination will be verified before being overwritten (if they seemingly match). if (config.Progress) Console.Write($"\rRenaming: {i}/{n}"); - Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", "Renaming {0} to {1}.{0} by deleting and re-uploading {2} bytes to {3}", f.Name, prefix, downloaded.Length, sb.DisplayName); + Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", + "Renaming {0} to {1}.{0} by deleting and re-uploading {2} bytes to {3}", + f.Name, prefix, + Duplicati.Library.Utility.Utility.FormatSizeString(downloaded.Length), + sb.DisplayName); try { @@ -614,7 +621,11 @@ destination will be verified before being overwritten (if they seemingly match). if (config.DryRun) { - Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", "Would rename {0} to {1}.{0} by deleting and re-uploading {2} bytes to {3}", f.Name, prefix, downloaded.Length, sb.DisplayName); + Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", + "Would rename {0} to {1}.{0} by deleting and re-uploading {2} bytes to {3}", + f.Name, prefix, + Duplicati.Library.Utility.Utility.FormatSizeString(downloaded.Length), + sb.DisplayName); } else { @@ -721,7 +732,11 @@ destination will be verified before being overwritten (if they seemingly match). if (config.Progress) Console.Write($"\rVerifying: {i}/{n}"); - Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", "Verifying {0} by downloading and comparing {1} bytes from {2} and {3}", f.Name, s_src.Length, b_dst.DisplayName, b_src.DisplayName); + Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", + "Verifying {0} by downloading and comparing {1} bytes from {2} and {3}", + f.Name, + Duplicati.Library.Utility.Utility.FormatSizeString(s_src.Length), + b_dst.DisplayName, b_src.DisplayName); try { From 4d4b6cf12246801c3fd84dd08154c80033ccd0a9 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 7 Feb 2025 09:51:24 +0100 Subject: [PATCH 122/362] Human readable formatting of elapsed milliseconds --- Tools/RemoteSynchronization/Program.cs | 50 ++++++++++++++++++++------ 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 1e0e6f9eb..0564ff010 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -413,7 +413,12 @@ destination will be verified before being overwritten (if they seemingly match). if (config.Progress) Console.WriteLine($"\rCopying: {n}/{n}"); - Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", "Copy | Get source: {0} ms, Put destination: {1} ms, Get destination: {2} ms, Get compare: {3} ms", sw_get_src.ElapsedMilliseconds, sw_put_dst.ElapsedMilliseconds, sw_get_dst.ElapsedMilliseconds, sw_get_cmp.ElapsedMilliseconds); + Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", + "Copy | Get source: {0} ms, Put destination: {1} ms, Get destination: {2} ms, Get compare: {3} ms", + TimeSpan.FromMilliseconds(sw_get_src.ElapsedMilliseconds), + TimeSpan.FromMilliseconds(sw_put_dst.ElapsedMilliseconds), + TimeSpan.FromMilliseconds(sw_get_dst.ElapsedMilliseconds), + TimeSpan.FromMilliseconds(sw_get_cmp.ElapsedMilliseconds)); return (successful_copies, errors); } @@ -462,7 +467,9 @@ destination will be verified before being overwritten (if they seemingly match). } sw.Stop(); - Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", "Delete: {0} ms", sw.ElapsedMilliseconds); + Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", + "Delete: {0} ms", + TimeSpan.FromMilliseconds(sw.ElapsedMilliseconds)); if (config.Progress) Console.WriteLine($"\rDeleting: {n}/{n}"); @@ -501,11 +508,15 @@ destination will be verified before being overwritten (if they seemingly match). sw.Restart(); var files_src = b_src.List(); sw.Stop(); - Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", "Prepare | List source: {0} ms", sw.ElapsedMilliseconds); + Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", + "Prepare | List source: {0} ms", + TimeSpan.FromMilliseconds(sw.ElapsedMilliseconds)); sw.Restart(); var files_dst = b_dst.List(); sw.Stop(); - Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", "Prepare | List destination: {0} ms", sw.ElapsedMilliseconds); + Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", + "Prepare | List destination: {0} ms", + TimeSpan.FromMilliseconds(sw.ElapsedMilliseconds)); // Shortcut for force if (config.Force) @@ -523,7 +534,9 @@ destination will be verified before being overwritten (if they seemingly match). var lookup_src = files_src.ToDictionary(x => x.Name); var lookup_dst = files_dst.ToDictionary(x => x.Name); sw.Stop(); - Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", "Prepare | Build lookup for source and destination: {0} ms", sw.ElapsedMilliseconds); + Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", + "Prepare | Build lookup for source and destination: {0} ms", + TimeSpan.FromMilliseconds(sw.ElapsedMilliseconds)); var to_copy = new List(); var to_delete = new HashSet(); @@ -554,7 +567,9 @@ destination will be verified before being overwritten (if they seemingly match). } } sw.Stop(); - Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", "Prepare | Check the files that are present in source against destination: {0} ms", sw.ElapsedMilliseconds); + Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", + "Prepare | Check the files that are present in source against destination: {0} ms", + TimeSpan.FromMilliseconds(sw.ElapsedMilliseconds)); // Find all of the files in dst that are not in src sw.Start(); @@ -569,12 +584,16 @@ destination will be verified before being overwritten (if they seemingly match). } } sw.Stop(); - Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", "Prepare | Check the files that are present in destination against source: {0} ms", sw.ElapsedMilliseconds); + Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", + "Prepare | Check the files that are present in destination against source: {0} ms", + TimeSpan.FromMilliseconds(sw.ElapsedMilliseconds)); sw.Restart(); var to_delete_lookedup = to_delete.Select(x => lookup_dst[x]).ToList(); sw.Stop(); - Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", "Prepare | Lookup the files to delete: {0} ms", sw.ElapsedMilliseconds); + Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", + "Prepare | Lookup the files to delete: {0} ms", + TimeSpan.FromMilliseconds(sw.ElapsedMilliseconds)); return (to_copy, to_delete_lookedup, to_verify); } @@ -656,7 +675,11 @@ destination will be verified before being overwritten (if they seemingly match). i++; } - Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", "Rename | Get source: {0} ms, Put destination: {1} ms, Delete source: {2} ms", sw_get_src.ElapsedMilliseconds, sw_put_dst.ElapsedMilliseconds, sw_del_src.ElapsedMilliseconds); + Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", + "Rename | Get source: {0} ms, Put destination: {1} ms, Delete source: {2} ms", + TimeSpan.FromMilliseconds(sw_get_src.ElapsedMilliseconds), + TimeSpan.FromMilliseconds(sw_put_dst.ElapsedMilliseconds), + TimeSpan.FromMilliseconds(sw_del_src.ElapsedMilliseconds)); if (config.Progress) Console.WriteLine($"\rRenaming: {n}/{n}"); @@ -697,7 +720,9 @@ destination will be verified before being overwritten (if they seemingly match). i++; } - Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", "Rename: {0} ms", sw.ElapsedMilliseconds); + Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", + "Rename: {0} ms", + TimeSpan.FromMilliseconds(sw.ElapsedMilliseconds)); if (config.Progress) Console.WriteLine($"\rRenaming: {n}/{n}"); @@ -774,7 +799,10 @@ destination will be verified before being overwritten (if they seemingly match). i++; } - Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", "Verify | Get: {0} ms, Compare: {1} ms", sw_get.ElapsedMilliseconds, sw_cmp.ElapsedMilliseconds); + Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", + "Verify | Get: {0} ms, Compare: {1} ms", + TimeSpan.FromMilliseconds(sw_get.ElapsedMilliseconds), + TimeSpan.FromMilliseconds(sw_cmp.ElapsedMilliseconds)); if (config.Progress) Console.WriteLine($"\rVerifying: {n}/{n}"); From bb60db47f0533ddfc90c829bcc1c401724585279 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 7 Feb 2025 09:55:13 +0100 Subject: [PATCH 123/362] Reformatted long logging lines --- Tools/RemoteSynchronization/Program.cs | 83 ++++++++++++++++++-------- 1 file changed, 58 insertions(+), 25 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 0564ff010..390130f91 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -228,13 +228,18 @@ destination will be verified before being overwritten (if they seemingly match). if (not_verified.Any()) { - Duplicati.Library.Logging.Log.WriteWarningMessage(LOGTAG, "rsync", null, "{0} files failed verification. They will be deleted and copied again.", not_verified.Count()); + Duplicati.Library.Logging.Log.WriteWarningMessage(LOGTAG, "rsync", null, + "{0} files failed verification. They will be deleted and copied again.", + not_verified.Count()); to_delete = to_delete.Concat(not_verified); to_copy = to_copy.Concat(not_verified); } } - Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "The remote synchronization plan is to {0} {1} files from {2}, then copy {3} files from {4} to {2}.", config.Retention ? "rename" : "delete", to_delete.Count(), b2s.DisplayName, to_copy.Count(), b1s.DisplayName); + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "The remote synchronization plan is to {0} {1} files from {2}, then copy {3} files from {4} to {2}.", + config.Retention ? "rename" : "delete", + to_delete.Count(), b2s.DisplayName, to_copy.Count(), b1s.DisplayName); // As this is a potentially destructive operation, ask for confirmation if (!config.Confirm) @@ -259,30 +264,37 @@ destination will be verified before being overwritten (if they seemingly match). if (config.Retention) { renamed = await RenameAsync(b2, to_delete, config); - Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Renamed {0} files in {1}", renamed, b2s.DisplayName); + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "Renamed {0} files in {1}", renamed, b2s.DisplayName); } else { deleted = await DeleteAsync(b2s, to_delete, config); - Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Deleted {0} files from {1}", deleted, b2s.DisplayName); + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "Deleted {0} files from {1}", deleted, b2s.DisplayName); } // Copy the files var (copied, copy_errors) = await CopyAsync(b1s, b2s, to_copy, config); - Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Copied {0} files from {1} to {2}", copied, b1s, b2s); + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "Copied {0} files from {1} to {2}", copied, b1s, b2s); // If there are still errors, retry a few times if (copy_errors.Any()) { - Duplicati.Library.Logging.Log.WriteWarningMessage(LOGTAG, "rsync", null, "Could not copy {0} files.", copy_errors.Count()); + Duplicati.Library.Logging.Log.WriteWarningMessage(LOGTAG, "rsync", null, + "Could not copy {0} files.", copy_errors.Count()); if (config.Retry > 0) { - Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Retrying {0} more times to copy the {1} files that failed", config.Retry, copy_errors.Count()); + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "Retrying {0} more times to copy the {1} files that failed", + config.Retry, copy_errors.Count()); for (int i = 0; i < config.Retry; i++) { await Task.Delay(5000); // Wait 5 seconds before retrying (copied, copy_errors) = await CopyAsync(b1s, b2s, copy_errors, config); - Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Copied {0} files from {1} to {2}", copied, b1s, b2s); + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "Copied {0} files from {1} to {2}", copied, b1s, b2s); if (!copy_errors.Any()) break; } @@ -290,24 +302,33 @@ destination will be verified before being overwritten (if they seemingly match). if (copy_errors.Any()) { - Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", null, "Could not copy {0} files. Not retrying any more.", copy_errors.Count()); + Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", null, + "Could not copy {0} files. Not retrying any more.", copy_errors.Count()); return copy_errors.Count(); } } // Results reporting if (verified > 0) - Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Verified {0} files in {1} that didn't need to be copied", verified, b2s.DisplayName); + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "Verified {0} files in {1} that didn't need to be copied", + verified, b2s.DisplayName); if (failed_verify > 0) - Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Failed to verify {0} files in {1}, which were then attempted to be copied", failed_verify, b2s.DisplayName); + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "Failed to verify {0} files in {1}, which were then attempted to be copied", + failed_verify, b2s.DisplayName); if (copied > 0) - Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Copied {0} files from {1} to {2}", copied, b1s.DisplayName, b2s.DisplayName); + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "Copied {0} files from {1} to {2}", copied, b1s.DisplayName, b2s.DisplayName); if (deleted > 0) - Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Deleted {0} files from {1}", deleted, b2s.DisplayName); + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "Deleted {0} files from {1}", deleted, b2s.DisplayName); if (renamed > 0) - Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Renamed {0} files in {1}", renamed, b2s.DisplayName); + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "Renamed {0} files in {1}", renamed, b2s.DisplayName); - Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Remote synchronization completed successfully"); + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "Remote synchronization completed successfully"); return 0; } @@ -344,7 +365,8 @@ destination will be verified before being overwritten (if they seemingly match). if (config.Progress) Console.Write($"\rCopying: {i}/{n}"); - Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", "Copying {0} from {1} to {2}", f.Name, b_src.DisplayName, b_dst.DisplayName); + Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", + "Copying {0} from {1} to {2}", f.Name, b_src.DisplayName, b_dst.DisplayName); try { @@ -393,7 +415,8 @@ destination will be verified before being overwritten (if they seemingly match). } catch (Exception e) { - Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e, "Error copying {0}: {1}", f.Name, e.Message); + Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e, + "Error copying {0}: {1}", f.Name, e.Message); errors.Add(f); } finally @@ -444,13 +467,15 @@ destination will be verified before being overwritten (if they seemingly match). Console.Write($"\rDeleting: {i}/{n}"); } - Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", "Deleting {0} from {1}", f.Name, b.DisplayName); + Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", + "Deleting {0} from {1}", f.Name, b.DisplayName); try { if (config.DryRun) { - Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", "Would delete {0} from {1}", f.Name, b.DisplayName); + Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", + "Would delete {0} from {1}", f.Name, b.DisplayName); } else { @@ -460,7 +485,8 @@ destination will be verified before being overwritten (if they seemingly match). } catch (Exception e) { - Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e, "Error deleting {0}: {1}", f.Name, e.Message); + Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e, + "Error deleting {0}: {1}", f.Name, e.Message); } i++; @@ -659,7 +685,8 @@ destination will be verified before being overwritten (if they seemingly match). } catch (Exception e) { - Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e, "Error renaming {0}: {1}", f.Name, e.Message); + Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e, + "Error renaming {0}: {1}", f.Name, e.Message); } finally { @@ -695,13 +722,17 @@ destination will be verified before being overwritten (if they seemingly match). if (config.Progress) Console.Write($"\rRenaming: {i}/{n}"); - Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", "Renaming {0} to {1}.{0} by calling Rename on {2}", f.Name, prefix, rb.DisplayName); + Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", + "Renaming {0} to {1}.{0} by calling Rename on {2}", + f.Name, prefix, rb.DisplayName); try { if (config.DryRun) { - Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", "Would rename {0} to {1}.{0} by calling Rename on {2}", f.Name, prefix, rb.DisplayName); + Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", + "Would rename {0} to {1}.{0} by calling Rename on {2}", + f.Name, prefix, rb.DisplayName); } else { @@ -713,7 +744,8 @@ destination will be verified before being overwritten (if they seemingly match). } catch (Exception e) { - Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e, "Error renaming {0}: {1}", f.Name, e.Message); + Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e, + "Error renaming {0}: {1}", f.Name, e.Message); sw.Stop(); } @@ -783,7 +815,8 @@ destination will be verified before being overwritten (if they seemingly match). catch (Exception e) { errors.Add(f); - Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e, "Error during verification of {0}: {1}", f.Name, e.Message); + Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e, + "Error during verification of {0}: {1}", f.Name, e.Message); } finally { From 2e134cffebe88b8ae59be35dc365c8e930b8be59 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 7 Feb 2025 09:56:54 +0100 Subject: [PATCH 124/362] Changed to UTC time for timestamps --- Tools/RemoteSynchronization/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 390130f91..3a0f216ff 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -635,7 +635,7 @@ destination will be verified before being overwritten (if they seemingly match). private static async Task RenameAsync(IBackend b, IEnumerable files, Config config) { long successful_renames = 0; - string prefix = $"{System.DateTime.Now:yyyyMMddHHmmss}.old"; + string prefix = $"{System.DateTime.UtcNow:yyyyMMddHHmmss}.old"; using var downloaded = new MemoryStream(); long i = 0, n = files.Count(); From 4144b3a25e15ebec66f7646b4a448f82a53cfd1f Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 7 Feb 2025 10:29:33 +0100 Subject: [PATCH 125/362] Changed to the log timer instead of stopwatch where applicable --- Tools/RemoteSynchronization/Program.cs | 109 ++++++++++--------------- 1 file changed, 42 insertions(+), 67 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 3a0f216ff..3d3126714 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -457,8 +457,7 @@ destination will be verified before being overwritten (if they seemingly match). { long successful_deletes = 0; long i = 0, n = files.Count(); - var sw = new System.Diagnostics.Stopwatch(); - sw.Restart(); + using var timer = new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Delete operation"); foreach (var f in files) { @@ -492,11 +491,6 @@ destination will be verified before being overwritten (if they seemingly match). i++; } - sw.Stop(); - Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", - "Delete: {0} ms", - TimeSpan.FromMilliseconds(sw.ElapsedMilliseconds)); - if (config.Progress) Console.WriteLine($"\rDeleting: {n}/{n}"); return successful_deletes; @@ -529,20 +523,13 @@ destination will be verified before being overwritten (if they seemingly match). /// A tuple of Lists each holding the files to copy, delete and verify. private static (IEnumerable, IEnumerable, IEnumerable) PrepareFileLists(IStreamingBackend b_src, IStreamingBackend b_dst, Config config) { - var sw = new System.Diagnostics.Stopwatch(); + IEnumerable files_src, files_dst; - sw.Restart(); - var files_src = b_src.List(); - sw.Stop(); - Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", - "Prepare | List source: {0} ms", - TimeSpan.FromMilliseconds(sw.ElapsedMilliseconds)); - sw.Restart(); - var files_dst = b_dst.List(); - sw.Stop(); - Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", - "Prepare | List destination: {0} ms", - TimeSpan.FromMilliseconds(sw.ElapsedMilliseconds)); + using (new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Prepare | List source")) + files_src = b_src.List(); + + using (new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Prepare | List destination")) + files_dst = b_dst.List(); // Shortcut for force if (config.Force) @@ -556,70 +543,58 @@ destination will be verified before being overwritten (if they seemingly match). return (files_src, [], []); } - sw.Restart(); - var lookup_src = files_src.ToDictionary(x => x.Name); - var lookup_dst = files_dst.ToDictionary(x => x.Name); - sw.Stop(); - Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", - "Prepare | Build lookup for source and destination: {0} ms", - TimeSpan.FromMilliseconds(sw.ElapsedMilliseconds)); + Dictionary lookup_src, lookup_dst; + using (new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Prepare | Build lookup for source and destination")) + { + lookup_src = files_src.ToDictionary(x => x.Name); + lookup_dst = files_dst.ToDictionary(x => x.Name); + } var to_copy = new List(); var to_delete = new HashSet(); var to_verify = new List(); // Find all of the files in src that are not in dst, have a different size or have a more recent modification date - sw.Restart(); - foreach (var f_src in files_src) - { - if (lookup_dst.TryGetValue(f_src.Name, out var f_dst)) + using (new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Prepare | Check the files that are present in source against destination")) + foreach (var f_src in files_src) { - if (f_src.Size != f_dst.Size || f_src.LastModification > f_dst.LastModification) + if (lookup_dst.TryGetValue(f_src.Name, out var f_dst)) { - // The file is different, so we need to copy it - to_copy.Add(f_src); - to_delete.Add(f_dst.Name); + if (f_src.Size != f_dst.Size || f_src.LastModification > f_dst.LastModification) + { + // The file is different, so we need to copy it + to_copy.Add(f_src); + to_delete.Add(f_dst.Name); + } + else + { + // The file seems to be the same, so we need to verify it if the user wants to + to_verify.Add(f_src); + } } else { - // The file seems to be the same, so we need to verify it if the user wants to - to_verify.Add(f_src); + // The file is not in the destination, so we need to copy it + to_copy.Add(f_src); } } - else - { - // The file is not in the destination, so we need to copy it - to_copy.Add(f_src); - } - } - sw.Stop(); - Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", - "Prepare | Check the files that are present in source against destination: {0} ms", - TimeSpan.FromMilliseconds(sw.ElapsedMilliseconds)); // Find all of the files in dst that are not in src - sw.Start(); - foreach (var f_dst in files_dst) - { - if (to_delete.Contains(f_dst.Name)) - continue; - - if (!lookup_src.ContainsKey(f_dst.Name)) + using (new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Prepare | Check the files that are present in destination against source")) + foreach (var f_dst in files_dst) { - to_delete.Add(f_dst.Name); - } - } - sw.Stop(); - Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", - "Prepare | Check the files that are present in destination against source: {0} ms", - TimeSpan.FromMilliseconds(sw.ElapsedMilliseconds)); + if (to_delete.Contains(f_dst.Name)) + continue; - sw.Restart(); - var to_delete_lookedup = to_delete.Select(x => lookup_dst[x]).ToList(); - sw.Stop(); - Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", - "Prepare | Lookup the files to delete: {0} ms", - TimeSpan.FromMilliseconds(sw.ElapsedMilliseconds)); + if (!lookup_src.ContainsKey(f_dst.Name)) + { + to_delete.Add(f_dst.Name); + } + } + + List to_delete_lookedup; + using (new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Prepare | Lookup the files to delete")) + to_delete_lookedup = [.. to_delete.Select(x => lookup_dst[x])]; return (to_copy, to_delete_lookedup, to_verify); } From 728a75a902c36e9f0bad7b1be6b8e5ca1cd402a2 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 7 Feb 2025 10:30:39 +0100 Subject: [PATCH 126/362] One stopwatch wasn't ensured always stopped in a try block --- Tools/RemoteSynchronization/Program.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 3d3126714..3ac9c0eef 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -721,6 +721,10 @@ destination will be verified before being overwritten (if they seemingly match). { Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e, "Error renaming {0}: {1}", f.Name, e.Message); + } + finally + { + // Ensure the timer is stopped sw.Stop(); } From b4a5820cf47d5114dcc3e775e2f5f619b535d62d Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 7 Feb 2025 14:06:57 +0100 Subject: [PATCH 127/362] Fixed incorrect dispose logic --- Duplicati/Library/Compression/ZipCompression/FileArchiveZip.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Compression/ZipCompression/FileArchiveZip.cs b/Duplicati/Library/Compression/ZipCompression/FileArchiveZip.cs index e2b8a465d..5cb89a96c 100644 --- a/Duplicati/Library/Compression/ZipCompression/FileArchiveZip.cs +++ b/Duplicati/Library/Compression/ZipCompression/FileArchiveZip.cs @@ -231,7 +231,7 @@ namespace Duplicati.Library.Compression.ZipCompression public void Dispose() { - if (m_fallbackArchive != null) + if (m_fallbackArchive == null) m_archive.Dispose(); m_fallbackArchive?.Dispose(); } From a8dea5c352eef1e7f1a510cdba24c12ac67a6357 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 7 Feb 2025 16:10:30 +0100 Subject: [PATCH 128/362] Fixed an issue with dropping existing permissions on Windows. When setting the folder permissions on an existing folder, it would remove the permissions on items inside the folder. This would cause the files and subfolders to have zero permissions, and reject any interaction. --- Duplicati/Library/Common/IO/SystemIOWindows.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Duplicati/Library/Common/IO/SystemIOWindows.cs b/Duplicati/Library/Common/IO/SystemIOWindows.cs index 3566de45a..72bafb0af 100644 --- a/Duplicati/Library/Common/IO/SystemIOWindows.cs +++ b/Duplicati/Library/Common/IO/SystemIOWindows.cs @@ -729,12 +729,16 @@ namespace Duplicati.Library.Common.IO security.AddAccessRule(new FileSystemAccessRule( CURRENT_USERNAME, FileSystemRights.FullControl, + InheritanceFlags.ContainerInherit | InheritanceFlags.ObjectInherit, // Apply to subfolders & files + PropagationFlags.None, // Keeps inheritance settings intact AccessControlType.Allow )); security.AddAccessRule(new FileSystemAccessRule( LOCAL_SYSTEM_NAME, FileSystemRights.FullControl, + InheritanceFlags.ContainerInherit | InheritanceFlags.ObjectInherit, // Apply to subfolders & files + PropagationFlags.None, // Keeps inheritance settings intact AccessControlType.Allow )); From 14207848ab594f774982bd8637577a8821baf3d3 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 7 Feb 2025 19:19:19 +0100 Subject: [PATCH 129/362] Fix test instability --- Duplicati/UnitTest/Issue4312.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Duplicati/UnitTest/Issue4312.cs b/Duplicati/UnitTest/Issue4312.cs index dea2f18a5..cbfa60e63 100644 --- a/Duplicati/UnitTest/Issue4312.cs +++ b/Duplicati/UnitTest/Issue4312.cs @@ -22,6 +22,7 @@ using System; using System.IO; using System.Linq; +using System.Threading; using NUnit.Framework; namespace Duplicati.UnitTest; @@ -100,6 +101,9 @@ public class Issue4312 : BasicSetupHelper // Remove the database File.Delete(testopts["dbpath"]); + // Get a second between the two backups + Thread.Sleep(2000); + // Recreate var rr = c.Repair(); Assert.AreEqual(0, rr.Errors.Count()); From 5c4a0594a547c0e2112aa0dd96aa4805b212b3d9 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sat, 8 Feb 2025 17:30:27 +0100 Subject: [PATCH 130/362] Improved TrayIcon detached. Implemented support for disconneted state with TrayIcon. Added some notification support (Windows and Linux). Changed some logic to prevent hangs on clicks. --- .../icons/linux/normal-disconnected.png | Bin 0 -> 10622 bytes .../icons/macos/dark/normal-disconnected.png | Bin 0 -> 10622 bytes .../icons/macos/light/normal-disconnected.png | Bin 0 -> 5635 bytes .../icons/windows/normal-disconnected.png | Bin 0 -> 12744 bytes .../Duplicati.GUI.TrayIcon/AvaloniaRunner.cs | 104 +++++++++++++++--- .../Duplicati.GUI.TrayIcon.csproj | 1 + .../GUI/Duplicati.GUI.TrayIcon/Program.cs | 30 +++-- .../Duplicati.GUI.TrayIcon/TrayIconBase.cs | 99 ++++++++++++----- .../Duplicati.Server.Serialization/Enums.cs | 41 +++---- Duplicati/Server/Program.cs | 2 +- 10 files changed, 205 insertions(+), 72 deletions(-) create mode 100644 Duplicati/GUI/Duplicati.GUI.TrayIcon/Assets/icons/linux/normal-disconnected.png create mode 100644 Duplicati/GUI/Duplicati.GUI.TrayIcon/Assets/icons/macos/dark/normal-disconnected.png create mode 100644 Duplicati/GUI/Duplicati.GUI.TrayIcon/Assets/icons/macos/light/normal-disconnected.png create mode 100644 Duplicati/GUI/Duplicati.GUI.TrayIcon/Assets/icons/windows/normal-disconnected.png diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Assets/icons/linux/normal-disconnected.png b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Assets/icons/linux/normal-disconnected.png new file mode 100644 index 0000000000000000000000000000000000000000..1c9813b2d1c36810e648938d53af35c89d45d90e GIT binary patch literal 10622 zcmeAS@N?(olHy`uVBq!ia0y~yU}OMc4mJh`hM1xiX$%Z}1yvytB|(Yh3I#>^X_+~x z3MG{VsS2qTnQ06R6}O(wiY`;tUDtA7(v`n)b;Co3rma>V*#ll30`JZ1acqT=$O)7R^4%HH?yJ^!cW`1wPh zBMAuHDJQFVM^{EK0<9h$PwlwZ6|NfX0aeqHe zzRqQPWWpa#%M6K!Ew%mHI?HA}?wgw2qrL2Rq_^p}9h3YOQ>Pc5dU#}3)uk&zKHBf5 z{4lXQuX^YAx9Z)ZQs&nV%rrEJc+mB`$Upzz%h#71&qazYI`C?K($PJOSm$I$JbM#= zPD$~h&BGOU80`1ntDhLbc5+T?VxCUHkDmrdbRM!spSKhmjC?C^V*d!wUYd{htf>8 zEcnIp*CO-CvF+dMD@*=O7ylQ|zvT(T{%h&?j=wi@ynW9yYtFwFdiSrhJPs4~bKU&* zOzV|7$MUzH^K@Oi_srbcOE&XAH{0KSMzl5l`#M$Mw@zZojB#6wC*SL{HheR!#Du>f z6bN95tG$v3BM zY}fV;Ud389-%yJ)@<_RuQeUb@iu>`Lt3R%)c7?{hdc8uu*zl(1i#G|Thu<1zcjV@+ zsysRO={4V~Y^8Nk$*0VoInUCrTE_ft``Qz$uSj&bto!W}FZXoZ1J_xJ|(Vcwg=IeWNBN&+SYK6>|T30JtZ(SBSW#*h$+gCHRG;InGUE+Ds z_x;3!TYSTgaXr>3c9j*p(Xgts7er0FN%X#Ih>eybd7~ubXeb&+4r{Bg>hXvXvNX7cmq$#y~ow7*)@6s zruK*kO@7r^`Am4x+<3vfEb7H=;u=fO+|Agosrqxm&Xp5Y(nO-8qfH%m9sRPv zGMHumRgH|z1@<%9LLW!V?bPou5t}0!`mk`l@Po|fmoAn2trc0e^;|=1gS4qZ((arc z%9+zXY3nVFm^L}{^gOHcU6aqB*cbk4>h#sFH_n}$+jMR5-UCM-U$gX({5RFOz3+0$ z8cw0>Dh~@Tr$A$#4eqMQm@Dg`2m0VYJS&u~e{k~4+VCsiBfacNi_5I{H}br0s4!h^Rx2^duAUt9HQnS=+cd6U z7kU5R)O%3YwOeSu;bWF56^lw(qOWWb`72%f{alFAq05EQ#x2`gf10ITYdEZ>y7|b@ zEUEQr5A9|jW9+yeyv=b5Q~4P)-#3QW0?+iS27F-Gt94iX9*jhWKR3>v8udY6qn>kx z)G?OXHml30JUOkU`QUW&g56y*H!}Aes^ofdr_01#>DCTC<+Xp@);a4b+RM&Wm0$en z*JgPc?UNP1CszNHF7DVY5+tYf)?#y{gj&W4sV(NWxPAy1ICro|bRErb3UX8}nsE0W z@AHSBJ4I5S`G>Q2dabR@n0U6<^sw378C542JF9!mzNFn05be9cEz!L_W{vfZ2ahkN zTQdrIz2Nqr&M)JziR0^5o?<+vZGBtHG4?_d7@{CeLBw;GkFRDRw&)yr!4 zIq$jMa#E(bs(S=ezt7dGSDmj zTpO}Uwq#zRx|AQ+G+E{`(!htN!fu+^)+!mzp!WeA?P%r|}9* zJ>*o}m!8L{nHduGt1-CVX5yyII(x;1)yJKj=Ioz(^?cvEy=z|0={0SPDp}taHv8=? z=S&-ewf9S}P3+5h-MsN!m`Fj~xj9=ewawhjS5OX zQ{*ygHb04ryzj)%qvbrw&v?&@nHyhZwTks6F|rCz*>BiXdC5Ys+M$)DD0g?Tw%Fq% zksmv`wF}?eN!V>=a_J9O0b8xAT5D;Kd$6%EtDU;B=`^FJcVd0bqVt7Ti*!u#X692C zs?vT{>aQ~?S8USU+}oE`8e8;N-(=W%U--4|k_~ zA7CxG@vSe&qagOV)IK#)f7X4=KkBOOU@V+r>}bT^rMLL1_|6jtZ{+#uwRV+<7YWPq z{>$@9yKTB@l|gQ>(d}hVJgvIk7L>6}JK~|hG~)=*V#jS?dE1vNZoQa&d+!BfjtBkI zr2BU>&p#2XGJW!zWczQ4GavhwmdAHr{1N#4+Y+mH);Ae%R4n$9IPu&g@x-L7?$Oyw z&R6xdlYeBU-<~4EWpl~mKzb$2E?C;q4oaaV{_0NZv%ugThu}NPLJ!}5V zxC@Lo0wmODEictd?bBrX#TdJXdrGFq{gU3>&bJKK7_asa`}-tq(hA12$>CgH{#UlD z&7aA{Wx&O;rzczaSZ_n)+p6z3Cog5X(qyn}e_?F%ZQVs$+ji#Oez$4H+z{=H5A)`- zTb`NQcjns{hV|1XyDT&hx^$XpbE?kcCCz6G-z@8mI2yn_pY+fn!UQ4 zCo)IwpO`|4e4wwB^|b!-?LoJhZFWc$_?WC;dHc#i_v2!7EPMhYq*gd~z3DVOxZMBl zzp{cDKD+kcH{JexKi=M$Vf$VGxPk1xvmd`7zSEns$NlQX60tuV0p4Oq(o|}0Fx$NQ zE%!S;?Luuy^ZgkI^=FxVf5~BZ;|ZlJgpc1Q(Rx<9Nagrg6*5dg6GQfOKdITZuE1HZAzJ?e~QOkC}>Ynfr`%{1O+a$QU>(tb5AyS;gKa=*p)*o@tOL{A$8&kS)llDKj){J#3~ z#}7|tYH=G1Hz+b!-b?G&;GHVMYHz^ABXwBaq@X{)ZNj6Di|;!`U4^x$9?A>(p73pX z!-Pcs1<#tNYBeQh@rQLqbx$oiE}k8v_x;y~|F$L{FZ}kOdtqggWuAZkY|ZN2oo$}4 zzuj1Czsqh)z{gEH=O_Loc?oKlQk0$Ix$7^M0=T^vIyZoQpbT@reA?|b{==UX{NoH#t5Bpz&;#>}~G zkEV*qF@X&MuisvX>+t`gv2^jhqFcrPG@NZ4+O3}Oe0U|g_n&Lj`>y7I4TKnXb0}4;JhdqnBe?G2wo#k~V z#TEf44#I?rw43jQIT6{vnVlOBaqs)Zc+)|A0s95s69L((Q_>z3KT!F=TEOk4VS zYW#X2D*24OYC}Tot^2p7{wQV%GQMHI#(4hs-OvdPf4KjBJ!8IFnqRz zWrEZ}^8>O69v?8x5`4&1&p3_2Uo!ce7sE;EfA`#t`MBD z_L=}^f^I>-;j&1k5B7gz8{>uNZ_{>BI?kGBc&tW^@rU?7n+N3wGPbsPFx_LxGdlLi zmFa`spPOs{{{5K6JJDqitB>KantSu)FDWGaJ~ZF4rfu3G{YkwS59F`i8o7(-vrzbL*3}c2f~rxO3;sY172kY_+oBR1x8tIBlAnn;YB1 zf*E`E*!=qT_Vj^9W=$=vMehA_b?+}pJvg^b&vY8YExm@L2b}A2R&U!jP1d?BAUgVV zr?C2!ADTXXetfO1tv>T?X1-p(|K9quc1b;vI*dES|7jkmue_Ps<)V~xZ_mtEud@8+ z*{pPXzr;wwJWuBGa({K1ZzYZyJTtO-Zt5{w@Yl;fh_$IoIR5hT@|!yfl|w^AO-xN! z9%Ob%OH&h8^GUe2CbH!1t*fig{*l~qP@>p}=|#m4-VKs7o;l`fX=&ZqQK)<|BcxZ> zVbi8fJAOW!J<}+)D}443S%c4Bj~RsyOm(g@|Il>1R9aeEQ(yo2hlhu=QXK2*?Ck$M z)zeGsfpX{QLcWb>Y!aO)agYdwVLEEM2-YbW`h(h6n88_0GnO`ezU4wblRp z^ptCVhH>@xcd19aM2{pHZueTYecojchVcJi{!0~fM^FEI-sUq8JD*Iz;bj(ktG<3J zt(LF-BKY*_)4XYa_HEr3z_i2a|C6xYYa%zRDbD|se5@z&+#E}Z#d7ncdF1WpxXag0 zSsA?iSDosHvbRy&Pc3EqufZ7eQ+T(0!EWiTt5&ZTHhN&5c1Ggaxw+n+o}8Q0&rf5C z;rQQQ_eHtK@J`k1wfnpio}Zh$=i@Qyys5h{M^&$7DX9D>ccLWf$%%=|d9!xkeQ4jV z@*wWV`H38yd%j%qHZe2X6{ywyBYLGu!|4YVOYiTj+kgN2Pkp}qDi7*@2&*!W5Y0+&D1PI6cPpx+=pt{a-uJX3q7UZT555hUIG?PG`BpU$3Mh(CJd~ zbZYn=jndgHA5z;M6`ZSA5h^U+a_#E}Z>D$c_JTqcf4^S;`KVie&NRt_Jii8(3$wVF zvzq1Kdvm+$Ute3iP=WoQI`0K5S9ae2e{bq^Cf{icdRaW`Z*FbXRz0N8xkJ8gziL2i z>|3{)L7WfP`xJdTTxYxc{MrxROoeVe5)Y$(=ubJ&!YOQX_xO!fxgK<9sP|Ohwfe zmIomW)!*KHEKOHsZ2d2zx5h+PiP7Q>)0L}Nf9^Xxso~K3OFNfqGb~s?S-vcD>NJ%F zAGXhKYhye9d>;!#-@P*hZ*FXCdAmbJ;LeBLCGToonHttx{Cw!4GU>wH=`2l>bq9Ji z58hQ_c=7B>`{ai||4nlEaESk7^kE@~Ce{a!CP%9>w%*TYT*1wdV)66g=JfMwXZ4jh zS=9gN&8?m(i$6Y6;2`pBvEFyKB4lE%|T>DuRn6wyM!r3FF(KU5h612{GQ2sLo5po+G}M~{*~7CKHn@aN}eQIJDInd088 z2HDO)k?oVbA8frI_j%2~eYL+cGBN_3y&04wCGBb~il3kRy5?Vppz@?oSDy$q1pKg# zx>Rj=@>^CBr-Q~HyViK22@Ea=4(oGzFep8EIGsg>LGZ%|ZziDzj*1_ujGPWEHh(5H zuqZIe)p5aF&jky^TOV<1Sgz~TQllKVH7ybR)#wNe3@lkw2L=7+(> zj47hpVICl(qJHTAF$e4C0qbvQC(BxLv_0tgl5*EDJGVZ8BcS47xV?h(3#Km&yBH-o zO*1r(+}xZl`cGxjfhJb&AFsmqzuLvZ5Lfk5b1wc^)5@eXKt0 zY|EM)FJHPe$&JC0uUR3kun+wm@US4_;dB-ur|EjJLGBFKZoOW+^}qBA)_*&< z*3bX+V84W#kY?yqleU(f7po6vsn zeCFRH_0m7&OD;wCU5_oFdq|&C#f6EP-AYbtHE*RrP4e2k!Z)t}>YVSVzHG-1u)Ru+ z4-dCn>jnxJS^nAE7~a|cEZ&*n#l5}OmX?-W`&AySW+^HzetB`R`==Pji0u8md$`Xi zB=ATYxjdZCQZu!|`}v&D+6>vbC!%&sRPgSY#!!^k)GLKzf7tD!B9wS@Q|h`n&MQ~0 zeE4$N|MI0vk8alW%iG`UHM(uB!;qaTw_)m!Ikk*uW*9b~Idf*pgoZ=!eajatTh_LG zURBq#XKClZnQLozXI@^mGcZfqA&ULbeBK3KH<}olX-Wl))^nOx$_Pb7;GP;6ZZJc-k*AH zO=L=H>d&G>Wly{u9)5r3bzuF6_QQ#3=jOb0Tlw|*zoc7&{{%iV+QYc;m*2jmgJ5T$D`A%|RZzwdu^AJ2BJrVqfj>b9H5XeSQ7+rMZ_bUE1;g-)~97 zq?SYPAE}mAPZRnrTGwD%{A@?*@t4xaW|ZI7Za8|x{kQFMlvY=b^$%`wy_5qDjD4?l zqPCpazF?~>6W4xop%2D?{{H&+uUE%TjS z@$snm&0VG1U%!5JQ4+k*zfU^2x}8_rEj3kjW5k)yfA`*+(QxSf*%=M^Y zYsu?tXM2nM^`f?9xX<3raP5}ek@O#xo6`P&d3ibK&W?$zR%y+eHA|@K!QqDnr>E%_ zKUI|9UH0}-t9aap-23{astZ0;7pp!<|H1ff&u2e1pBWEM1+W|3kO;oNOq(HlE#p7t z9S6Jp*L8Jqg{_M@*~rXZ^K;7{DP3LNl(e*Th=*X@rn z)rr_};H>7+|9L$>XIoc<&tm_uV)n!B*EUss%?eu^HTB9v_KL4p!((4hTC#NM#)N}R zU0q#M7Kx|{b^Z|h_vlg5_QUsiboM>@Svhlqn6UiU^3+Cu9kJx6r>5@t`|Y-7;fBQ* zH8w`{EOc()6{Ii1WtetGBK|mwOIA=&(4jj!i|g9C{S3+=TcQ=)h(vGD13d~x@)!1=axro%@Vzs5t6>!!aprdEj2Z@NA~qEp5JT_ zULTt=Id>P+hHG`aKkRoDoWCD+`qWhIoBQkQPsvW$oPItiKK}hLt^I%g)VTG@oc#Cq z_s=(*&zC&o{`K|sasT>X!RzI8J~tRX>!1F6*-n>?`RvbjZG3ulb$D=OF@6B z?2$A+Hu+Zc_Po@?ZM>W_Kd1=#TgW_{7M-_n%a$vvefKccT$d=m`SVkzcqz9 zsuhM;uU^f$wMFwO|0Q<$ngcH`EsSfHC$X=dv2XkKTwiB=!+WePV!yOH`E9%$YM=4kv#4^y$R;^XUf~7)3-yFYc`_-(Q|7 zT5x{qufNlU4+tNLzj8CtZ@%5x^Y#Be`&DD;XJ_3oYK73w3@tCY@jM zE3EHm(v<1bH|O8Cn_Hw5xq>ZCBH3^IZtly!mOq}VKifRt??+YL{*1i5XDgS_dvw

Qmas-}+)3>zG`Xpkvi5EN;qqT^Z){Y) z;QsjQ_4xFYlT>AFt3sA9`!LJD!~av{t!vS>;k%g%_Ao8nw$1E=`_hFA8&$oh1;oa# z4GyU|s`j95+nEp9{R}d;4DYY344$>?mRE4Fu>QUuO8b8t)wd{m(jn!t<7naoy$5N@ zXX5YZyoq4Q+q-6y@yCzHpIw*r8s-%g zXC`fwur7a>a(ZU$8(3mB`_(5?)-;d(owXtrq`D%`OGTNS9_V;aQWTaF)RU)t)?metG!% zxR%3-n>KBFac!-1?5+|;Tie)PRYr?fOcqtb%MQJd4PUr$VPjhx+r9e#wdc15$Hl#S z@%;JO+2SuAKiTqE$k!uKSZRj`gOdk?69=aQhaw|~q6&jz3yVU_1cnxYcs~CVg*CHO z7!-r1^z`s7SGAkW(Bcso6LC$Kkz?YjFR!k8GYKfIc=G(YwQivclYo+|q*>04%R;|Z z85Dyi1ZW)j@bK{5X)C8Pw0JnO@k%{8rjX!k#Bt!}>Q<*B53TwxOT`oZOh%;;PsW=mzQ(aempAPn6o|a?k>ZuD;me=|2sCre)%4Tm>E2ZDh*STZf(iT ztND=Y+qK$3=3X~X`(dx3u1ib3#W@$uee&dqLFy?HbFW-)F~&J%K??f%{+?cebL4g# zr=Me4clOVd3SeqE4TPA$-0K;J%;X+Uw*eS7fxaSU~e6@ zWWj=lIdf$E=2$FDS;}Br{VnCggM)opd%oYRp1ZBqtyk*k-SYcyzg3I-Gxn?BeqWVm z^uug<>#<(x+eworA8zKiJ5l@n?(?JG;qr#ZRPL^L*&5E! zH^VTQO;l9W#MD$X(bFd^OziS9-{P3(7Zy6tUA5l6^i{|*|M~B(-kQeWuzd2za(nfL z+O>}vu3K4dRP&i}U|a6(DVtQ%9v$iIo4(P^%ica!nSJ5 z^zK_p2bhn|c(z2e!SGmAc0H?3K~T=MHIX5qp{1`5E(}qV;WI9N=5xHX?A@K2L5}|Z z{^~M(msf}DgGSgkBp%k#(ke=}t&fh#=~?M>?VI8Ae@Sa)S@%5Mk^pHQhTobr!Ls<- zi@x6dIqb`%l9N2OPkdH0c-FteKt18q6wN#voxmMF`|JLS)YY$KPg5`cv0tSjEUswr zi$6a1X^1>FqdYhwaCWRcQtIv-bASR&RTL=@!*sxQw$Ohv6!~hKiq#- zVzY8+*^eOq)|#FlYq^h4nL71jZ+?yQa@QseJ-xJZb1bts>Ma<3jF07bEuH&Mno%N| zk70jdHuLwpLH2ev7V^6?G_o6xmKnF)wfLs}lFv|64xzm=bg8 z^-P8+)eF)U;_TY2>yOMdP6v%Ytyr@rWcz~Nl+;wi!bdKkp2Ux*a*w)dylN!hPd+xo z?WcM@heDHoyW?%l{%F5t3cXv8_$9jD>~lY-Tiy7_wnfcRZOvsA@NtQ zUb)q7fA80G!AJdv)+_G&RqAiFXQ}y1P+0RMFw60Bb+vrf`e2*v(>=?HX~!IvP0_zS zq7pCdtPq}&)ML4J83*6OGHGQyjZb&43A>f>t!k9y+Rt(!jPnoc45pkg;bk`K8O^1V z?Lvb+7~W1i@ba2=V|}pkGM&|pvwIBZuhdj&=<`^>x`V%hW%@itg_?z+hQrh~{j0_P zCEU?US*y!>jp^@|HJ)eH56C3u!0l3?TcB^i{$m$kqjn9)8s>Tat=lhlFmOLmlB^NdV9sOy^}1k7w^f4ehI0~| zqt7Y^`7wx}OaRZF6excPm7S-5k>L%CR^#=?a+4M6R=o^*mqR6(MB+Y_>V7nRb<=_G z0`CcrkI5H!YW)73HA?STAGuzAU)tY0yS_@r-r^=axx)cc!qWVojc4QRsP$h9Rx&U! OFnGH9xvX^X_+~x z3MG{VsS2qTnQ06R6}O(wiY`;tUDtA7(v`n)b;Co3rma>V*#ll30`JZ1acqT=$O)7R^4%HH?yJ^!cW`1wPh zBMAuHDJQFVM^{EK0<9h$PwlwZ6|NfX0aeqHe zzRqQPWWpa#%M6K!Ew%mHI?HA}?wgw2qrL2Rq_^p}9h3YOQ>Pc5dU#}3)uk&zKHBf5 z{4lXQuX^YAx9Z)ZQs&nV%rrEJc+mB`$Upzz%h#71&qazYI`C?K($PJOSm$I$JbM#= zPD$~h&BGOU80`1ntDhLbc5+T?VxCUHkDmrdbRM!spSKhmjC?C^V*d!wUYd{htf>8 zEcnIp*CO-CvF+dMD@*=O7ylQ|zvT(T{%h&?j=wi@ynW9yYtFwFdiSrhJPs4~bKU&* zOzV|7$MUzH^K@Oi_srbcOE&XAH{0KSMzl5l`#M$Mw@zZojB#6wC*SL{HheR!#Du>f z6bN95tG$v3BM zY}fV;Ud389-%yJ)@<_RuQeUb@iu>`Lt3R%)c7?{hdc8uu*zl(1i#G|Thu<1zcjV@+ zsysRO={4V~Y^8Nk$*0VoInUCrTE_ft``Qz$uSj&bto!W}FZXoZ1J_xJ|(Vcwg=IeWNBN&+SYK6>|T30JtZ(SBSW#*h$+gCHRG;InGUE+Ds z_x;3!TYSTgaXr>3c9j*p(Xgts7er0FN%X#Ih>eybd7~ubXeb&+4r{Bg>hXvXvNX7cmq$#y~ow7*)@6s zruK*kO@7r^`Am4x+<3vfEb7H=;u=fO+|Agosrqxm&Xp5Y(nO-8qfH%m9sRPv zGMHumRgH|z1@<%9LLW!V?bPou5t}0!`mk`l@Po|fmoAn2trc0e^;|=1gS4qZ((arc z%9+zXY3nVFm^L}{^gOHcU6aqB*cbk4>h#sFH_n}$+jMR5-UCM-U$gX({5RFOz3+0$ z8cw0>Dh~@Tr$A$#4eqMQm@Dg`2m0VYJS&u~e{k~4+VCsiBfacNi_5I{H}br0s4!h^Rx2^duAUt9HQnS=+cd6U z7kU5R)O%3YwOeSu;bWF56^lw(qOWWb`72%f{alFAq05EQ#x2`gf10ITYdEZ>y7|b@ zEUEQr5A9|jW9+yeyv=b5Q~4P)-#3QW0?+iS27F-Gt94iX9*jhWKR3>v8udY6qn>kx z)G?OXHml30JUOkU`QUW&g56y*H!}Aes^ofdr_01#>DCTC<+Xp@);a4b+RM&Wm0$en z*JgPc?UNP1CszNHF7DVY5+tYf)?#y{gj&W4sV(NWxPAy1ICro|bRErb3UX8}nsE0W z@AHSBJ4I5S`G>Q2dabR@n0U6<^sw378C542JF9!mzNFn05be9cEz!L_W{vfZ2ahkN zTQdrIz2Nqr&M)JziR0^5o?<+vZGBtHG4?_d7@{CeLBw;GkFRDRw&)yr!4 zIq$jMa#E(bs(S=ezt7dGSDmj zTpO}Uwq#zRx|AQ+G+E{`(!htN!fu+^)+!mzp!WeA?P%r|}9* zJ>*o}m!8L{nHduGt1-CVX5yyII(x;1)yJKj=Ioz(^?cvEy=z|0={0SPDp}taHv8=? z=S&-ewf9S}P3+5h-MsN!m`Fj~xj9=ewawhjS5OX zQ{*ygHb04ryzj)%qvbrw&v?&@nHyhZwTks6F|rCz*>BiXdC5Ys+M$)DD0g?Tw%Fq% zksmv`wF}?eN!V>=a_J9O0b8xAT5D;Kd$6%EtDU;B=`^FJcVd0bqVt7Ti*!u#X692C zs?vT{>aQ~?S8USU+}oE`8e8;N-(=W%U--4|k_~ zA7CxG@vSe&qagOV)IK#)f7X4=KkBOOU@V+r>}bT^rMLL1_|6jtZ{+#uwRV+<7YWPq z{>$@9yKTB@l|gQ>(d}hVJgvIk7L>6}JK~|hG~)=*V#jS?dE1vNZoQa&d+!BfjtBkI zr2BU>&p#2XGJW!zWczQ4GavhwmdAHr{1N#4+Y+mH);Ae%R4n$9IPu&g@x-L7?$Oyw z&R6xdlYeBU-<~4EWpl~mKzb$2E?C;q4oaaV{_0NZv%ugThu}NPLJ!}5V zxC@Lo0wmODEictd?bBrX#TdJXdrGFq{gU3>&bJKK7_asa`}-tq(hA12$>CgH{#UlD z&7aA{Wx&O;rzczaSZ_n)+p6z3Cog5X(qyn}e_?F%ZQVs$+ji#Oez$4H+z{=H5A)`- zTb`NQcjns{hV|1XyDT&hx^$XpbE?kcCCz6G-z@8mI2yn_pY+fn!UQ4 zCo)IwpO`|4e4wwB^|b!-?LoJhZFWc$_?WC;dHc#i_v2!7EPMhYq*gd~z3DVOxZMBl zzp{cDKD+kcH{JexKi=M$Vf$VGxPk1xvmd`7zSEns$NlQX60tuV0p4Oq(o|}0Fx$NQ zE%!S;?Luuy^ZgkI^=FxVf5~BZ;|ZlJgpc1Q(Rx<9Nagrg6*5dg6GQfOKdITZuE1HZAzJ?e~QOkC}>Ynfr`%{1O+a$QU>(tb5AyS;gKa=*p)*o@tOL{A$8&kS)llDKj){J#3~ z#}7|tYH=G1Hz+b!-b?G&;GHVMYHz^ABXwBaq@X{)ZNj6Di|;!`U4^x$9?A>(p73pX z!-Pcs1<#tNYBeQh@rQLqbx$oiE}k8v_x;y~|F$L{FZ}kOdtqggWuAZkY|ZN2oo$}4 zzuj1Czsqh)z{gEH=O_Loc?oKnj11=`LG3{w4`E{-7;x8BaJE(yK5_q~1b^R1jBP8=Rj5)ZaaW9Hnp zM^i=Qn81dB*KaSxb@+eLSi1OL(XHZt8qT&2?N(2CKD-iL{Jm<*rw=04R$Lhmzufp> zq?REnDC?voQk?72aLW1M-pM)7ekQYbO>v*}^+t7it$lLJ0fi^q!=6RmKOa}T&hk2w zVvB$i2Vp`*+Rb;uoQUk-%+3vmxcB{Hyy+mmfc*mRiGXa?DQORiAEe56l$eb%iuu zHGVx1m3&5CwILz)*8SU3e-yI>8Q-v9V?6)+Zs-JtKivPm9%w%B*C9ezNzYOEgLcA7 z!?gJ<7c86Nd2E8Xro@;Lk8IYW2DPlsJmE+q|O392797(QFV zGC}H~`2pDjj}Mq;2|i@1XPn01FPVJKi{Yg7zk3I?6567iD-Osv-0n5x6>|90`v3L; zsRvP8+dfQCJ|KH+#{J!aKFvYrn&mfL(_@PZ{?W?+g|RMuZS9V&yH{#Au(a6=R|w8n zdrg2dLARjaa9JeN2m3#Cpp{-oZE2lCf$joiibLG6#? zqwPOyWItFJK4Pi<{%+~Ib$!QCKXrKt(4`6OIh6It^1*45Q#|48mQC{gUg^rGSi?*_>k&m42Lw6t#QC{(_f5z;H` zuxZn#9Y3GVo@tcY6+Ziitiflm$BaS;raD)de`va0DlILosjvV1!^6W_DUNk@cJ_ZB z^1r;iykGq3dxK+ZCNkWT|M%{I-FE%It5>ax*q$eQ_UzfLl)`yEJv^sQosuxmoAY|U z`GMHHIbXu6zBg*KGu_XZjPnIV!3(JJo0vP+~sSh ztPEcMt4?)8+1se?rjJ0tPz++1%@PtMKh=clp6 zaQyGD`=Z=qc&F<1+I`*$&(F=>^YNH;-qhWfqpH`k6jc6`J5dt#NKD2LF zc@X#G{6r4UJzp+)o0ysH3e@WU5xr8S;q-%wrT2H%?Z5y1r#|0)l?Qb{gjE}#JxiNc z`z><&sk0Lq^knZC?)b27Uso5`{q!HPdVeN0{C?;^h2_=j*Ee^Su9jZ9FK_MNWh@1v z#$_$V+|`pbVG7K0ZXB3toE~F)U6tXS{;!>9Gw1rwHv742!}7Hcr?cGQuUAqL=ya)g zIyL-`M(J#p523_xB z|8G1l_c>M0!`HW0I+aV6L8Ii;;pKjFKfS$p_hQDBr>Cb^D>W`CVYhPralVu}rlM*K z%YzVx>Thp8mZqyRw*HsVTVo=t#Axw`>B`lsKldG;)NttirJc*Q85XRcEMJy6b(+e8 z58G$AwXq$4zK?~W@7|e$H#au6yxpN9aOcDBl6SSPObzQTem?Y2nRH?9be1N`x&ytM z2k)vdym+jP(@qgqen>~3mvB(`1A9#D99n9OmS~k zgKTG@$o5Iz54K*9`@H7gzS`dz85sf2-V92Tl6Ex~#m~=uUGuL)PO38o1Q_}Et0*vbh;Y}NNG;vCF%YC#;PC$&$^9EkUIy{WTB(4{$#`gA^TXg` z#uQQQFb|MXQ9tzmn1l86fb}=DlVvSA+8*?LNx5s7om-#45m0e3++IQY1=AOXU5t{P zrWqPXZf;H&{iibNKocwXk5}RQU+rRHh^u<3x@POik1u75{`|Z(>K}38Dq{t6Jn0l$D(yhL%W~_ z>*s%Z^7N_ZpGgj@=2hP-XI!)O=(St_8L}GK8x7`mWUt%l2GaOx_t&?#uV;3)O=v%O zKJ)L9dg&kXC6}W6uE&X;r=|Mx3M^E1ouorcc}P9**? zZZtjM%u*5S=;+w^{BDDDJKx89)$cX`=>2==Z}X95e$A)O*IR#odFdP#H7ou1z0K+V zZ2WR>wwy7W({e59w&)FpGj0ZXcP#v0A5SuzSN+cN+nbwp&GWyT~ya1pYZ9)$-FyN>x_@t+<6BI47Lx_348oz?@zt9 zCNd>8^=HwcvL{{+55GV2Iezv3Z_)FP&_ys$l{^M3oW8T+m4|7|h+6Ws9swp`+t=^iSFmif-E z_;^(O=B`rhuV25qC<)%@-zObh-OelRmYS-%G2+bUzk6@ZXgKu#?2Lx_56_=Ck#Mky zwdD1+v%N+BdQn?4+-L7*xOU6#NcxY;O=$-NW_z^y$0QO^cfa8~o^|Gb``v#l$_XR&`+G5g{6Yn!URW`(VdntJ6Sd&SqQ;jynLEm^vBW5PkE zuCA^ri$v6fI)8}$d-N!2`{DaMI{Tjdtem+)Oj!PFd1|A-j#%>3Q&ac+{dQZkaKqw@ z8XF^e7CN`@3ep$hGE6%o5r3S;B`YW>=+K>=#dj;7{cl}kaBR))E9PfXSw3Vx_?_1Z zYD~&l6euj^bE&Gb>K4;o6uv%g-L=~1bIT*PW{F<&97g-mdxN*+@r3o-Fh-^76)$2x#<)dI(7YP`Ih=GZ*ETh`1rVS?k$t`;l0}!f_n_D*MF1O7ducVvv1?^ zsMC**c3a<7nbIk&UiACz_TQVw*Ge1=~bbte>`aBFIm-@^4RwQ`&-Wcm5dC|g%)qkg*rbRlg=;s z71nn&Y0C8JoAdA6%`H-jT)~zmk?gm9H}~aV%O6kGpKYG+_oJ$Ae@0&3vz5!|JvwWC z|H?{rp|jZsdXLTUw>D|J`0IJf^6HJ**Vj30tJug2@ zxm3(F<4KDCpSrSmqh8KG-?r~hJv+;E))D1DOIRZ~?xgkRS{&>5y{SaWwIP-h;H{ zGx2wH-b66u?On6U_~Xap@|!a*DqUL_dpq-*b4Sb9uk%&*&HFz+FXmIc&#udQ4f6_$ zGm|z-SeL&`IX};~=DGF##79Ru=jo{l&3*laF^)}4I@#_^k@K;r-)aWC7PC&%kN5NS z?d7`}dg9!4 zJuS8^_x7<96O}b}byFW4Xv~se{Ghm@??>_P+E};Qd^JZs8Ewxl`};ODGSV|ORdwgi zojo#^lO|0Ps`~zJ?J-WZ`zFWU_+2^HyY*Y}f^>K8KR=G!UvrqwUGnzUR7PgDfQX0_ zCnhTE|D0!k@u$JD87+DDqi(a5{79|%{QUg&Sta}&?T4Q{c`{-8bocxFYEPa&zdU?> zT+89aO`A5oxVBb0c2|j_t!-?tDx<|KCW|WJWryC!hA&*Wu(7R;?Oy%=+Vk6jetCZ1ES5pKSRn}!O4TciG$ODLy?g~QH4RVg+-xd0z->HJfHuG!kSqs z42nThdU|-4tJ+OwXz>V)iMXcA$T4x%mseN4nFN$pJbC`yTDQ=JNkGX}(ky4jWuf1y z42nS$0yK_%czAg3w3SmCT09)tc%_~kQ%LYN;y7?~b*od6hgN-;rQ!+yCM`|P!o_Fi zr3f5&?#J~a@cPQ`%gZ@yKOPlt%-NoIcb8$-6^-Nb{~eoQzkClv%nTkym4+!vx3*;F z)qKeH?ON?1bFZ7H{jgV1*QKT2;+zZSK6&!QAoY}pxmT{Y7~`C>AO(GWe^0N#IdZ#= z)6cQ2JNs_OV?J}czq1?udxxA6-m}N%-LBVq+fS|aoOWNkBhX>C1w{+}o6dAt?%TzhV1vRv3(U0zgFG-a8Hvbp(mIom3WRe~+wozhZM zK~<9Z@m=%Gj=45*{+9CL!NI<)J>TzD&)rt*)+=@NZu$MU->Sv^8T-|5zpu(O z`eC-b^;oa;@|2|mFZ~vy`?WD<*4>$AMov8hO_xVxpaCyUHDtA}BYz=4V zn_-yDCMqgwVrr_H=;;#{CU$w5Z*k1?3k#j+u3B$j`YL3Z|NM7XZ%yNGSU&kgrdrr>PhJ*sszM7FV?R z#h;&_Po6n*MRS5mN=iz>|G(c?EB<*b@_?D)YfaCOwcN+2Or83%H^0VtxoeY#o?hCyIhI)*^%jgi#>aBJmd^br%_xz~ z$FRRJoB8|Qa(Px(Rx8_@WeXNGu*+3A@IK`K>fotW`)U69-%I~pjIx%waW-C^%Th_K zJ1{Qp+`YZkrK^5?Ir;wpGyjXuv;Q{mY8W58vi#Ax*{5p{N9|_%(K%Z>dUsi_m#>oN zRlEIqziSoQY`AnhzeU%|iaL}(3T}zryYl}z>-QV7u4pKxdCK(Z!SV6ygHN)8S~GhpKfk!Ud%M)(Xg}uvTso%5 zO1eVx!-X3Z|NYq{8JAP@`r6u@8ygxuR2BuVW^Os0n78w(n3uQrRf&7-|1BSUOo_Sl zdL~1Z>ILZvadvIi^+#qJr-R0yR;*bQvVB2sN@}WM;UgDNPvXZ@xkp_!UNw^MCm);P z_mjcp^0facYv=OpVXH7(Bh%}4b5kmJXlQ8FrVFVrZf;h8e7ryX)|Sk=wJF?6fsgdx zR!IKXYPc+T|KB59zS>7+%d?67+!)NxFPHNB+uJKwuCTm&Dl9&E_H6IOM8%~`m-fj& zR@L+7{dl`Fy79lsF_vdbvili$K6pPUvoHT}Wo7W2ySu0F+GQmrC3VieXVc1+oe?_6 zZf;Hw{>9MZq&H?*6^k`*`&JDyG=Rkoc=t zuiR?4zxV68;G_OS>lOF?D)l$ov()@0D6DxBnB{o6x>`PKeXvdT>7M1pv||p-rs&@u zQHhs!RtV2X>akqAjDv4snY6N<#;3d2gxyN`Ry9g;?Ps|V#`%YJ22)O$@G_hAjOJ3w zcA>!@3~whMczI2`u|C*%na=9Q**%8yS8A#>^m!~`-N9eMGJT$+Ld`-@!(r;0{?%gt z67FcFtkq?`#`O2f8qc%p2V|0arloSFhck%B^5(GTDRkbe)%(}xdw`c~e?ct6S|N+H zo|!kT%s%|Ty2tY{s5S8H8e@#&9q|LKN;g<|RP9P5^tpX<;{SVsn$7W2$!li0R2Z=p zv~FxnJaG8H_m2gJ6MqO9KI)DH)%qdD9W9&>m;`y0{&j5J&%UMYs`RXec*g%pJvY}s z6Yx}S;C3m{Ezmb$|FMg&QM-m?4f8zz*6kNN7`PuON!AE!Fy}GVjS%f%Gt@;0_dUpLsuAAz@NqQ@nrBt>`t;(*^`1W^e-tnxr*UmTE@0>e%duWnf zZQ*~FJ@RwQWo+H+=hp0=AMX6`Fzd^=8vW-pu_cp4Z%)TE6itpYlOJ>zwDdOS**f zx3kQ=lychYxFWN7bF^M@NuEXhInnvCrH7;S?Ih%DCcNFWzV6qfx$=3H=Oe#A>z?!b zTJdhhR%!plrNTX&;d8d{tp5>zpP%=wt%=K_%lDI8ZOpEZp<7bSkTbe0fP$BW759fE6_R|_O9xK^a` zB%s@_O(MC=a#}>Pw_U49u$N_~?zT6@Ck&U*DL!es{gJoE#wC-yHMg1s$9}z%wPxkB zsI0YX3oo}lJ$7}iiC@X}T7KJaSBmxbelzo!FX$$d`6SRy_N*n>A3Y<+c~G;+&2aP8K+%g2Q$q@tX(TB~Oxv;JDevy`C;twd z4fy_F#QeLU<>Cu}d7tz+eP`HdUs&C6E2K_7s$DGUveA_VQT{(SCQN!6qEr4#z^Y_0yXc&qkZ>}K-U^S^(WPkH^6 z*Oz3fRy*Eb(;}y|)aCN`OJ`1{h;O~%zpmA2<(3KG?p%GUwAwWKTaoo+{@B*{-+1!g zd6cnTGJ0xxYRM^|ytV66PfM+gFjqD|VEX&7VUW`|iN~5u{`a%zt({?a<-Feq0Xglj zUi02hlzLmx|M2s)iMnSecD-L)WtFg8{+(l4{NLo#)`u$u)6GjRPrb1{cV(DY`+<(P zSLOu-^yxWjJYJRBzbQ2+pFPr5hcWT}8!-^Mve0GDZi(%^1CkK`tQ+xt*CwB?l}u~wXqf2 zIc4d@DEmbFv$b+H^2eXK77(%bc%QP@>(x#d&z_I;_PV6~-=pNU_q4Nz$`aylEZ_2F zk)`c|@?y)!9Zq*&@n{y#SvYs|j^jHv$OYY={xxWJ#nw2jN!s_&$LZT3!c+cM{@s>D3QH*D|M#3C+C!C$C`eDw!#cNxwG(HwBGm_}5edXvWl0IR- zg|PIbMKzj!q&rX!$VtO!CoDiig=i-HqQV-pwQGJ@vx$khOUOe)_<6yuF&KrwY8O=ZKCBM=o_2Z2x zwtVJ+QA<^B3Hn_*CD8C_x7O*nJ0T~vK1U@!^!T=A@jau!-H#q9oxQ1AY8R4kB5?C! zCItU zn0;)6Fr$x+==o1m&YkT&KlQBR;=L(ia#0?izD+!Hc6(0U2aSwBH3u4ccW>9yUQo2` z&9N&R_vrPcubufLcB}gKz-Pf5~{+|0ijmvy1A^oSo&qU*&mz#=)K^|CaMr z%f9?u-~Q{kNqze-$*g~TUwzvq=dv*y@mzAY~^``<86e$!^lFMq5o z_Zu2-))us$*(|Sj>G#Tn(v}CSlrz4Zd35Dh#;KF1B(kDw5^f#cwnO01zMFjinw%Ex z+)%Oo&kX~;CEph8ue&%w|I78FC9ALA;yzQb^0sZv(QkJebavIAFj$r>J2^Ei?=+9< z)1F^v{Z@QAyyfE@Yu;b8R_MQ9?%i{t=UcSFySDUg6ITa@P02pae(bWH!Cowm!OaX;<4Oecp3>N_T8$dmf)L_4Hjpvs>EPI&zWZSXsyqcpIR%N_> z#e8$^=6gPS=XYI@SI`kHv$(ZXd-dXX#eQl#Ub$^jF|A2i8nQZa@AF;hUndH0{i~=P zh!f%LHbXL~JSyj0_TA_?}a>Lw9Y(i3xHu zR&HDxeZgl__kxF-U-)A6&hOp(axw3lQjuU4%86!$YA`AbK# zL|(39%r4dR=9TMWs*8BMe>=Cl7On%4OKRgVc7Zuc&J$Ka#?x_wc zoOvngMaRqNszt5#UJs*x&Q727ee0{G!94ao8nP~u6HgvcXOLgJyX*Zfe~;Cdqn2bZ z{^*zVK*TgKO4HYG)Anm$j<3pXV$nLp`giKJl{#Eo53(uB%Zf+(t4KYSKK}D);grzl zZ+6)}`g>zxX7uZ&(ii5WaGPIwGfl9oe1fPdga4P6l^1K{c0E7Y%T)RB#T4I#(cO=> zy~vrR@3cBr`u;RUSMz;SFUcmgg{D7_miILedc4JdyUtntZ#wq*k##S3AKTiNA9Zqd zV8O4;9odXK7piPs!^a|Y@|5%1*vQ#p>@6pEirif?m0x@UXWG%0?m;g7r5U&mcP1k)r#j*89V1aFk~ZK2M&g2@CKZe#ljL z&Rb|!OQ+un4=;mXMJpJ@@-}?QTX8Nh@L%CY{nbKjTYZ)pB#LO<%bL3C(rkO5tm_y5 zYa&q5P z)^zlS|M6VeHBzmfGaq3Jrd=QrLO+XVLfo@eaw=_FeYu;nHWg-=6w&Wqn+3__O78 zU$b@_ZLd@HH(MB=RPuSwrTBCR`6d3siTKBh|2sSLuu5lsk6*^Js7UhGVg?2V2DT(` zcNYd7hRF=JQU1n<7#J8h3p^r=85rJJfG}gB_9+Vn1_t&LPhVH|mrNX-(xROg`)4yS zaGdpYaSW-r_4ck~&}|P1hlkVDZ-5L8@6bV_LWBgINp$a*Grgmb43SFuYV`V{)kB zXk-wOcT!+D0uuVD$in1M*U`W*L0$80lV07O?zcR6*Z%$D4NFhIU(-Lcj5&YTzD-ZQ zD%eNO7PM~gO0WO4xN+`*1oj7K_>Gg<175%JuQS;uEWxrMh~e3!|Ct#_)c3a(ugQGI z!QsI2^R2_FliQbY>|hUQD3qBkd95qons@WxdC9)sXQwcTB=nj8{FAHUs_riK zkYf~>o`^GaF#Kd?P;!6@37CMyz}{eJU|3*URd3M2z#!Y9$;8yK;8S-4gTjtKN(u}d zdiyyz7&_FMm>3RqHZUmc$p8z^;NW0r>Q-f9YIxw(*}$-%a3Wv3^X7gvafS)|FF9Mg zI4}r=)CmeOuuP9vZV589nj-Hv6tWdhDlI6#i`eXX5)rym6J_36tW(TFrkRTFUj`<;!vT zIsL<};W`%ku$?f=jEwr`ztH{*@BWtqG0 zhQ;?BKYskn=jZ3&ZEJcj8S!blrR0^E>pki#9xndQua8o%{FAjW;~e7jC@uIx1^fo8{B|Qj6l;tpwd3oMwH=9S`R`UiRqnu1WjPbgnA?X}^Ve$CS7;8lNT!_MWh> zc_zmoXmyA4z?qoM%sVF7|99LVe4yvk{+Cr#WgXTx+xIP=%OJKl`p@z&H}^l^eDCa8 z->Qm=6TeL5<>W4H&%b}^?AhMT<<&nclTSr<^H|xL`U~=I68T{-0@rEb=8RGZX`R=d#Yn1jP>fe;K zU#Sc3-OSsbx&3w|M~UwL)Bhj5l5P0@lezX}KKp}B?2S*)i#5Esohazg|H=OE@ooQS zYW(eURG9ulkCAJ`&730wcTdbuWq;7CRmS`x_@9n7!>8N43ugUrey9KI=&or`HJR#S zI>gTXU{YXF*vD{2wkf7xk@+L*0_VK^{Q2^(&)O7UKK~`p?ZBL4Hv4~VLzlvYcm|e- z+w1LogQeOY{yg8aKfAA0fkF1;ui&PlKOEm~&7O3z{>shYzOnCFSeTR<8WifBlM~!oqdG zMcMnuA3qv?pFFGWC<{Zxu0P&&9j~?Ad#9D9n=oCM+gvE)ob#Leg#DH;Qk!k0-c;DM z?NPpCeEr_py4N>by^}i4|8%i3L|i(j@worjEW6+7d)Wn+&N!EMI&ON?r`wDPk~(jH z?GF-Y7mNRWI$!JbZ>z=jdRB)v=s%PQzsJhEc3rb9LyQy4v(lMwm^!)@(*IB4+AEl` zGJp0P$>S}5t};#dnLD|_=S2L+*=N3a-_v8w`Fm2nfIZ>E3IBil_TOA1|20Y7q3#p= z9YzV4Pm}lG&$s%`zV9aIg!z9OnH89xsMp2oFYsVFRv7bV{p4%UO00ec%K!Y$(Qvn6 zv)#wvm4-*{|HN{1`2X1!A2L1P3f0*>Q zZvM5`r6)9P=AXCQHT#3_-h%#SQ*n;E1+E8t6n-2OWBlCkfIWug3{#VQiJ$>HN8JP0 z13U)~iZeY@{w2`C@H0`MrQuJ99gmVjjRcpXf*nu0fPj3P3r7e4VI{UD?FR*!A01}k mWV#6&zs1tpV3-i(^)mj}wXn4FX-gh}6nMJ&xvXY~w&-4#vxkdJ=3Taa@$`GS=r+TI4UL<)jW#MP zFEPL47&P;5{r~DS|36I4-m11NsQC2FPomK|`#vwbl^_56>DI{VFLs}f@BMRTZ|u`E z<@+mYpa1yx;`zMm2eK>betg@rK)SVD{zU%zcZ=um{&M*I;oHf7%9U5GH~&<#|92(V zy4&C5zuul-9Ao5DZN&NlR8vcGc|MzrLk^6sm%ytO)b|~(8?46O=DpfP1 zuKj@j-`f#)YV&sY&tCWG!P6a=tn;4zIP%wQeg40%zYj~#H5OGm{w6;8=#D+%Qgaqb zd@&E7C{P~vTH`Lm{<{A^T$6TsKGAC3X>>5}cyaf{JKJw@JyWwPNPJ{!C$IPVDO*cN zx;ooqCdmyqTc2#&eDwDtnZh+smKztz`+R8@*)io$(iFS437&pmtIXzWzKeT3xxUVa zdt!%3k;al`t$yp0d<L;?i6;rATY#vT0R(md~aX>qYLp;jOW8$s}*hty?Z- zuh_V3R`$xR+iv-O*1fQL<}=#(M(S{Ecub0>5E2$=hsRY$$kPWM#y(v8m-tt;}e*-&x$`MfD&t=3-K zFSc)cys>hP)FRum=WMJgpQ0W`>78Gr)KNCsPO(1gtBU<|lZ{U|=;VkrT)(jL?L8%9 zrRv<HD)rN$0MYA0K=X*=R!~TtD z=SvO4%%JUSRoo;WNv(bRHbZn(6>ojie8u;rD=eOEQM2`4@zNr^ceU~Fz;}CRvNP&d zznkLxqEWE3I^m_M|DyS(DN5d;*bsBqnw5%Xk!{Ju$5tG*RXkIb?+-pLm*_0CBrnZsLN#4>V~EozI> zyxcfV*_FjzFSqGR@=3>#HKEDa!!CYzQ_GtJVS^Up#FXKjcTP45y zak{G_#oBnM?VMNEo~m)}g3jM(<@2g1mb$pu?i2l0#k4GAo2Rt*1B>n6pX=5v-S);< zzG1)FuZhPzndjWM{zo(zRje&Oxy=8}A*X9|8F%qKstYUd zG&KCsr0tZRaLZZF#F*Ef^YtlNwo6+yE-qe`ZXj|^a$%)O9P_M?>Lc8UJ zo%iQOIn|aMrFW@1g={Tdc6hB@;l94!KEI8Iw>Np_bIvlYkSjf}ewtNvqt%+|?@l)) zyv_=89e7+3&G7gDtIbAkBdMHMQ=eEIjPnR$2(DhaM=kFu^W%gG*}Tr}c~Uz*WJXx! ze%aqIJ1e)P`Q-iOyVrh+wcWTnIMifTUhWN!efuqP%9xe-XE0VAeI?8Kz)wgpcNvqP z;^|<=3R+dJzK__qfzRE<=uOs4bStKgzxR@kCznOvq@3H z-+qxY$Fd8hPo7?5NNJ2zxB1Y}vRprTVP97U`=)Oj-|4B{z8P~zebSlTcg`*PAiekY zV-3yoFO6(3PEO7WSi5+G7Mrr5eap=xyDt(`?0pyfeJ(J=@&c>K;XfCcc36I3P--aQ zFIe=fbMBV_G1K6*g=bvb*FOu>e$C??`0m2*dm&}*j-DB!4Gixpw~3rMHzzSc=vbHI0DAX`nujFzQ(9}^_qWGyH zG2E9{OFEzFz&BlE!nFZNEQD#fn{hSqvvKV|BP2H)#Fmv$^2fbEqa$ zBJ=W*T=NqPkC&u5|KfZ8>YR<-$;|;_~pY_vbQIe)$U!p`aqS1!(KzsBTkUeM?! zTOqi^()$-%ke~7qhMg0hR$3_>tLflunJ=(M_}{|D6NhK?vf1xFqog^_prS2$J}>Vj z)f_M2Ehp6+88OAIr`a0cV?}d}vRkG%>i@kH(Ysyf*=KmT|^9Yqo zZO10di80(ZD~K^ZWaG8tp}%AoU?i`fZodeV6u6r<=Q^ve(;I zUXm(5)}SsW^5}7V*om5{+?oqE8C$$CHn=Wj=^I_+*_GJs88%aF!~9EYBAU;f5cZIo zsK)T4Nbq5!$I4l=x=hPNbaLOR9hzuLlMJo4}{(pRZ!}-eU*{8SPkF(nJFMBQ9^4dG{fp`Ab)<5`b73e2v^{~=| z-+j-=gu}}@F7-3qUUc!BY|zx3R(-F|J-b)Kq4c;R^1?Jd1>ZFPXv-{9MLo#!hSU zT3$~{JNDGA>uBP&ny2?Ru-x)hUX+nJYkge3e)=8m+f80EPs{F0aW8CrxcgAL^M?d( z(YD; z`TSU`>={45{N(oDJW;{LO^v<#kDmR;@vXika_=mEk6+9u(s8lVXO2&p_*FQ& zWB!cZU&{?t-F6Dq@8es{m@a;9xxCBY;8{#_zAv?$*j;um-@|LlQ;n6r%`H~jEhJyG zmi(8yw7>7gv+ooBmAaVkZ~GE|qu%cCw{Lgz&#u=~zqUT#Il?|p_1=$pW@qkyKK6Y} z;=g(3`IfrH3p!o1(i*E}q@C z``ZjJg9RcDm%d9xDa~YD`DUH^jma~`mnF)Yc1gAgU2(tSvO?JT-k=xNx2QaI*;vdXCYZTF?XBLLfD2z) z(kts)@2~xlGu?QqtKE2OW_n0M`i*1CPGjhMCs6>wfw{yXPG{gG`s^CDl%E)+@P zUw3nnN{XI*w_0u#qpB9~^egMHR*6@C*5&-kwoN~PK}~brv{kv`6N+xX)toPV{B?bm zU3sd5;^Bv145oWNjFe(^j zTJW~w?Stv_w=TYMDx=Z-XC#-$jj(rR;Y?*&syD7|`4%3!qgoNx*@)nZQ`S4o4<9ieR)$G#k7oh8r+=y`p&zJ1ZBZUNE$>7phv!&8c;NitgOT%7gosu-8e=sMb(2a6G$r_NN~@ zP2YE@pStmbE8w}_2uf3h?zm*Wn!?QE*tf25v8*>vl= zt83ntciv*Xc*^vK$gRee3Va`$%dDm-e)Lm!sPH`P>1Zw3&6dAW@@@HBtCdwzzqY)+ zrd7LehI9Ysn2pmNZwo3FKUthroiIsi{+G6I6X#4jJ$vIV`>SrXS7W7G3eVrM(Khdz zshZ{};Hst`_;GD}fN?~B9lJVLhSWEMlCKy03ZiGHor!wd8D+Kr_(s$bcq-@0}xOV;VXV%>`mGx3-j|D7T0 z+dbcAh4`wIxt2Fqhi=i|UZqiRw%OQd*9ocr4}6v`E@gYnnr>BU^5%?qMnDJCtlg)T zRhJ!NPh{-<^G29gMrEDo26koUIP>6*yjevz+_Ja4<^L$QT9{efI{sUyOwjw%bn(=K zOOD;WnHv%_=j+WIYbS0GliuCNG|O5;?fEwcKlYPi)yGQwUJ1-+lRd<=V7`mfm18_d zgWr6xC_Y-5c`7aNqFjWttWf%kuZ`kU`U|q#+Gp<&FHp=|^f$UDY~|z4Ub%C{RvMaW zEGzu^jzQz-<htQ}7IDb=Ornd&{1h6y1ui zzSnWmM){Gte$2MMo`xHND?{po~Ll?>{VyZTUoZ)RdhypyYaNe)dBXGUlzD6`1L_;!ke8;w#h|$OHJnS z7$~grl$6?PVJF~QWjkl$rPqdYwtULC^Z2O*-z42v5`ELUIwI@*9zI$0m}SFJmQ~y( zEX!ZGxGF96&}Mk>PET*Tc&o_=>l2o93m5JyULlv}5L+5C=i-vU!`_VipUaqDZTU96 zePMz8&)o9AlW#Np^_7w}4G}kB;92aR-2QH5i0=Ct45ELl9_q)KUMl_2_~;df!OXS$ zVip(}vOS!)Z(-M7#}^CqH|no_vMNs`(L*&^hYThjc@%~cx~Yq9n} zU=*6f@Yr)hU`Od1z4I)a1NZQL>-b*deD&vCnS<3?f?TbuGVK0cPSmeHud%d$nL$ra zUE{Y5!}{}u3#+-A6;3cuJnLR)daG>nt!x!pPR`bCE|$&Wo3jtghnz}#d~M@>1?Ij~ zmGz5b3qA`mS7{WQxTr4c_jCSzH(8%~!ckVYmtvU~yUyu0yW~|}7kqf*h>HJ$;FB8N zD`usxQBXNh(PDqZ?BPU(^TqkGTjgGq>Rry+XfsPI>#MTG-0z2EGJ*rHv|O}^niDG) zJ$a*+XZ0eDnzi!A20v1-UXWTPW~{Qcv)%EQN6+H7pRNQsYTr}3dG${CfhZjDn71wPmvDjH`Il^m?3OH;cbIp@9+&95hj|%ZFsdFp(z8fxm(H2kkeV4+Y~1w( zT6tN&7Clii`SIDZu(9RJvSz;+hj+7MCN0@+9Q2ao&c`3DN87%+1(uy)3rm|55*;P0 z`e@HTw;gF$+0(P(OCRY^ zUG&p=-tn9xCr4ZNpf=lD5eb=$Tj zn|Xe%a=&_Vjii{)LFQvw-*{CHPdU*)^SY$qd&iS$TS6}zdUBb&FkLH=GkTl(%u1Tw zU#Qg1Y=MP`()PW}mn5@!A2^mX z(D3TWIS}=}lwVd%DOKMm>t67y*C!L)lY2V-CMFaeI@#5ARZ;beuY*+a@0HC58KNXK ztfbdCzWHHzx5~^}<%5dkMYGOd7fx<_H{C|&OXs2U>Y1{Y5iw!jx7sy-?KjU^aV+`j zUiqj;g@LcveX)+2;&w^tT8DzwtsB95e}3tSUf#MQHsTI!R1F`$%UW(0QLk?ith-lv6?-Vt^P=pxlX#b0 zv=!dwD{U>9@TzHA>XI)4Df>$ma$fyspJA0Ysq~Y`j=#MLrtyJm;sw_{)9(7J-E_6a zbz`0DhFRxDUtd3;{$Kt1=l`dr=Wdo-_T)b!vsQ^ujo`4nJa0`PlBg3pY5)2Fs>?NMQuIw+FI5?%vw730x$-p4h@9E+gQgQ3; z-0G6ht9#$u7eC+1DdNQ8@g(tJ%QR-rZF@9TM2-n;2zdSWLR^Rc7mcNh?-kuD{-@z= z+t6|*%aEN=~FUFe= z;tSX>@SX_BR-Kaep!k8x2i5|%AI1U9SGLt%l}P@l!`PwCT4!J&_(RMjtj}r4MA2d!{lp$^J4tAoIXXAzoKV<5lC=15wFmieKr5jw%DLize8cTtLtY_=Pp$uNACP(wwYBZT1my#=$7bB$9q7{>bgo%`(=|P| zxZoeH{9hRB(%07R*t&bAb^}YBy>Nx#jJ4MUI1_XW`VE&wGJUZB6WbUsJb#SnH>hJHCu3Oi4?3i1hthJkp z(88TNXHJ_Ywq~o91*eJ#*TiYl+}zyQ9u~~lv&ZJwx3{McG%{;yX)SW^m#ce!N$SD5 zZF;8D7;fn`96jJ%m$Q1?wrR4~WdYIAr#pq!ul&&T@$=(rZEf|LXEXEl`u+FTpS4Ts ziPT}-A^uPEKz-%S)GimLoO^p_zIv7AH_v9J+xsO(66Sd_mzVpi%X}+w%;1@k)pJvi z(SpBT{z0rwRl@O?mzUq%QK%dm8fs!{y7C~iLt2`eu$oW8wKb6?Z*N^)efE#!j)M}# zK1?qve(-LPobk*tS4&Il#*RYeiy0xkvJRUzZQAkk+3cA{sa@f-f5;kq_Ik`HbYQA; zmHCIJ+ojUd(wh4E&p$jooR#8OS7&Gc=OO>g%gg)4pT0LZwq_#3E%|@%4%lti|GRqC zs)+4*vS-hp%}Obp*VDsu>eML-^Sn8)=bIn6uJ_%Bk@LYPrV61S8)r}dlYMQ?!cCik zdbRaVFY%nLp{-rK`wds}8G8?gT>fms4`zC{C)LHeKb|o@|KZ>7_p1w!hH7eQCEeRo zxn$|mouQjre>6N`7q53VX4F4>IIpe#=clJ!`!kHIzrRa8+9i4<$#A>ZvhDLOdoYCm z|MFj|pgVf{-}5$~dD!`60uC>;*jx4WQ)#t)?H9qPPoL&Z`?GKBwg9FbR{x)b?OqeP zSxs^NpX6gbiRb26N-UO}C(R>oH^*JRcFM}&<-h7wHV{<<&9J%)FxUa#Hfo$&nJ+&v$UN#{-7 zeL1RnElWY=Ke-bnQBO`xRL+~V`|d;gc9jQlKh96&;N0`&lDCPO*{(pX?jO-BRT@q| zs91V`XWjn$-+$`!?N@nF_d{5<@!7Mqd9~jnx1TyYkwH)Pj^U0E>-KeZaotb<5v%uS zQp4|u{!>_9y?%XjSLtf$rTg;M{$0jWAZlFJQp{aFNfV~PEa%37na1fcw%1h|&guW! zc{X#d?`*T5yEZIe`*1qT9sYVH6@gBdil7b^BEVVq@RB%?#pvu->QW)8RVX-RIYS@MbD>>ydaE z^+SKkffi0-o4fBHAMY>z@W63->ONgYhHJO}S=21Q-zRHrbN~F8FC}?2i<8q%<@Im%>&n!y-s0y&50yz5=1ym6lB_$>t9kIQ3d4(MPueFx{P}N^!-qrsAEOToIW)08 zcr-a$m9h1HKH~~*h7^mR4>zZuS39e(#L1%mM{jQROd$t#5xYGo0r z<8)vNVdC1)qQIoZ*c#6yz&N$xkUnEagO|g@=?pFostZ1NGbk+(3izSQAQ-@@@kgkE zV+B>TH9mTj1hUX^>VZE$KZ}AK63P_!W;Mum28wK-k?P-WzFV6pi#sewg-Nv_V*fk}Xof4_!O7KXU0 zm#S;Fp8WVyw&>5#yHn=5G|Kb%u?VP{*`NS z;k8@pYqowpz?dLfP`|20UO`#e`C)juLMZEpN)5(^@AKEz9*Np5e}napzJh(|?|_E| z5f7)c2susHiw$yTxOVIH+O7YkSFrxuxwU@&rzcOJYW|tzuxeiQy>iAiTaRA5^`9ZD zfxXdSZb$aIoo*nFpLTzJd;5B3XWNAKgXc5<9;ui9AzyMSy6<{y`P@VLoGLC%%U_QR_m`K>QBkwff8X1j?$5?A_h!o(vpFr-qHc@cU^wGukax$z|Ml@C z!+F*3EWf?Ex%k6w7Zstzo10SC#c^J_a^=I9%l?-yU3zr0u3z5%Ua!$@YaNE{T)7QX zf6S?6JTt?v`OKL!QzkSVdhc7lVA-;^<@2h#o;^!D|IJ)myF2spvYmlh(hgDVhvxI< zupQI-R_aAk?ISKFY?Y(mK>cty3PHap*-r=HTVr~xd(5+2p?%au) zrWgBaho7q}>+9?5zc0<7* zADgk?y8ho5!#}|d|8L7BZkg_(a%h?F?23;^#c%E^)&BbRtBaE0eg1vY!PV`&(r&4# zsv9HDeEz%l){KTj@6XO?nE&wni4zG2n^;R;Upw1d`|{O+>1k6Oj!KIGolFI8Rep}JW0LHZBI zcY8kjsrk%!a4LY^;D$u-{bkw=*=rg9G4D9o?Z2+8iz{qh%*jS(_L`qt_DJdK>ZYWn z1w}->*b(;s-(SnhPb#IQrTgCeJ{k`4YB*C}`~D?MT4v|%I=Qy!ymWHht`G8OLj6|l zX1s2HObXn}^FK9tbN2OhHvfJ+UU&9M)wi4JH}_U=&;LDt-n?^rzu!B3tXEpJZu)wb zS+i!<@YWX=8h(C$zWi+Zfq-P6+}SJ%Yv20Sc&=YP{aT6cm+Nh~l9d&);viH48T^zNoP=qGzFV`>r5;5iY~DGZOK~SzNM$f`Sg+ z*;#zI;@SV!H3rAl?7m`tCY9ww_JiMft)RxFj75RMQa+ceDywcW-9_Q+%YU$ujZm#Hyn6L&&aEw)Px&vg%hw!uadGje2lA^=rutZpA(7+S=VGC#xIR|EuBR;@WfTjQW9+HWcVjp zapdK+ei_R{lT^JY)ot0o_RHJb)7#~$PF!BT$7ONgww#+zp`laPzm{*Q|MKSMBt*|EAjC zWm#ES7IlAiOqzV?$+Krc5fL7lnJcCHKW1rj_ZsF#-OAGAcqj4b{j1_5^8Y@t$JPCO z>Q`xyzxQj{GQYX6UYq&PwJNQ%EOB;bcJG%{mFm58<%)}rnD?si$?OvYG+d&hq-M^X z*>X7X)2B}-&Yw>|(7-4nDtd8mb@~4COwoe#Q-A%PE_^`vNc@$XiGK6#&YrLT_t~$y z`?SPi+wXTC-(0XjfUi>Y$2`MjXN{*ht$k>IAlGmEzRb(Z{A%lON1a|3y86e1X8w{@ zohgrfAF#jW{9nn);9O|&)?BFb!!hanl3!tcN0X*ZpT0T&zTMm+rN|X*X%fkP+jnzc z{h@>kr!24Dn0MrJi#O`!{PS)5{?xOxOlKWY z{;kIpNAKaHNT-d1UJuM(Mc5QG-#Zk2fZQIU#$nIy5v1NFFWo7WJUAMe~ zgN60?{ZQKf+vA` zz8xNkB%!MB@75mURJ(6-?2X@*W4&9y z1usZ<=l=8KxcxPU>D(o6Z%t)nW($ajIB{a4vi{F`_7{H|9GlUScR%VjOUaMaiqFr_ zU!PUN&(VJP$&)7&rcZajzpwV>`SZ)e*T=OSPTaI<(~E0srDJ!MDB9Y__Np>kykfGb z5?*%beQfx`g$oze2tqfx=2VJQ$oj7@Rmb z9XJ#jITTeG6kAvnS|%{G2*mUGpD3)ErNW>XG^MA9XSu4~WQG=xz?g_@x{MqXSABVP z)tgB`X~mQ0&#iR}U6=%vTqVtNW?UBft;(PnG$BCa$cKlA=T2KWm7&GMk&RdC$uWfl zUn7nKH&?eh6?tgYcUdZ)@Nd%6)GSMhQ>VD6JA zPYhB|iI{ukdW$j6DGO52*Z24I3Y;Ui+c^Ck%eu4gc0A@YxBEN0;lGy{qltxP!CBMm z8lIY6m)-m2?dxm;-kN4z(J+7amwC^tZ{BuuISyQ0+wpqcZa+eqY@==XJ==HTbkC^v*-VLqMpZFQO~vKRwm1Z zz18JKMMYDVi71t>z%`^&q# zvs<~vcS+VYJnu1dpZxN>jk$0N`v-gLs3i*)G|ZVJ<2T1*Vaiel+v;yAA08a+%i8n( zUiI8c|q$$r83zTc&s4N;<%NY{s)Cq78<}qO$8*bqa!VuC0j- z2@NfMb#P&bnhc+D@iU*}rDgB#%nWk$_xD$q;k&##Tpu*TwjuGbhL%=QvTc2IL{862 zpKIR?pZ`l*E6cj)>6QdY^Dz9@qzRVA&tCNP?$2RgCY7AzseR(Jn!&UF9R}(Nr>1D; z+2{oB@Y!GYSER0fC3~8B@sIr~4PkLbi(mZt`T69TGgmYxsHCK%6#W1DeYN7B$08qO zlYK7fPumf}@SnfnPUHK#yUkNmQ>}O>wcOmCo_%#$y-P#nu^HvTiRt@Q8m>h)uD+{b zQ}!l8&3D$4%Zdl5OqpVkc!0q;=sf#x1Fweh zu`A0Totu5S_Hfj0rXQWNrK5M3<$C!ld0w^KulKuFk>L*u5+N zpR<0yA?vEvS+SCtXU_P5hViV+)<_+${>EhQ^Gvd?kM)99qjh6wUjCO86P2e-pB@|^ zzdraRE2uTIr}FcQySukb9gg;6{?DajdaR@?G(TLpLGj<8O_FgrHLtI&&AG9m(L-fX z@M`9k!-;u2pNe^Tdta5f$Nt~)!N-)CORr}#M5$hot`KL}W?g?|rg1uG{AtCSH6hy< z^roby8Wuis0rez)Je7OYRpV77`F`@T8Gb(*TrN-hpR#r?&mOi4vo$ikZZ|ija)*Y7 zR&BbF`r_th^~cBi({F9btXrGHtrYl3|80fjkFAEwg7^PDvgNCNRJJ^u*w2l@?EG>m zzrVe`a^(ukyQjk9lV{KNPE1r>x^!uu{9{!;Z{Cl$E2A6#n;c_#wj{fsf#-wwgEIT_ z4_8(Ozqz}6`mSA8Qc_aq?0Yt?T-h0+bL{5k^x$6%El!G;E?qit=8TFA-{k4j-RtV? znAvz1tXR?U?(Xj2d%cfG@2_HtZ48OOdiBb!cKdt3o(n$eKeS$P->*`CqdiN_UxLD# zCxKaxm#eGgv(^XOWS{O?PE0%IuxyI{?GcrDX=jD-jHDjRwaYm87M4jX+i85ddrjD_ zgl|=&B-ehH3t^mpSZ6Thgb6RRSWS#jhE@HZk*j?IDe(4 zN<*K=0@fY;6)e-|DJs+~1T`F{uIXPb_AlX%R?1pk)@w|EudMMrtA0Qxxo27`S9&;u zcr0%Yd!9n)y;{A0ZN3M1x%L;tGOQJ{Nb8w-)5`3_|EqgE4})3*&#p1XDBcl2z^ZhE zg-6w{G(w--Cnx^DC#cyRFO|GzmP>^ZTS4o_#>4}M4}AYvU^wxIkl~~5NKmaGQrywP z`G84~N9kY3#{KMD+OA5^YKUk2pVV`6{WAeif*c^RUG02ZW{A2=n?xaBZL#XUL{fi85ShO0i zH8qO#d>436czjI0z*FP*=d4kB$NI?i>ig3E-r4n4 nD)tsP;mI8ikP?>W|7<)PXGg97TCkFVfq}u()z4*}Q$iB}Gom~a literal 0 HcmV?d00001 diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/AvaloniaRunner.cs b/Duplicati/GUI/Duplicati.GUI.TrayIcon/AvaloniaRunner.cs index b77cf1ba7..d8f07df43 100644 --- a/Duplicati/GUI/Duplicati.GUI.TrayIcon/AvaloniaRunner.cs +++ b/Duplicati/GUI/Duplicati.GUI.TrayIcon/AvaloniaRunner.cs @@ -19,6 +19,8 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +#nullable enable + using System; using System.Collections.Generic; using System.Linq; @@ -34,15 +36,64 @@ using Avalonia.Themes.Fluent; using Avalonia.Threading; using Duplicati.Library.AutoUpdater; using Duplicati.Library.Logging; +using DesktopNotifications.Windows; +using DesktopNotifications.FreeDesktop; namespace Duplicati.GUI.TrayIcon { + public static class AppBuilderExtensions + { + public static AppBuilder SetupDesktopNotifications(this AppBuilder builder, out DesktopNotifications.INotificationManager? manager) + { + manager = null; + try + { + DesktopNotifications.INotificationManager? realManager = null; + if (OperatingSystem.IsWindows()) + { + var context = WindowsApplicationContext.FromCurrentProcess(); + realManager = new WindowsNotificationManager(context); + } + else if (OperatingSystem.IsLinux()) + { + var context = FreeDesktopApplicationContext.FromCurrentProcess(); + realManager = new FreeDesktopNotificationManager(context); + } + else + { + // MacOS not supported + realManager = null; + } + + if (realManager == null) + return builder; + + realManager.Initialize().GetAwaiter().GetResult(); + builder.AfterSetup(b => + { + if (b.Instance?.ApplicationLifetime is IControlledApplicationLifetime lifetime) + { + lifetime.Exit += (s, e) => { realManager?.Dispose(); }; + } + }); + + manager = realManager; + } + catch (Exception ex) + { + Log.WriteErrorMessage(Log.LogTagFromType(), "DesktopNotificationInitFailed", ex, "Failed to initialize desktop notifications"); + } + return builder; + } + } + public class AvaloniaRunner : TrayIconBase { private static readonly string LOGTAG = Log.LogTagFromType(); - private AvaloniaApp application; + private AvaloniaApp? application; private ProcessBasedActionDelay actionDelayer = new ProcessBasedActionDelay(); private IEnumerable menuItems = Enumerable.Empty(); + private DesktopNotifications.INotificationManager? notificationManager; public override void Init(string[] args) { @@ -94,6 +145,7 @@ namespace Duplicati.GUI.TrayIcon .Configure(() => new AvaloniaApp() { Name = AutoUpdateSettings.AppName }) .UsePlatformDetect() .With(new MacOSPlatformOptions() { ShowInDock = false }) + .SetupDesktopNotifications(out notificationManager) .SetupWithLifetime(lifetime); #if DEBUG @@ -106,6 +158,8 @@ namespace Duplicati.GUI.TrayIcon #endif application = builder.Instance as AvaloniaApp; + if (application == null) + throw new InvalidOperationException("Failed to create Avalonia app"); application.SetMenu(menuItems); application.Configure(); @@ -114,7 +168,7 @@ namespace Duplicati.GUI.TrayIcon private async void CheckForAppInitialized(IClassicDesktopStyleApplicationLifetime lifetime) { - Exception lastEx = null; + Exception? lastEx = null; // Jump out of the UI thread, ot ensure that the check is done with posting to the UI thread, // as opposed to just calling the method directly @@ -164,8 +218,11 @@ namespace Duplicati.GUI.TrayIcon return new AvaloniaMenuItem(this, text, icon, callback, subitems); } - protected override void NotifyUser(string title, string message, NotificationType type) + public override void NotifyUser(string title, string message, NotificationType type) { + if (notificationManager == null) + return; + //var icon = Win32NativeNotifyIcon.InfoFlags.NIIF_INFO; switch (type) @@ -181,7 +238,12 @@ namespace Duplicati.GUI.TrayIcon break; } - //m_ntfIcon.ShowBalloonTip(title, message, icon); + notificationManager.ShowNotification(new DesktopNotifications.Notification() + { + Title = title, + Body = message + }, DateTimeOffset.UtcNow.AddSeconds(5)); + } protected override void Exit() @@ -209,7 +271,7 @@ namespace Duplicati.GUI.TrayIcon internal static string GetThemePath() { - var isDark = string.Equals(Application.Current.ActualThemeVariant.ToString(), "Dark", StringComparison.OrdinalIgnoreCase); + var isDark = string.Equals(Application.Current?.ActualThemeVariant.ToString(), "Dark", StringComparison.OrdinalIgnoreCase); if (OperatingSystem.IsMacOS()) return isDark ? "macos/dark" : "macos/light"; @@ -237,11 +299,12 @@ namespace Duplicati.GUI.TrayIcon public string Text { get; private set; } public Action Callback { get; private set; } - public IList SubItems { get; private set; } + public IList? SubItems { get; private set; } public bool Enabled { get; private set; } public MenuIcons Icon { get; private set; } public bool IsDefault { get; private set; } - private NativeMenuItem nativeMenuItem; + public bool Hidden { get; private set; } + private NativeMenuItem? nativeMenuItem; private readonly AvaloniaRunner parent; public AvaloniaMenuItem(AvaloniaRunner parent, string text, MenuIcons icon, Action callback, IList subitems) @@ -278,6 +341,9 @@ namespace Duplicati.GUI.TrayIcon ///

private void UpdateIcon() { + if (nativeMenuItem == null) + return; + nativeMenuItem.Icon = this.Icon switch { MenuIcons.Status => AvaloniaRunner.LoadBitmap("context-menu-open.png"), @@ -301,6 +367,14 @@ namespace Duplicati.GUI.TrayIcon // Not currently supported by Avalonia, // used to set the menu bold on Windows to indicate the default entry } + + public void SetHidden(bool hidden) + { + this.Hidden = hidden; + if (nativeMenuItem != null) + parent.RunOnUIThread(() => nativeMenuItem.IsVisible = !this.Hidden); + + } #endregion public NativeMenuItem GetNativeItem() @@ -324,9 +398,9 @@ namespace Duplicati.GUI.TrayIcon public class AvaloniaApp : Application { - private Avalonia.Controls.TrayIcon trayIcon; + private Avalonia.Controls.TrayIcon? trayIcon; - private List menuItems; + private List? menuItems; public override void Initialize() { @@ -355,13 +429,17 @@ namespace Duplicati.GUI.TrayIcon default: this.trayIcon.Icon = AvaloniaRunner.LoadIcon("normal.png"); break; + case TrayIcons.Disconnected: + this.trayIcon.Icon = AvaloniaRunner.LoadIcon("normal-disconnected.png"); + break; } } public void Configure() { this.Name = Duplicati.Library.AutoUpdater.AutoUpdateSettings.AppName; - this.trayIcon.ToolTipText = this.Name; + if (this.trayIcon != null) + this.trayIcon.ToolTipText = this.Name; } public void SetMenu(IEnumerable menuItems) @@ -413,10 +491,10 @@ namespace Duplicati.GUI.TrayIcon public bool IsEnabled(LogEventLevel level, string area) => level >= _minLevel; - public void Log(LogEventLevel level, string area, object source, string messageTemplate) - => Log(level, area, source, messageTemplate, null); + public void Log(LogEventLevel level, string area, object? source, string messageTemplate) + => Log(level, area, source, messageTemplate, []); - public void Log(LogEventLevel level, string area, object source, string messageTemplate, params object[] propertyValues) + public void Log(LogEventLevel level, string area, object? source, string messageTemplate, params object?[] propertyValues) => Console.WriteLine($"Avalonia [{level}]: {source} {messageTemplate} {string.Join(" ", propertyValues)}"); } diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Duplicati.GUI.TrayIcon.csproj b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Duplicati.GUI.TrayIcon.csproj index 2e38301db..410d7ae12 100644 --- a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Duplicati.GUI.TrayIcon.csproj +++ b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Duplicati.GUI.TrayIcon.csproj @@ -32,6 +32,7 @@ + diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs index 28cff9188..d9e924630 100644 --- a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs +++ b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs @@ -24,6 +24,8 @@ using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; +using System.Net.Http; +using System.Threading; using Duplicati.Library.AutoUpdater; using Duplicati.Library.Interface; using Duplicati.Library.RestAPI; @@ -85,7 +87,7 @@ namespace Duplicati.GUI.TrayIcon if (OperatingSystem.IsWindows() && !Library.Utility.Utility.ParseBoolOption(options, DETACHED_PROCESS)) Library.Utility.Win32.AttachConsole(Library.Utility.Win32.ATTACH_PARENT_PROCESS); - + if (HelpOptionExtensions.IsArgumentAnyHelpString(args)) { Console.WriteLine("Supported commandline arguments:"); @@ -204,11 +206,19 @@ No password provided, unable to connect to server, exiting"); do { + if (reSpawn > 0) + Thread.Sleep(1000); + try { ServicePointManager.SecurityProtocol = SecurityProtocolType.SystemDefault; using (Connection = new HttpServerConnection(serverURL, password, passwordSource, disableTrayIconLogin, acceptedHostCertificate, options)) { + // Make sure we have the latest status + // This will throw an exception if the server is not running, + // or the password is incorrect + Connection.UpdateStatus().Await(); + using (var tk = RunTrayIcon()) { if (hosted != null && Server.Program.ApplicationInstance != null) @@ -219,16 +229,20 @@ No password provided, unable to connect to server, exiting"); // TODO: If we change to hosted browser this should be a callback if (openui) { - try + Connection.GetStatusWindowURLAsync().ContinueWith(t => { - tk.ShowUrlInWindow(Connection.StatusWindowURL); + if (t.IsFaulted) + { + Console.WriteLine("Failed to get status window URL: " + t.Exception.Message); + tk.NotifyUser("Failed to get status window URL", t.Exception.Message, NotificationType.Error); + return; + } + tk.ShowUrlInWindow(t.Result); Server.Program.IsFirstRun = false; Server.Program.ServerPortChanged = false; - } - catch - { - } + + }); } // If the server shuts down, shut down the tray-icon as well @@ -255,7 +269,7 @@ No password provided, unable to connect to server, exiting"); } } } - catch (WebException ex) + catch (HttpRequestException ex) { System.Diagnostics.Trace.WriteLine("Request error: " + ex); Console.WriteLine("Request error: " + ex); diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/TrayIconBase.cs b/Duplicati/GUI/Duplicati.GUI.TrayIcon/TrayIconBase.cs index 4906daf85..1b769718f 100644 --- a/Duplicati/GUI/Duplicati.GUI.TrayIcon/TrayIconBase.cs +++ b/Duplicati/GUI/Duplicati.GUI.TrayIcon/TrayIconBase.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -43,7 +43,8 @@ namespace Duplicati.GUI.TrayIcon IdleWarning, IdleError, PausedError, - RunningError + RunningError, + Disconnected } public enum WindowIcons @@ -64,11 +65,16 @@ namespace Duplicati.GUI.TrayIcon void SetDefault(bool isDefault); void SetIcon(MenuIcons icon); void SetText(string text); + void SetEnabled(bool enabled); + void SetHidden(bool hidden); } public abstract class TrayIconBase : IDisposable { + protected IMenuItem m_reconnectMenu; + protected IMenuItem m_openMenu; protected IMenuItem m_pauseMenu; + protected IMenuItem m_quitMenu; protected bool m_stateIsPaused; protected Action m_onSingleClick; protected Action m_onDoubleClick; @@ -105,7 +111,7 @@ namespace Duplicati.GUI.TrayIcon protected abstract void SetMenu(IEnumerable items); - protected abstract void NotifyUser(string title, string message, NotificationType type); + public abstract void NotifyUser(string title, string message, NotificationType type); protected virtual void RegisterStatusUpdateCallback() { @@ -149,23 +155,49 @@ namespace Duplicati.GUI.TrayIcon protected IEnumerable BuildMenu() { - var tmp = CreateMenuItem("Open", MenuIcons.Status, OnStatusClicked, null); - tmp.SetDefault(true); + m_reconnectMenu = CreateMenuItem("Reconnect", MenuIcons.Resume, Reconnect, null); + m_reconnectMenu.SetHidden(true); + m_openMenu = CreateMenuItem("Open", MenuIcons.Status, OnStatusClicked, null); + m_openMenu.SetDefault(true); return new IMenuItem[] { - tmp, + m_reconnectMenu, + m_openMenu, m_pauseMenu = CreateMenuItem("Pause", MenuIcons.Pause, OnPauseClicked, null ), - CreateMenuItem("Quit", MenuIcons.Quit, OnQuitClicked, null), + m_quitMenu = CreateMenuItem("Quit", MenuIcons.Quit, OnQuitClicked, null), }; } + private void Reconnect() + { + Program.Connection.UpdateStatus().ContinueWith(t => + { + if (t.IsFaulted) + { + Console.WriteLine("Failed to reconnect: " + t.Exception.Message); + NotifyUser("Failed to reconnect", t.Exception.Message, NotificationType.Error); + } + + }); + } + protected void ShowStatusWindow() { - var window = ShowUrlInWindow(Program.Connection.StatusWindowURL); - if (window != null) + Program.Connection.GetStatusWindowURLAsync().ContinueWith(t => { - window.SetIcon(WindowIcons.Regular); - window.SetTitle("Duplicati status"); - } + if (t.IsFaulted) + { + Console.WriteLine("Failed to get status window URL: " + t.Exception.Message); + NotifyUser("Failed to get status window URL", t.Exception.Message, NotificationType.Error); + return; + } + + var window = ShowUrlInWindow(t.Result); + if (window != null) + { + window.SetIcon(WindowIcons.Regular); + window.SetTitle("Duplicati status"); + } + }); } protected void OnStatusClicked() @@ -207,6 +239,9 @@ namespace Duplicati.GUI.TrayIcon case SuggestedStatusIcon.Paused: this.SetIcon(TrayIcons.Paused); break; + case SuggestedStatusIcon.Disconnected: + this.SetIcon(TrayIcons.Disconnected); + break; case SuggestedStatusIcon.Ready: default: this.SetIcon(TrayIcons.Idle); @@ -226,6 +261,10 @@ namespace Duplicati.GUI.TrayIcon m_pauseMenu.SetText("Resume"); m_stateIsPaused = true; } + + m_openMenu.SetHidden(status.SuggestedStatusIcon == SuggestedStatusIcon.Disconnected); + m_pauseMenu.SetHidden(status.SuggestedStatusIcon == SuggestedStatusIcon.Disconnected); + m_reconnectMenu.SetHidden(status.SuggestedStatusIcon != SuggestedStatusIcon.Disconnected); }); } diff --git a/Duplicati/Server/Duplicati.Server.Serialization/Enums.cs b/Duplicati/Server/Duplicati.Server.Serialization/Enums.cs index 31c615634..714e450d3 100755 --- a/Duplicati/Server/Duplicati.Server.Serialization/Enums.cs +++ b/Duplicati/Server/Duplicati.Server.Serialization/Enums.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -53,7 +53,8 @@ namespace Duplicati.Server.Serialization ReadyError, Paused, Active, - ActivePaused + ActivePaused, + Disconnected } diff --git a/Duplicati/Server/Program.cs b/Duplicati/Server/Program.cs index 25b37a954..2886f1bba 100644 --- a/Duplicati/Server/Program.cs +++ b/Duplicati/Server/Program.cs @@ -265,7 +265,7 @@ namespace Duplicati.Server DataConnection.LogError(null, "Error in updater", obj); }; - DuplicatiWebserver = StartWebServer(commandlineOptions, DataConnection).ConfigureAwait(false).GetAwaiter().GetResult(); + DuplicatiWebserver = StartWebServer(commandlineOptions, DataConnection).Await(); UpdatePoller.Init(Library.Utility.Utility.ParseBoolOption(commandlineOptions, DISABLE_UPDATE_CHECK_OPTION)); From 4be739ff2ff35750693ec47427e66571a97bdca5 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sat, 8 Feb 2025 17:31:49 +0100 Subject: [PATCH 131/362] Updated logic for reconnect. This fixes a case where using a wrong password could cause an infinite loop. --- .../HttpServerConnection.cs | 122 ++++++++++-------- 1 file changed, 65 insertions(+), 57 deletions(-) diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/HttpServerConnection.cs b/Duplicati/GUI/Duplicati.GUI.TrayIcon/HttpServerConnection.cs index cf610569a..d6f6b7a54 100644 --- a/Duplicati/GUI/Duplicati.GUI.TrayIcon/HttpServerConnection.cs +++ b/Duplicati/GUI/Duplicati.GUI.TrayIcon/HttpServerConnection.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -88,6 +88,7 @@ namespace Duplicati.GUI.TrayIcon private readonly string m_baseUri; private string m_password; private string m_accesstoken; + private bool m_isTryingWithPassword; public delegate void StatusUpdateDelegate(IServerStatus status); public event StatusUpdateDelegate OnStatusUpdated; @@ -161,10 +162,6 @@ namespace Duplicati.GUI.TrayIcon } }; - // TODO: Not nice to do in constructor - // Get a connection - UpdateStatus(false); - m_waitLock = new AutoResetEvent(false); m_requestThread = new Thread(ThreadRunner); m_pollThread = new Thread(LongPollRunner); @@ -176,11 +173,14 @@ namespace Duplicati.GUI.TrayIcon m_pollThread.Start(); } - private void UpdateStatus(bool longpoll) + public Task UpdateStatus() + => UpdateStatusAsync(false); + + private async Task UpdateStatusAsync(bool longpoll) { var query = longpoll ? $"?longpoll=true&lastEventId={m_lastEventId}&duration={LONGPOLL_TIMEOUT}" : ""; - m_status = PerformRequest("GET", $"/serverstate{query}", null, longpoll ? Library.Utility.Timeparser.ParseTimeSpan(LONGPOLL_TIMEOUT) : null); + m_status = await PerformRequestAsync("GET", $"/serverstate{query}", null, longpoll ? Library.Utility.Timeparser.ParseTimeSpan(LONGPOLL_TIMEOUT) : null); m_lastEventId = m_status.LastEventID; if (OnStatusUpdated != null) @@ -189,19 +189,19 @@ namespace Duplicati.GUI.TrayIcon if (m_lastNotificationId != m_status.LastNotificationUpdateID) { m_lastNotificationId = m_status.LastNotificationUpdateID; - UpdateNotifications(); + await UpdateNotificationsAsync().ConfigureAwait(false); } if (m_lastDataUpdateId != m_status.LastDataUpdateID) { m_lastDataUpdateId = m_status.LastDataUpdateID; - UpdateApplicationSettings(); + await UpdateApplicationSettingsAsync().ConfigureAwait(false); } } - private void UpdateNotifications() + private async Task UpdateNotificationsAsync() { - var notifications = PerformRequest("GET", "/notifications", null, null); + var notifications = await PerformRequestAsync("GET", "/notifications", null, null).ConfigureAwait(false); if (notifications != null) { foreach (var n in notifications.Where(x => x.Timestamp > m_firstNotificationTime)) @@ -213,9 +213,9 @@ namespace Duplicati.GUI.TrayIcon } } - private void UpdateApplicationSettings() + private async Task UpdateApplicationSettingsAsync() { - var settings = PerformRequest>("GET", "/serversettings", null, null); + var settings = await PerformRequestAsync>("GET", "/serversettings", null, null).ConfigureAwait(false); if (settings != null && settings.TryGetValue("disable-tray-icon-login", out var str)) m_disableTrayIconLogin = Library.Utility.Utility.ParseBool(str, false); } @@ -235,12 +235,15 @@ namespace Duplicati.GUI.TrayIcon if (waitTime.TotalSeconds > 0) Thread.Sleep(waitTime); started = DateTime.Now; - UpdateStatus(true); + UpdateStatusAsync(true).Await(); errorCount = 0; } catch (Exception ex) { errorCount++; + // More than 1 error, and we are disconnected + if (errorCount >= 2) + Task.Run(() => { OnStatusUpdated?.Invoke(new ServerStatusImpl(LiveControlState.Paused, SuggestedStatusIcon.Disconnected, m_lastEventId, m_lastDataUpdateId, m_lastNotificationId)); }).Await(); System.Diagnostics.Trace.WriteLine("Request error: " + ex.Message); Library.Logging.Log.WriteWarningMessage(LOGTAG, "TrayIconRequestError", ex, "Failed to get response"); } @@ -269,7 +272,7 @@ namespace Duplicati.GUI.TrayIcon if (req != null) { any = true; - PerformRequest(req.Method, req.Endpoint, req.Body, req.Timeout); + PerformRequestAsync(req.Method, req.Endpoint, req.Body, req.Timeout).Await(); } } while (req != null); @@ -300,18 +303,18 @@ namespace Duplicati.GUI.TrayIcon private sealed record SigninResponse(string AccessToken); - private T PerformRequest(string method, string urlfragment, string body, TimeSpan? timeout) + private async Task PerformRequestAsync(string method, string urlfragment, string body, TimeSpan? timeout) { if (string.IsNullOrWhiteSpace(m_accesstoken) && !urlfragment.StartsWith("/auth/")) - ObtainAccessToken(); + await ObtainAccessTokenAsync().ConfigureAwait(false); - var hasTriedPassword = false; + var hasTriedPassword = m_isTryingWithPassword; while (true) { try { - return PerformRequestInternal(method, urlfragment, body, timeout); + return await PerformRequestInternalAsync(method, urlfragment, body, timeout).ConfigureAwait(false); } catch (Exception ex) when (ex is HttpRequestException hex && (hex.StatusCode == HttpStatusCode.Unauthorized || hex.StatusCode == HttpStatusCode.Forbidden) @@ -321,20 +324,31 @@ namespace Duplicati.GUI.TrayIcon if (hasTriedPassword) throw; + // TODO: This error handling is error prone and can end up in infinite recursion + // Should rewrite the entire class and use websockets instead + // Only try once, and clear the token for the next try hasTriedPassword = true; m_accesstoken = null; - ObtainAccessToken(); + try + { + m_isTryingWithPassword = true; + await ObtainAccessTokenAsync().ConfigureAwait(false); + } + finally + { + m_isTryingWithPassword = false; + } } } } - private void ObtainAccessToken() + private async Task ObtainAccessTokenAsync() { - var token = ObtainSignInToken(); + var token = await ObtainSignInTokenAsync().ConfigureAwait(false); if (string.IsNullOrWhiteSpace(token)) return; - m_accesstoken = PerformRequestInternal("POST", "/auth/signin", JsonSerializer.Serialize(new { SigninToken = token }), null).AccessToken; + m_accesstoken = (await PerformRequestInternalAsync("POST", "/auth/signin", JsonSerializer.Serialize(new { SigninToken = token }), null).ConfigureAwait(false)).AccessToken; } private async Task PerformRequestInternalAsync(string method, string endpoint, string body, TimeSpan? timeout) @@ -358,13 +372,10 @@ namespace Duplicati.GUI.TrayIcon if (typeof(T) == typeof(string)) return (T)(object)await response.Content.ReadAsStringAsync().ConfigureAwait(false); - using (var stream = await response.Content.ReadAsStreamAsync()) + using (var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false)) return await JsonSerializer.DeserializeAsync(stream, serializerOptions).ConfigureAwait(false); } - private T PerformRequestInternal(string method, string endpoint, string body, TimeSpan? timeout) - => PerformRequestInternalAsync(method, endpoint, body, timeout).Await(); - private void ExecuteAndNotify(string method, string urifragment, string body) { lock (m_lock) @@ -391,10 +402,10 @@ namespace Duplicati.GUI.TrayIcon } private sealed record SigninTokenResponse(string Token); - private string IssueSigninToken(string password) - => PerformRequest("POST", "/auth/issuesignintoken", JsonSerializer.Serialize(new { Password = password }), null).Token; + private async Task IssueSigninTokenAsync(string password) + => (await PerformRequestInternalAsync("POST", "/auth/issuesignintoken", JsonSerializer.Serialize(new { Password = password }), null).ConfigureAwait(false)).Token; - private string ObtainSignInToken() + private async Task ObtainSignInTokenAsync() { string signinjwt = null; @@ -413,20 +424,17 @@ namespace Duplicati.GUI.TrayIcon // If we know the password, issue a token from the API if (string.IsNullOrWhiteSpace(signinjwt) && !string.IsNullOrWhiteSpace(m_password)) - signinjwt = IssueSigninToken(m_password); + signinjwt = await IssueSigninTokenAsync(m_password); return signinjwt; } - public string StatusWindowURL + public async Task GetStatusWindowURLAsync() { - get - { - var signinjwt = m_disableTrayIconLogin ? null : ObtainSignInToken(); - return string.IsNullOrWhiteSpace(signinjwt) - ? m_baseUri + STATUS_WINDOW - : m_baseUri + SIGNIN_WINDOW + $"?token={signinjwt}"; - } + var signinjwt = m_disableTrayIconLogin ? null : await ObtainSignInTokenAsync().ConfigureAwait(false); + return string.IsNullOrWhiteSpace(signinjwt) + ? m_baseUri + STATUS_WINDOW + : m_baseUri + SIGNIN_WINDOW + $"?token={signinjwt}"; } } } From d9258bfad2104a002c0d2b646fceef6cc3181d78 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sun, 9 Feb 2025 09:48:04 +0100 Subject: [PATCH 132/362] Removed desktop notifications, it did not work --- .../Duplicati.GUI.TrayIcon/AvaloniaRunner.cs | 60 ------------------- .../Duplicati.GUI.TrayIcon.csproj | 1 - 2 files changed, 61 deletions(-) diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/AvaloniaRunner.cs b/Duplicati/GUI/Duplicati.GUI.TrayIcon/AvaloniaRunner.cs index d8f07df43..ae31a1828 100644 --- a/Duplicati/GUI/Duplicati.GUI.TrayIcon/AvaloniaRunner.cs +++ b/Duplicati/GUI/Duplicati.GUI.TrayIcon/AvaloniaRunner.cs @@ -36,64 +36,15 @@ using Avalonia.Themes.Fluent; using Avalonia.Threading; using Duplicati.Library.AutoUpdater; using Duplicati.Library.Logging; -using DesktopNotifications.Windows; -using DesktopNotifications.FreeDesktop; namespace Duplicati.GUI.TrayIcon { - public static class AppBuilderExtensions - { - public static AppBuilder SetupDesktopNotifications(this AppBuilder builder, out DesktopNotifications.INotificationManager? manager) - { - manager = null; - try - { - DesktopNotifications.INotificationManager? realManager = null; - if (OperatingSystem.IsWindows()) - { - var context = WindowsApplicationContext.FromCurrentProcess(); - realManager = new WindowsNotificationManager(context); - } - else if (OperatingSystem.IsLinux()) - { - var context = FreeDesktopApplicationContext.FromCurrentProcess(); - realManager = new FreeDesktopNotificationManager(context); - } - else - { - // MacOS not supported - realManager = null; - } - - if (realManager == null) - return builder; - - realManager.Initialize().GetAwaiter().GetResult(); - builder.AfterSetup(b => - { - if (b.Instance?.ApplicationLifetime is IControlledApplicationLifetime lifetime) - { - lifetime.Exit += (s, e) => { realManager?.Dispose(); }; - } - }); - - manager = realManager; - } - catch (Exception ex) - { - Log.WriteErrorMessage(Log.LogTagFromType(), "DesktopNotificationInitFailed", ex, "Failed to initialize desktop notifications"); - } - return builder; - } - } - public class AvaloniaRunner : TrayIconBase { private static readonly string LOGTAG = Log.LogTagFromType(); private AvaloniaApp? application; private ProcessBasedActionDelay actionDelayer = new ProcessBasedActionDelay(); private IEnumerable menuItems = Enumerable.Empty(); - private DesktopNotifications.INotificationManager? notificationManager; public override void Init(string[] args) { @@ -145,7 +96,6 @@ namespace Duplicati.GUI.TrayIcon .Configure(() => new AvaloniaApp() { Name = AutoUpdateSettings.AppName }) .UsePlatformDetect() .With(new MacOSPlatformOptions() { ShowInDock = false }) - .SetupDesktopNotifications(out notificationManager) .SetupWithLifetime(lifetime); #if DEBUG @@ -220,9 +170,6 @@ namespace Duplicati.GUI.TrayIcon public override void NotifyUser(string title, string message, NotificationType type) { - if (notificationManager == null) - return; - //var icon = Win32NativeNotifyIcon.InfoFlags.NIIF_INFO; switch (type) @@ -237,13 +184,6 @@ namespace Duplicati.GUI.TrayIcon //icon = Win32NativeNotifyIcon.InfoFlags.NIIF_ERROR; break; } - - notificationManager.ShowNotification(new DesktopNotifications.Notification() - { - Title = title, - Body = message - }, DateTimeOffset.UtcNow.AddSeconds(5)); - } protected override void Exit() diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Duplicati.GUI.TrayIcon.csproj b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Duplicati.GUI.TrayIcon.csproj index 410d7ae12..2e38301db 100644 --- a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Duplicati.GUI.TrayIcon.csproj +++ b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Duplicati.GUI.TrayIcon.csproj @@ -32,7 +32,6 @@ - From 273280c9b757554fee80e8d0af46fa989604b1ed Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sun, 9 Feb 2025 10:28:44 +0100 Subject: [PATCH 133/362] Added handler for ctrl+c --- Duplicati/Server/Program.cs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Duplicati/Server/Program.cs b/Duplicati/Server/Program.cs index 2886f1bba..9bedbfcf8 100644 --- a/Duplicati/Server/Program.cs +++ b/Duplicati/Server/Program.cs @@ -311,6 +311,21 @@ namespace Duplicati.Server ApplicationExitEvent.Set(); }); + var stopCounter = 0; + Console.CancelKeyPress += (sender, e) => + { + if (Interlocked.Increment(ref stopCounter) <= 1) + { + Log.WriteInformationMessage(LOGTAG, "CancelKeyPressed", "Cancel key pressed, stopping server"); + Task.Run(() => DuplicatiWebserver?.Stop()); + } + else + { + Log.WriteWarningMessage(LOGTAG, "CancelKeyPressed", null, "Cancel key pressed twice, terminating now"); + Environment.Exit(0); + } + }; + ServerStartedEvent.Set(); ApplicationExitEvent.WaitOne(); } From da1f6562ea7dc9eae9857db525fb1133953c5059 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sun, 9 Feb 2025 10:28:56 +0100 Subject: [PATCH 134/362] Added option to log to console --- Duplicati/Library/RestAPI/Strings.cs | 1 + Duplicati/Server/Program.cs | 51 +++++++++++++++++++--------- 2 files changed, 36 insertions(+), 16 deletions(-) diff --git a/Duplicati/Library/RestAPI/Strings.cs b/Duplicati/Library/RestAPI/Strings.cs index a7e7d3e0c..e7321da48 100644 --- a/Duplicati/Library/RestAPI/Strings.cs +++ b/Duplicati/Library/RestAPI/Strings.cs @@ -41,6 +41,7 @@ Error message: {0}", message); } public static string SkippingSourceArgumentsOnNonBackupOperation { get { return @"The --source argument was specified in the parameter file, but the current operation is not a backup operation, so the argument is ignored"; } } public static string LogfileCommandDescription { get { return LC.L(@"Output log information to the file given"); } } public static string LoglevelCommandDescription { get { return LC.L(@"Determine the amount of information written in the log file"); } } + public static string LogConsoleDescription { get { return LC.L(@"Output log information to the console"); } } public static string PortablemodeCommandDescription { get { return LC.L(@"Activate portable mode where the database is placed below the program executable"); } } public static string SeriousError(string message) { return LC.L(@"A serious error occurred in Duplicati: {0}", message); } public static string TearDownError(string message) { return LC.L(@"An error occurred on server tear down: {0}", message); } diff --git a/Duplicati/Server/Program.cs b/Duplicati/Server/Program.cs index 9bedbfcf8..ed00a0456 100644 --- a/Duplicati/Server/Program.cs +++ b/Duplicati/Server/Program.cs @@ -21,7 +21,6 @@ using System; using System.Collections.Generic; -using System.Globalization; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -52,6 +51,10 @@ namespace Duplicati.Server private const string REQUIRE_DB_ENCRYPTION_KEY_OPTION = "require-db-encryption-key"; private const string SETTINGS_ENCRYPTION_KEY_OPTION = "settings-encryption-key"; private const string DISABLE_UPDATE_CHECK_OPTION = "disable-update-check"; + private const string LOG_FILE_OPTION = "log-file"; + private const string LOG_LEVEL_OPTION = "log-level"; + private const string LOG_CONSOLE_OPTION = "log-console"; + #if DEBUG private const bool DEBUG_MODE = true; @@ -683,30 +686,45 @@ namespace Duplicati.Server private static async Task ApplySecretProvider(Dictionary commandlineOptions, CancellationToken cancellationToken) => FIXMEGlobal.SecretProvider = await SecretProviderHelper.ApplySecretProviderAsync([], [], commandlineOptions, TempFolder.SystemTempPath, FIXMEGlobal.SecretProvider, cancellationToken).ConfigureAwait(false); + private class ConsoleLogDestination(LogMessageType level) : ILogDestination + { + public void WriteMessage(LogEntry entry) + { + if (entry.Level >= level) + Console.WriteLine(entry.AsString(true)); + } + } + private static void ConfigureLogging(Dictionary commandlineOptions) { //Log various information in the logfile - if (DEBUG_MODE && !commandlineOptions.ContainsKey("log-file")) + if (DEBUG_MODE && !commandlineOptions.ContainsKey(LOG_FILE_OPTION)) { var prefix = System.Reflection.Assembly.GetEntryAssembly().GetName().Name.StartsWith("Duplicati.Server") ? "server" : "trayicon"; - commandlineOptions["log-file"] = System.IO.Path.Combine(StartupPath, $"Duplicati-{prefix}.debug.log"); - commandlineOptions["log-level"] = Duplicati.Library.Logging.LogMessageType.Profiling.ToString(); - if (System.IO.File.Exists(commandlineOptions["log-file"])) - { - System.IO.File.Delete(commandlineOptions["log-file"]); - } + commandlineOptions[LOG_FILE_OPTION] = System.IO.Path.Combine(StartupPath, $"Duplicati-{prefix}.debug.log"); + commandlineOptions[LOG_LEVEL_OPTION] = Duplicati.Library.Logging.LogMessageType.Profiling.ToString(); + if (System.IO.File.Exists(commandlineOptions[LOG_FILE_OPTION])) + System.IO.File.Delete(commandlineOptions[LOG_FILE_OPTION]); } - // Setup the log redirect - Library.Logging.Log.StartScope(LogHandler, null); + Log.StartScope(LogHandler, null); - if (commandlineOptions.ContainsKey("log-file")) + if (commandlineOptions.ContainsKey(LOG_FILE_OPTION)) { var loglevel = Library.Logging.LogMessageType.Warning; - if (commandlineOptions.ContainsKey("log-level")) - Enum.TryParse(commandlineOptions["log-level"], true, out loglevel); + if (commandlineOptions.ContainsKey(LOG_LEVEL_OPTION)) + Enum.TryParse(commandlineOptions[LOG_LEVEL_OPTION], true, out loglevel); - LogHandler.SetServerFile(commandlineOptions["log-file"], loglevel); + LogHandler.SetServerFile(commandlineOptions[LOG_FILE_OPTION], loglevel); + } + + if (Library.Utility.Utility.ParseBoolOption(commandlineOptions, LOG_CONSOLE_OPTION)) + { + var loglevel = Library.Logging.LogMessageType.Information; + if (commandlineOptions.ContainsKey(LOG_LEVEL_OPTION)) + Enum.TryParse(commandlineOptions[LOG_LEVEL_OPTION], true, out loglevel); + + LogHandler.AppendLogDestination(new ConsoleLogDestination(loglevel), loglevel); } if (commandlineOptions.TryGetValue(WINDOWS_EVENTLOG_OPTION, out var source) && !string.IsNullOrEmpty(source)) @@ -957,8 +975,9 @@ namespace Duplicati.Server new Duplicati.Library.Interface.CommandLineArgument("help", Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.Program.HelpCommandDescription, Strings.Program.HelpCommandDescription), new Duplicati.Library.Interface.CommandLineArgument("parameters-file", Library.Interface.CommandLineArgument.ArgumentType.Path, Strings.Program.ParametersFileOptionShort, Strings.Program.ParametersFileOptionLong2, "", ParameterFileOptionStrings), new Duplicati.Library.Interface.CommandLineArgument(DataFolderManager.PORTABLE_MODE_OPTION, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.Program.PortablemodeCommandDescription, Strings.Program.PortablemodeCommandDescription, DataFolderManager.PORTABLE_MODE.ToString().ToLowerInvariant()), - new Duplicati.Library.Interface.CommandLineArgument("log-file", Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Path, Strings.Program.LogfileCommandDescription, Strings.Program.LogfileCommandDescription), - new Duplicati.Library.Interface.CommandLineArgument("log-level", Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Enumeration, Strings.Program.LoglevelCommandDescription, Strings.Program.LoglevelCommandDescription, Library.Logging.LogMessageType.Warning.ToString(), null, Enum.GetNames(typeof(Duplicati.Library.Logging.LogMessageType))), + new Duplicati.Library.Interface.CommandLineArgument(LOG_FILE_OPTION, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Path, Strings.Program.LogfileCommandDescription, Strings.Program.LogfileCommandDescription), + new Duplicati.Library.Interface.CommandLineArgument(LOG_LEVEL_OPTION, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Enumeration, Strings.Program.LoglevelCommandDescription, Strings.Program.LoglevelCommandDescription, Library.Logging.LogMessageType.Warning.ToString(), null, Enum.GetNames(typeof(Duplicati.Library.Logging.LogMessageType))), + new Duplicati.Library.Interface.CommandLineArgument(LOG_CONSOLE_OPTION, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.Program.LogConsoleDescription, Strings.Program.LogConsoleDescription, false.ToString()), new Duplicati.Library.Interface.CommandLineArgument(WebServerLoader.OPTION_WEBROOT, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Path, Strings.Program.WebserverWebrootDescription, Strings.Program.WebserverWebrootDescription, WebServerLoader.DEFAULT_OPTION_WEBROOT), new Duplicati.Library.Interface.CommandLineArgument(WebServerLoader.OPTION_PORT, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.String, Strings.Program.WebserverPortDescription, Strings.Program.WebserverPortDescription, WebServerLoader.DEFAULT_OPTION_PORT.ToString()), new Duplicati.Library.Interface.CommandLineArgument(WebServerLoader.OPTION_WEBSERVICE_DISABLEHTTPS, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.String, Strings.Program.WebserverDisableHTTPSDescription, Strings.Program.WebserverDisableHTTPSDescription), From f22026cf5985a999e76d7375c79340abe0cb92c3 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sun, 9 Feb 2025 10:52:47 +0100 Subject: [PATCH 135/362] Fixed a compile issue with older SDKs --- Duplicati/Library/Utility/Utility.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index 4d2d8e423..fdb2a6713 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -807,7 +807,7 @@ namespace Duplicati.Library.Utility return @default; var flags = 0; - foreach (var s in value.Split([','], StringSplitOptions.RemoveEmptyEntries)) + foreach (var s in value.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries)) { var trimmed = s.Trim(); if (Enum.TryParse(trimmed, true, out T flag)) From 3da02c552c932834fe90603c324233685b596c8e Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sun, 9 Feb 2025 11:01:21 +0100 Subject: [PATCH 136/362] Added force terminate support for Windows --- Duplicati/Server/Program.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Duplicati/Server/Program.cs b/Duplicati/Server/Program.cs index ed00a0456..33aedb0bf 100644 --- a/Duplicati/Server/Program.cs +++ b/Duplicati/Server/Program.cs @@ -325,7 +325,10 @@ namespace Duplicati.Server else { Log.WriteWarningMessage(LOGTAG, "CancelKeyPressed", null, "Cancel key pressed twice, terminating now"); - Environment.Exit(0); + if (OperatingSystem.IsWindows()) + Environment.FailFast("Cancel key pressed twice, terminating now"); + else + Environment.Exit(0); } }; From bb3ea2f6cdf101974b86f0b566624cda52602a00 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sun, 9 Feb 2025 11:05:54 +0100 Subject: [PATCH 137/362] Fixed a rare startup issue --- Duplicati/Library/RestAPI/Scheduler.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/RestAPI/Scheduler.cs b/Duplicati/Library/RestAPI/Scheduler.cs index ba7bd40cf..281efc0fd 100644 --- a/Duplicati/Library/RestAPI/Scheduler.cs +++ b/Duplicati/Library/RestAPI/Scheduler.cs @@ -152,7 +152,7 @@ namespace Duplicati.Server /// public List WorkerQueue { - get { return (from t in m_worker.CurrentTasks where t != null select t).ToList(); } + get { return (from t in m_worker?.CurrentTasks where t != null select t)?.ToList() ?? []; } } /// From 42044543ff84f3165257f2c3996cec6101d8c64b Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sun, 9 Feb 2025 11:28:52 +0100 Subject: [PATCH 138/362] Added support for disabled signin tokens and trayicon --- .../HttpServerConnection.cs | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/HttpServerConnection.cs b/Duplicati/GUI/Duplicati.GUI.TrayIcon/HttpServerConnection.cs index d6f6b7a54..6a489ac09 100644 --- a/Duplicati/GUI/Duplicati.GUI.TrayIcon/HttpServerConnection.cs +++ b/Duplicati/GUI/Duplicati.GUI.TrayIcon/HttpServerConnection.cs @@ -345,9 +345,16 @@ namespace Duplicati.GUI.TrayIcon private async Task ObtainAccessTokenAsync() { + if (!string.IsNullOrWhiteSpace(m_password)) + { + m_accesstoken = (await PerformRequestInternalAsync("POST", "/auth/login", JsonSerializer.Serialize(new { Password = m_password }), null).ConfigureAwait(false)).AccessToken; + return; + } + var token = await ObtainSignInTokenAsync().ConfigureAwait(false); if (string.IsNullOrWhiteSpace(token)) return; + m_accesstoken = (await PerformRequestInternalAsync("POST", "/auth/signin", JsonSerializer.Serialize(new { SigninToken = token }), null).ConfigureAwait(false)).AccessToken; } @@ -411,12 +418,20 @@ namespace Duplicati.GUI.TrayIcon // If we host the server, issue the token from the service if (FIXMEGlobal.IsServerStarted && m_passwordSource == Program.PasswordSource.HostedServer) + { + if (FIXMEGlobal.DataConnection.ApplicationSettings.DisableSigninTokens) + return null; + signinjwt = FIXMEGlobal.Provider.GetRequiredService().CreateSigninToken("trayicon"); + } // If we have database access, grab the issuer key from the db and issue a token if (string.IsNullOrWhiteSpace(signinjwt) && m_passwordSource == Program.PasswordSource.Database) { Program.databaseConnection.ApplicationSettings.ReloadSettings(); + if (Program.databaseConnection.ApplicationSettings.DisableSigninTokens) + return null; + var cfg = Program.databaseConnection.ApplicationSettings.JWTConfig; if (!string.IsNullOrWhiteSpace(cfg)) signinjwt = new JWTTokenProvider(JsonSerializer.Deserialize(cfg)).CreateSigninToken("trayicon"); @@ -431,7 +446,18 @@ namespace Duplicati.GUI.TrayIcon public async Task GetStatusWindowURLAsync() { - var signinjwt = m_disableTrayIconLogin ? null : await ObtainSignInTokenAsync().ConfigureAwait(false); + string signinjwt = null; + if (!m_disableTrayIconLogin) + { + try + { + signinjwt = await ObtainSignInTokenAsync().ConfigureAwait(false); + } + catch + { + } + } + return string.IsNullOrWhiteSpace(signinjwt) ? m_baseUri + STATUS_WINDOW : m_baseUri + SIGNIN_WINDOW + $"?token={signinjwt}"; From d5cc0ffbb5dd7798978e48f198bd5cc599c451a3 Mon Sep 17 00:00:00 2001 From: "Marcelo C." Date: Sun, 9 Feb 2025 12:13:12 -0300 Subject: [PATCH 139/362] Adding 429 observance to backend This will make listing buckets with large count of files more reliable, if a 429 response is received it will conform to retry-after headers or if not present (as is the case today) will default to a 5 second delay. --- Duplicati/Library/Backend/Backblaze/B2.cs | 139 +++++++++++------- .../Library/Backend/Backblaze/B2AuthHelper.cs | 41 +++--- .../Library/Interface/CustomExceptions.cs | 48 +++--- 3 files changed, 134 insertions(+), 94 deletions(-) diff --git a/Duplicati/Library/Backend/Backblaze/B2.cs b/Duplicati/Library/Backend/Backblaze/B2.cs index d07ca477f..08c2437dc 100644 --- a/Duplicati/Library/Backend/Backblaze/B2.cs +++ b/Duplicati/Library/Backend/Backblaze/B2.cs @@ -1,30 +1,33 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +using System.Diagnostics; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using Duplicati.Library.Utility; using System.Net; +using System.Net.Http.Headers; using System.Reflection; using Duplicati.Library.Backend.Backblaze.Model; +using FileEntry = Duplicati.Library.Common.IO.FileEntry; namespace Duplicati.Library.Backend.Backblaze; @@ -77,6 +80,11 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend /// Recommended chunk size as per Backblaze B2 documentation (100MB) /// private const int B2_RECOMMENDED_CHUNK_SIZE = 100 * 1024 * 1024; + + /// + /// Default retry-after time in seconds for B2 API requests + /// + private const int B2_RETRY_AFTER_SECONDS = 5; /// /// The name of the B2 bucket being accessed @@ -455,53 +463,72 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend /// Implementation of interface method for listing remote folder contents /// /// List of IFileEntry with directory listing result - private async Task> ListAsync(CancellationToken cancellationToken) + private async Task> ListAsync(CancellationToken cancellationToken, bool testMode = false) { _filecache = null; var cache = new Dictionary>(); string nextFileId = null; string nextFileName = null; + string listVersionUrl = $"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_list_file_versions"; + + var listRetryHelper = RetryAfterHelper.CreateOrGetRetryAfterHelper(listVersionUrl); + do { - - using var timeoutToken = new CancellationTokenSource(); - timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS)); - using var combinedCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken); - - var resp = await _b2AuthHelper.PostAndGetJsonDataAsync( - $"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_list_file_versions", - new ListFilesRequest - { - BucketID = GetBucketAsync(cancellationToken).Await().BucketID, - MaxFileCount = _pageSize, - Prefix = _prefix, - StartFileID = nextFileId, - StartFileName = nextFileName - }, - combinedCancellationToken.Token - ).ConfigureAwait(false); - - nextFileId = resp.NextFileID; - nextFileName = resp.NextFileName; - - if (resp.Files == null || resp.Files.Length == 0) - break; - - foreach (var file in resp.Files) + try { - if (!file.FileName.StartsWith(_prefix, StringComparison.Ordinal)) - continue; + using var timeoutToken = new CancellationTokenSource(); + timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS)); + using var combinedCancellationToken = + CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken); - var name = file.FileName[_prefix.Length..]; - if (name.Contains('/')) - continue; + var resp = await _b2AuthHelper.PostAndGetJsonDataAsync( + $"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_list_file_versions", + new ListFilesRequest + { + BucketID = GetBucketAsync(cancellationToken).Await().BucketID, + MaxFileCount = _pageSize, + Prefix = _prefix, + StartFileID = nextFileId, + StartFileName = nextFileName + }, + combinedCancellationToken.Token + ).ConfigureAwait(false); - cache.TryGetValue(name, out var lst); - if (lst == null) - cache[name] = lst = new List(1); - lst.Add(file); + nextFileId = resp.NextFileID; + nextFileName = resp.NextFileName; + + if (resp.Files == null || resp.Files.Length == 0) + break; + + foreach (var file in resp.Files) + { + if (!file.FileName.StartsWith(_prefix, StringComparison.Ordinal)) + continue; + + var name = file.FileName[_prefix.Length..]; + if (name.Contains('/')) + continue; + + cache.TryGetValue(name, out var lst); + if (lst == null) + cache[name] = lst = new System.Collections.Generic.List(1); + lst.Add(file); + } } - + catch (TooManyRequestException tex) + { + // Backblaze's B2 API rate limit reached. Presently they don't add Retry-After headers, we will default to a delay, but respect it if present. + if (tex.RetryAfter == null || (tex.RetryAfter.Date == null && tex.RetryAfter.Delta == null)) + listRetryHelper.SetRetryAfter( + new RetryConditionHeaderValue(TimeSpan.FromSeconds(B2_RETRY_AFTER_SECONDS))); + else + listRetryHelper.SetRetryAfter(tex.RetryAfter); + + await listRetryHelper.WaitForRetryAfterAsync(cancellationToken); + } + if (testMode) // For testing purposes, it suffices to get the first page of results. + break; } while (nextFileId != null); _filecache = cache; @@ -589,7 +616,7 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend /// Cancellation Token public Task TestAsync(CancellationToken cancelToken) { - this.TestList(); + _ = ListAsync(cancelToken, true).Await(); return Task.CompletedTask; } diff --git a/Duplicati/Library/Backend/Backblaze/B2AuthHelper.cs b/Duplicati/Library/Backend/Backblaze/B2AuthHelper.cs index 6c9003a66..18ad6113a 100644 --- a/Duplicati/Library/Backend/Backblaze/B2AuthHelper.cs +++ b/Duplicati/Library/Backend/Backblaze/B2AuthHelper.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System.Net; @@ -202,6 +202,9 @@ public class B2AuthHelper(string userid, string password, HttpClient httpClient) if (ex is not HttpRequestException || responseContext == null) return; + if (ex is HttpRequestException && responseContext != null && responseContext.StatusCode == HttpStatusCode.TooManyRequests) + throw new TooManyRequestException(responseContext.Headers.RetryAfter); + using var stream = responseContext.Content.ReadAsStream(); using var reader = new StreamReader(stream); var rawData = reader.ReadToEnd(); diff --git a/Duplicati/Library/Interface/CustomExceptions.cs b/Duplicati/Library/Interface/CustomExceptions.cs index 5722ab0ad..bbd50cc87 100644 --- a/Duplicati/Library/Interface/CustomExceptions.cs +++ b/Duplicati/Library/Interface/CustomExceptions.cs @@ -1,25 +1,26 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; +using System.Net.Http.Headers; using Duplicati.Library.Localization.Short; namespace Duplicati.Library.Interface @@ -225,4 +226,13 @@ namespace Duplicati.Library.Interface : base(Strings.Common.SettingsKeyMissingExceptionError, "SettingsKeyMissing") { } } + + /// + /// An exception for carrying the Retry-After header value on 429 Exceptions + /// + [Serializable] + public class TooManyRequestException(RetryConditionHeaderValue retryAfter) : Exception + { + public readonly RetryConditionHeaderValue RetryAfter = retryAfter; + } } From 8c935a7344df2678cd8af638e59af86e375719e7 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sun, 9 Feb 2025 16:14:22 +0100 Subject: [PATCH 140/362] This creates a new ISourceProvider interface for describing a data source. With this interface it is possible to wrap the current support for snapshots and USN and simplify the enumeration. This makes a number of assumptions into fields such as `IsMetaEntry` and `IsRootEntry` that removes code that was checking for various properties. This first commit has support for VSS and USN as well as "naked" files, keeping feature parity. There is also an experimental support for loading a backend-as-a-source. The support for this is currently limited as the backends need to be extended to support a folder concept. --- Duplicati.sln | 15 + .../Duplicati.Library.Backend.File.csproj | 1 + Duplicati/Library/Backend/File/FileBackend.cs | 138 ++++++-- .../Library/Backend/SSHv2/SSHv2Backend.cs | 42 +-- .../Library/Common/IO/VssBackupComponents.cs | 40 ++- .../Duplicati.Library.DynamicLoader.csproj | 1 + .../DynamicLoader/SourceProviderLoader.cs | 180 ++++++++++ .../Interface/IFolderEnabledBackend.cs | 53 +++ .../Library/Interface/ISourceFileEntry.cs | 126 +++++++ .../Library/Interface/ISourceProvider.cs | 51 +++ .../Interface/ISourceProviderModule.cs | 39 +++ .../Main/Duplicati.Library.Main.csproj | 2 + .../Library/Main/Operation/Backup/Channels.cs | 3 +- .../Operation/Backup/CountFilesHandler.cs | 58 ++-- .../Operation/Backup/FileBlockProcessor.cs | 67 ++-- .../Backup/FileEnumerationProcess.cs | 325 +++++++++++------- .../Operation/Backup/FilePreFilterProcess.cs | 66 ++-- .../Operation/Backup/MetadataGenerator.cs | 59 ++-- .../Operation/Backup/MetadataPreProcess.cs | 95 ++--- .../Library/Main/Operation/BackupHandler.cs | 143 ++++++-- .../Main/Operation/TestFilterHandler.cs | 86 ++--- .../Library/Modules/Builtin/ReportHelper.cs | 2 +- .../Modules/Builtin/SendJabberMessage.cs | 3 +- .../Duplicati.Library.Snapshots.csproj | 4 +- .../Library/Snapshots/ISnapshotService.cs | 102 +++--- Duplicati/Library/Snapshots/LinuxSnapshot.cs | 103 ++++-- .../Library/Snapshots/NoSnapshotLinux.cs | 74 ++-- .../Library/Snapshots/NoSnapshotWindows.cs | 134 ++++---- Duplicati/Library/Snapshots/Program.cs | 2 +- .../Snapshots/RecursiveFilterHelper.cs | 229 ++++++++++++ Duplicati/Library/Snapshots/SnapshotBase.cs | 89 +++-- .../Snapshots/SnapshotSourceFileEntry.cs | 143 ++++++++ .../Library/Snapshots/SnapshotUtility.cs | 92 ++--- .../Library/Snapshots/UsnJournalService.cs | 227 +----------- .../Library/Snapshots/WindowsSnapshot.cs | 93 ++--- .../Builtin/BackendSourceProvider.cs | 211 ++++++++++++ .../SourceProvider/Builtin/Combiner.cs | 82 +++++ ...cati.Library.SourceProvider.Builtin.csproj | 27 ++ .../Library/SourceProvider/Builtin/File.cs | 52 +++ .../Library/SourceProvider/Builtin/Strings.cs | 29 ++ .../Duplicati.Library.SourceProviders.csproj | 14 + .../SourceProviders/SourceProviderModules.cs | 45 +++ Duplicati/Library/Utility/MapStream.cs | 176 ++++++++++ Duplicati/Library/Utility/TempFileStream.cs | 87 +++++ Duplicati/Library/Utility/Utility.cs | 65 +--- .../WebserverCore/Endpoints/V1/Filesystem.cs | 8 +- 46 files changed, 2702 insertions(+), 981 deletions(-) create mode 100644 Duplicati/Library/DynamicLoader/SourceProviderLoader.cs create mode 100644 Duplicati/Library/Interface/IFolderEnabledBackend.cs create mode 100644 Duplicati/Library/Interface/ISourceFileEntry.cs create mode 100644 Duplicati/Library/Interface/ISourceProvider.cs create mode 100644 Duplicati/Library/Interface/ISourceProviderModule.cs create mode 100644 Duplicati/Library/Snapshots/RecursiveFilterHelper.cs create mode 100644 Duplicati/Library/Snapshots/SnapshotSourceFileEntry.cs create mode 100644 Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs create mode 100644 Duplicati/Library/SourceProvider/Builtin/Combiner.cs create mode 100644 Duplicati/Library/SourceProvider/Builtin/Duplicati.Library.SourceProvider.Builtin.csproj create mode 100644 Duplicati/Library/SourceProvider/Builtin/File.cs create mode 100644 Duplicati/Library/SourceProvider/Builtin/Strings.cs create mode 100644 Duplicati/Library/SourceProviders/Duplicati.Library.SourceProviders.csproj create mode 100644 Duplicati/Library/SourceProviders/SourceProviderModules.cs create mode 100644 Duplicati/Library/Utility/MapStream.cs create mode 100644 Duplicati/Library/Utility/TempFileStream.cs diff --git a/Duplicati.sln b/Duplicati.sln index 879a03397..eaf3aee27 100644 --- a/Duplicati.sln +++ b/Duplicati.sln @@ -166,6 +166,12 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.Crashlog" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.Backend.CIFS", "Duplicati\Library\Backend\CIFS\Duplicati.Library.Backend.CIFS.csproj", "{836E0557-B40C-4DC7-9A2A-5C062F9ACC6B}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.SourceProviders", "Duplicati\Library\SourceProviders\Duplicati.Library.SourceProviders.csproj", "{8A2C7A9F-3EC8-4DE5-A9FA-9E4BCF955EF3}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SourceProviders", "SourceProviders", "{424CEF73-4984-430E-9C8B-E61CC0F22074}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.SourceProvider.Builtin", "Duplicati\Library\SourceProvider\Builtin\Duplicati.Library.SourceProvider.Builtin.csproj", "{6D9742A7-6F32-4571-8276-79716168B525}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -472,6 +478,14 @@ Global {836E0557-B40C-4DC7-9A2A-5C062F9ACC6B}.Debug|Any CPU.Build.0 = Debug|Any CPU {836E0557-B40C-4DC7-9A2A-5C062F9ACC6B}.Release|Any CPU.ActiveCfg = Release|Any CPU {836E0557-B40C-4DC7-9A2A-5C062F9ACC6B}.Release|Any CPU.Build.0 = Release|Any CPU + {8A2C7A9F-3EC8-4DE5-A9FA-9E4BCF955EF3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8A2C7A9F-3EC8-4DE5-A9FA-9E4BCF955EF3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8A2C7A9F-3EC8-4DE5-A9FA-9E4BCF955EF3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8A2C7A9F-3EC8-4DE5-A9FA-9E4BCF955EF3}.Release|Any CPU.Build.0 = Release|Any CPU + {6D9742A7-6F32-4571-8276-79716168B525}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6D9742A7-6F32-4571-8276-79716168B525}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6D9742A7-6F32-4571-8276-79716168B525}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6D9742A7-6F32-4571-8276-79716168B525}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -532,6 +546,7 @@ Global {CA1A0EE8-DFFF-42AF-B15D-EF538357A2DE} = {E1A9B303-F281-45C5-A4F6-CADD9DE3F3C4} {8ACA2736-3C69-4FA5-BCF9-5EDF50CAF332} = {566EBBDA-19A4-4056-A615-D901D57D2439} {836E0557-B40C-4DC7-9A2A-5C062F9ACC6B} = {E1A9B303-F281-45C5-A4F6-CADD9DE3F3C4} + {6D9742A7-6F32-4571-8276-79716168B525} = {424CEF73-4984-430E-9C8B-E61CC0F22074} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {8B40BAFE-D862-4397-9495-8F5EAF5CE80C} diff --git a/Duplicati/Library/Backend/File/Duplicati.Library.Backend.File.csproj b/Duplicati/Library/Backend/File/Duplicati.Library.Backend.File.csproj index f8a99e4c3..60e2917a7 100644 --- a/Duplicati/Library/Backend/File/Duplicati.Library.Backend.File.csproj +++ b/Duplicati/Library/Backend/File/Duplicati.Library.Backend.File.csproj @@ -11,6 +11,7 @@ + diff --git a/Duplicati/Library/Backend/File/FileBackend.cs b/Duplicati/Library/Backend/File/FileBackend.cs index 89dbb44fc..de049627b 100644 --- a/Duplicati/Library/Backend/File/FileBackend.cs +++ b/Duplicati/Library/Backend/File/FileBackend.cs @@ -1,29 +1,32 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; +using Duplicati.Library.Snapshots; using System; using System.Collections.Generic; using System.IO; +using System.Linq; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Versioning; using System.Threading; @@ -33,7 +36,7 @@ namespace Duplicati.Library.Backend { // ReSharper disable once UnusedMember.Global // This class is instantiated dynamically in the BackendLoader. - public class File : IBackend, IStreamingBackend, IQuotaEnabledBackend, IRenameEnabledBackend + public class File : IBackend, IStreamingBackend, IQuotaEnabledBackend, IRenameEnabledBackend, IFolderEnabledBackend { private const string OPTION_DESTINATION_MARKER = "alternate-destination-marker"; private const string OPTION_ALTERNATE_PATHS = "alternate-target-paths"; @@ -452,16 +455,107 @@ namespace Duplicati.Library.Backend long? sourceStreamLength = sourceStream == null ? null : Utility.Utility.GetStreamLength(sourceStream, out isStreamPostion); if (sourceStreamLength.HasValue && targetFileInfo.Length != sourceStreamLength.Value) - throw new FileMissingException($"Target file size ({targetFileInfo.Length:n0}) is different from the source length ({sourceStreamLength.Value:n0}){(isStreamPostion ? " - ending stream position)" : "")}. Target: {targetFilePath}"); + throw new FileMissingException($"Target file size ({targetFileInfo.Length:n0}) is different from the source length ({sourceStreamLength.Value:n0}){(isStreamPostion ? " - ending stream position" : "")}. Target: {targetFilePath}"); if (expectedLength.HasValue && targetFileInfo.Length != expectedLength.Value) throw new FileMissingException($"Target file size ({targetFileInfo.Length:n0}) is different from the expected length ({expectedLength.Value:n0}). Target: {targetFilePath}"); } catch { - try { System.IO.File.Delete(targetFilePath); } catch { } + try { System.IO.File.Delete(targetFilePath); } + catch { } throw; } } + + /// + public async IAsyncEnumerable ListAsync(string path, [EnumeratorCancellation] CancellationToken cancellationToken) + { + PreAuthenticate(); + + var remotePath = string.IsNullOrWhiteSpace(path) ? m_path : GetRemoteName(path); + + if (!systemIO.DirectoryExists(remotePath)) + throw new FolderMissingException(Strings.FileBackend.FolderMissingError(m_path)); + + var service = SnapshotUtility.CreateNoSnapshot([remotePath], false, false); + + await foreach (var folderEntry in systemIO.EnumerateDirectories(remotePath).ToAsyncEnumerable().WithCancellation(cancellationToken).ConfigureAwait(false)) + yield return new SourceFileEntry(service, folderEntry, true); + + await foreach (var fileEntry in systemIO.EnumerateFiles(remotePath).ToAsyncEnumerable().WithCancellation(cancellationToken).ConfigureAwait(false)) + yield return new SourceFileEntry(service, fileEntry, false); + } + + /// + public Task GetEntryAsync(string path, CancellationToken cancellationToken) + { + var service = SnapshotUtility.CreateNoSnapshot([path], false, false); + var isFolder = service.DirectoryExists(path); + return Task.FromResult(new SourceFileEntry(service, path, isFolder)); + } + + private class SourceFileEntry(ISnapshotService service, string path, bool isFolder) : ISourceFileEntry + { + /// + public bool IsFolder => isFolder; + + /// + public bool IsMetaEntry => false; + /// + public bool IsRootEntry => false; + + /// + public DateTime CreatedUtc => service.GetCreationTimeUtc(path); + + /// + public DateTime LastModificationUtc => service.GetLastWriteTimeUtc(path); + + /// + public string Path => path; + + /// + public long Size => isFolder ? -1 : service.GetFileSize(path); + + /// + public bool IsSymlink => service.IsSymlink(path, Attributes); + + /// + public string SymlinkTarget => service.GetSymlinkTarget(path); + + /// + public FileAttributes Attributes => service.GetAttributes(path); + + /// + public Dictionary MinorMetadata => service.GetMetadata(path, IsSymlink); + + /// + public bool IsBlockDevice => service.IsBlockDevice(path); + + /// + public bool IsCharacterDevice => false; + + /// + public bool IsAlternateStream => false; + + /// + public string HardlinkTargetId => service.HardlinkTargetID(path); + + /// + public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) + => service.EnumerateFilesystemEntries(this).ToAsyncEnumerable(); + + /// + public Task FileExists(string filename, CancellationToken cancellationToken) + => Task.FromResult(service.FileExists(systemIO.PathCombine(Path, filename))); + + /// + public Task OpenMetadataRead(CancellationToken cancellationToken) + => Task.FromResult(new MemoryStream(System.Text.Json.JsonSerializer.SerializeToUtf8Bytes(MinorMetadata))); + + /// + public Task OpenRead(CancellationToken cancellationToken) + => service.OpenReadAsync(path, cancellationToken); + } } } diff --git a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs index 073217e2b..05a1bfb7d 100644 --- a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs +++ b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. @@ -396,7 +396,7 @@ namespace Duplicati.Library.Backend { if (!filename.StartsWith(KEYFILE_URI, StringComparison.OrdinalIgnoreCase)) { - return String.IsNullOrEmpty(password) + return string.IsNullOrEmpty(password) ? new PrivateKeyFile(filename) : new PrivateKeyFile(filename, password); } @@ -409,7 +409,7 @@ namespace Duplicati.Library.Backend ms.Position = 0; - return String.IsNullOrEmpty(password) ? new PrivateKeyFile(ms) : new PrivateKeyFile(ms, password); + return string.IsNullOrEmpty(password) ? new PrivateKeyFile(ms) : new PrivateKeyFile(ms, password); } } catch (Exception ex) diff --git a/Duplicati/Library/Common/IO/VssBackupComponents.cs b/Duplicati/Library/Common/IO/VssBackupComponents.cs index 8b87aa044..d2524c140 100644 --- a/Duplicati/Library/Common/IO/VssBackupComponents.cs +++ b/Duplicati/Library/Common/IO/VssBackupComponents.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -147,6 +147,8 @@ namespace Duplicati.Library.Common.IO } } + public IEnumerable SnapshotLocalPaths => _volumeMap.Keys; + private List GetPathsFromComponent(IVssWMComponent component) { var paths = new List(); diff --git a/Duplicati/Library/DynamicLoader/Duplicati.Library.DynamicLoader.csproj b/Duplicati/Library/DynamicLoader/Duplicati.Library.DynamicLoader.csproj index 512e0f909..0ed1b1e42 100644 --- a/Duplicati/Library/DynamicLoader/Duplicati.Library.DynamicLoader.csproj +++ b/Duplicati/Library/DynamicLoader/Duplicati.Library.DynamicLoader.csproj @@ -17,6 +17,7 @@ + diff --git a/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs b/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs new file mode 100644 index 000000000..5b38ac617 --- /dev/null +++ b/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs @@ -0,0 +1,180 @@ +// Copyright (C) 2024, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using System; +using System.Collections.Generic; +using Duplicati.Library.Interface; +using System.Linq; +using System.Runtime.ExceptionServices; +using Duplicati.Library.SourceProviders; + +namespace Duplicati.Library.DynamicLoader +{ + /// + /// Loads all SourceProviders dynamically and exposes a list of those + /// + public class SourceProviderLoader + { + /// + /// Implementation overrides specific to SourceProviders + /// + private class SourceProviderLoaderSub : DynamicLoader + { + /// + /// Returns the protocol key + /// + /// The item to load the key for + /// The protocol key + protected override string GetInterfaceKey(ISourceProviderModule item) + { + return item.Key; + } + + /// + /// Returns the subfolders searched for SourceProviders + /// + protected override string[] Subfolders => ["SourceProviders"]; + + /// + /// The built-in modules + /// + protected override IEnumerable BuiltInModules => SourceProviderModules.BuiltInSourceProviderModules; + + /// + /// Instanciates a specific SourceProvider, given the url and options + /// + /// The url to create the instance for + /// The options to pass to the instance constructor + /// The instanciated SourceProvider or null if the url is not supported + public ISourceProviderModule GetSourceProvider(string url, Dictionary options) + { + var uri = new Utility.Uri(url); + + LoadInterfaces(); + + var newOpts = new Dictionary(options); + foreach (var key in uri.QueryParameters.AllKeys) + newOpts[key] = uri.QueryParameters[key]; + + lock (m_lock) + { + try + { + if (m_interfaces.ContainsKey(uri.Scheme)) + return (ISourceProviderModule)Activator.CreateInstance(m_interfaces[uri.Scheme].GetType(), url, newOpts); + } + catch (System.Reflection.TargetInvocationException tex) + { + if (tex.InnerException != null) + { + // Unwrap exceptions for nicer display. The ExceptionDispatchInfo class allows us to + // rethrow an exception without changing the stack trace. + ExceptionDispatchInfo.Capture(tex.InnerException).Throw(); + } + + throw; + } + + return null; + } + } + + /// + /// Gets the supported commands for a certain url + /// + /// The url to find commands for + /// The supported commands or null if the url scheme was not supported + public IReadOnlyList GetSupportedCommands(string url) + { + var uri = new Utility.Uri(url); + + LoadInterfaces(); + + // TODO: The loading logic is replicated in the "GetSourceProvider" method, should be refactored + lock (m_lock) + { + ISourceProviderModule b; + if (m_interfaces.TryGetValue(uri.Scheme, out b) && b != null) + return GetSupportedCommandsCached(b).ToList(); + else if (uri.Scheme.EndsWith("s", StringComparison.Ordinal)) + { + var tmpscheme = uri.Scheme.Substring(0, uri.Scheme.Length - 1); + if (m_interfaces.TryGetValue(tmpscheme, out b) && b != null) + return GetSupportedCommandsCached(b).ToList(); + } + + return null; + } + } + } + + /// + /// The static instance used to access SourceProvider information + /// + private static readonly SourceProviderLoaderSub _SourceProviderLoader = new SourceProviderLoaderSub(); + + #region Public static API + + /// + /// Gets a list of loaded SourceProviders, the instances can be used to extract interface information, not used to interact with the SourceProvider. + /// + public static ISourceProviderModule[] SourceProviders { get { return _SourceProviderLoader.Interfaces; } } + + /// + /// Gets a list of keys supported + /// + public static string[] Keys { get { return _SourceProviderLoader.Keys; } } + + /// + /// Gets the supported commands for a given SourceProvider + /// + /// The url to find the commands for + /// The supported commands or null if the url is not supported + public static IReadOnlyList GetSupportedCommands(string url) + { + if (string.IsNullOrEmpty(url)) + throw new ArgumentNullException(nameof(url)); + + return _SourceProviderLoader.GetSupportedCommands(url); + } + + /// + /// Instanciates a specific SourceProvider, given the url and options + /// + /// The url to create the instance for + /// The options to pass to the instance constructor + /// The instanciated SourceProvider or null if the url is not supported + public static ISourceProviderModule GetSourceProvider(string url, Dictionary options) + { + return _SourceProviderLoader.GetSourceProvider(url, options); + } + #endregion + + /// + /// Adds a SourceProvider to the loader + /// + /// The SourceProvider to add + public static void AddSourceProvider(ISourceProviderModule SourceProvider) + { + _SourceProviderLoader.AddModule(SourceProvider); + } + } +} diff --git a/Duplicati/Library/Interface/IFolderEnabledBackend.cs b/Duplicati/Library/Interface/IFolderEnabledBackend.cs new file mode 100644 index 000000000..2ccd3f44c --- /dev/null +++ b/Duplicati/Library/Interface/IFolderEnabledBackend.cs @@ -0,0 +1,53 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace Duplicati.Library.Interface; + +/// +/// An interface a backend may implement if it supports streaming operations. +/// Backends that implement this interface can be throttled and correctly shows +/// the progressbar when transferring data. +/// +public interface IFolderEnabledBackend : IBackend +{ + /// + /// Enumerates a list of files and folders found on the remote location + /// + /// The path to list, empty or null for the root folder + /// The cancellation token + /// The list of files + IAsyncEnumerable ListAsync(string? path, CancellationToken cancellationToken); + + /// + /// Gets a single file or folder entry + /// + /// The path to get + /// The cancellation token + /// The file or folder entry + Task GetEntryAsync(string path, CancellationToken cancellationToken); +} + diff --git a/Duplicati/Library/Interface/ISourceFileEntry.cs b/Duplicati/Library/Interface/ISourceFileEntry.cs new file mode 100644 index 000000000..0f41fb690 --- /dev/null +++ b/Duplicati/Library/Interface/ISourceFileEntry.cs @@ -0,0 +1,126 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +#nullable enable + +namespace Duplicati.Library.Interface; + +/// +/// Interface for an instance of a file +/// +public interface ISourceFileEntry +{ + /// + /// True if the entry represents a folder, false otherwise + /// + bool IsFolder { get; } + /// + /// True if the entry is a meta entry, false otherwise + /// + bool IsMetaEntry { get; } + /// + /// True if the entry is the root entry, false otherwise + /// + bool IsRootEntry { get; } + /// + /// The creation time of the file or folder + /// + DateTime CreatedUtc { get; } + /// + /// The time the file or folder was last modified + /// + DateTime LastModificationUtc { get; } + /// + /// The name of the file or folder + /// + string Path { get; } + /// + /// The size of the file or folder + /// + long Size { get; } + /// + /// True if the file is a symlink, false otherwise + /// + bool IsSymlink { get; } + /// + /// The target of a symlink, if the file is a symlink + /// + string? SymlinkTarget { get; } + /// + /// The entry attributes + /// + FileAttributes Attributes { get; } + /// + /// The minor metadata of the file, should be less than 1KB + /// + Dictionary MinorMetadata { get; } + /// + /// True if the file is a block device, false otherwise + /// + bool IsBlockDevice { get; } + /// + /// True if the file is a character device, false otherwise + /// + bool IsCharacterDevice { get; } + /// + /// True if the files is an alternate stream, false otherwise + /// + bool IsAlternateStream { get; } + /// + /// The hardlink target id, if the file is a hardlink + /// + string? HardlinkTargetId { get; } + + /// + /// Opens the file for reading + /// + /// A token that can be used to cancel the operation + /// A stream that can be read + Task OpenRead(CancellationToken cancellationToken); + + /// + /// Opens the metadata for reading + /// + /// A token that can be used to cancel the operation + /// A stream that can be read + Task OpenMetadataRead(CancellationToken cancellationToken); + + /// + /// Checks if the specified file exists + /// + /// The file to check for existence + /// A token that can be used to cancel the operation + /// true if the file exists, false otherwise + Task FileExists(string filename, CancellationToken cancellationToken); + + /// + /// Returns the contents of the object, if it is a folder + /// + /// A token that can be used to cancel the operation + /// The contents of the folder + IAsyncEnumerable Enumerate(CancellationToken cancellationToken); +} diff --git a/Duplicati/Library/Interface/ISourceProvider.cs b/Duplicati/Library/Interface/ISourceProvider.cs new file mode 100644 index 000000000..4b5725739 --- /dev/null +++ b/Duplicati/Library/Interface/ISourceProvider.cs @@ -0,0 +1,51 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace Duplicati.Library.Interface; + +/// +/// Implements a data source that can be backed up +/// +public interface ISourceProvider : IDisposable +{ + /// + /// Gets the root entries + /// + /// The cancellation token + /// An enumerable of file entries + IAsyncEnumerable Enumerate(CancellationToken cancellationToken); + + /// + /// Gets a specific entry + /// + /// The path to use + /// True if the path is a folder + /// The cancellation token + /// The file entry + Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken); +} diff --git a/Duplicati/Library/Interface/ISourceProviderModule.cs b/Duplicati/Library/Interface/ISourceProviderModule.cs new file mode 100644 index 000000000..469db1fb0 --- /dev/null +++ b/Duplicati/Library/Interface/ISourceProviderModule.cs @@ -0,0 +1,39 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +namespace Duplicati.Library.Interface; + +/// +/// Implements a data source that can be backed up +/// +public interface ISourceProviderModule : ISourceProvider, IDynamicModule +{ + /// + /// Gets the module key + /// + string Key { get; } + /// + /// Gets the display name of the module + /// + string DisplayName { get; } +} diff --git a/Duplicati/Library/Main/Duplicati.Library.Main.csproj b/Duplicati/Library/Main/Duplicati.Library.Main.csproj index dd7f67065..031c46fd1 100644 --- a/Duplicati/Library/Main/Duplicati.Library.Main.csproj +++ b/Duplicati/Library/Main/Duplicati.Library.Main.csproj @@ -19,6 +19,7 @@ + @@ -47,6 +48,7 @@ + diff --git a/Duplicati/Library/Main/Operation/Backup/Channels.cs b/Duplicati/Library/Main/Operation/Backup/Channels.cs index 6a6576f67..b5948e13e 100644 --- a/Duplicati/Library/Main/Operation/Backup/Channels.cs +++ b/Duplicati/Library/Main/Operation/Backup/Channels.cs @@ -20,6 +20,7 @@ // DEALINGS IN THE SOFTWARE. using CoCoL; +using Duplicati.Library.Interface; namespace Duplicati.Library.Main.Operation.Backup { @@ -51,7 +52,7 @@ namespace Duplicati.Library.Main.Operation.Backup /// /// When enumerating the source folders, all discovered paths are sent by the to the /// - public readonly IChannel SourcePaths = ChannelManager.CreateChannel(); + public readonly IChannel SourcePaths = ChannelManager.CreateChannel(); /// /// All progress events are communicated over this channel, to ensure that parallel progress is reported as a if it was sequential /// diff --git a/Duplicati/Library/Main/Operation/Backup/CountFilesHandler.cs b/Duplicati/Library/Main/Operation/Backup/CountFilesHandler.cs index 83f46b264..ffefa0abc 100644 --- a/Duplicati/Library/Main/Operation/Backup/CountFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/Backup/CountFilesHandler.cs @@ -1,42 +1,40 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System.Collections.Generic; using System.Threading.Tasks; using Duplicati.Library.Utility; -using Duplicati.Library.Snapshots; using CoCoL; -using Duplicati.Library.Common.IO; +using Duplicati.Library.Interface; +using Duplicati.Library.Snapshots; namespace Duplicati.Library.Main.Operation.Backup { internal static class CountFilesHandler { public static async Task Run( - IEnumerable sources, - ISnapshotService snapshot, + ISourceProvider sources, UsnJournalService journalService, BackupResults result, Options options, - IFilter sourcefilter, IFilter filter, HashSet blacklistPaths, Common.ITaskReader taskreader, @@ -47,9 +45,8 @@ namespace Duplicati.Library.Main.Operation.Backup using (Logging.Log.StartIsolatingScope(true)) { Channels channels = new(); - var enumeratorTask = Backup.FileEnumerationProcess.Run( - channels, sources, snapshot, journalService, - options.FileAttributeFilter, sourcefilter, filter, + var enumeratorTask = FileEnumerationProcess.Run( + channels, sources, journalService, options.FileAttributeFilter, filter, options.SymlinkPolicy, options.HardlinkPolicy, options.ExcludeEmptyFolders, options.IgnoreFilenames, blacklistPaths, options.ChangedFilelist, taskreader, null, token); @@ -68,14 +65,15 @@ namespace Duplicati.Library.Main.Operation.Backup { while (await taskreader.ProgressRendevouz() && !token.IsCancellationRequested) { - var path = await self.Input.ReadAsync(); + var entry = await self.Input.ReadAsync(); + if (entry.IsFolder) + continue; count++; try { - if (SystemIO.IO_OS.FileExists(path)) - size += snapshot.GetFileSize(path); + size += entry.Size; } catch { diff --git a/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs b/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs index 7dcfead03..b5b5eeb11 100644 --- a/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs +++ b/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs @@ -1,29 +1,28 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using CoCoL; using Duplicati.Library.Main.Operation.Common; using System.Threading.Tasks; -using System.Threading; namespace Duplicati.Library.Main.Operation.Backup { @@ -37,7 +36,7 @@ namespace Duplicati.Library.Main.Operation.Backup /// private static readonly string FILELOGTAG = Logging.Log.LogTagFromType(typeof(FileBlockProcessor)) + ".FileEntry"; - public static Task Run(Channels channels, Snapshots.ISnapshotService snapshot, Options options, BackupDatabase database, BackupStatsCollector stats, ITaskReader taskreader) + public static Task Run(Channels channels, Options options, BackupDatabase database, BackupStatsCollector stats, ITaskReader taskreader) { return AutomationExtensions.RunTask( new @@ -57,7 +56,7 @@ namespace Duplicati.Library.Main.Operation.Backup try { - var hint = options.GetCompressionHintFromFilename(e.Path); + var hint = options.GetCompressionHintFromFilename(e.Entry.Path); var oldHash = e.OldId < 0 ? null : await database.GetFileHashAsync(e.OldId); StreamProcessResult filestreamdata; @@ -73,15 +72,15 @@ namespace Duplicati.Library.Main.Operation.Backup if (res.Item1) return res.Item2; - Logging.Log.WriteWarningMessage(FILELOGTAG, "UnexpectedMetadataLookup", null, "Metadata was reported as not changed, but still requires being added?\nHash: {0}, Length: {1}, ID: {2}, Path: {3}", e.MetaHashAndSize.FileHash, e.MetaHashAndSize.Blob.Length, res.Item2, e.Path); + Logging.Log.WriteWarningMessage(FILELOGTAG, "UnexpectedMetadataLookup", null, "Metadata was reported as not changed, but still requires being added?\nHash: {0}, Length: {1}, ID: {2}, Path: {3}", e.MetaHashAndSize.FileHash, e.MetaHashAndSize.Blob.Length, res.Item2, e.Entry.Path); e.MetadataChanged = true; } - return (await MetadataPreProcess.AddMetadataToOutputAsync(e.Path, e.MetaHashAndSize, database, self.StreamBlockChannel)).Item2; + return (await MetadataPreProcess.AddMetadataToOutputAsync(e.Entry.Path, e.MetaHashAndSize, database, self.StreamBlockChannel)).Item2; }); - using (var fs = snapshot.OpenRead(e.Path)) - filestreamdata = await StreamBlock.ProcessStream(self.StreamBlockChannel, e.Path, fs, false, hint); + using (var fs = await e.Entry.OpenRead(taskreader.ProgressToken)) + filestreamdata = await StreamBlock.ProcessStream(self.StreamBlockChannel, e.Entry.Path, fs, false, hint); await stats.AddOpenedFile(filestreamdata.Streamlength); @@ -92,36 +91,36 @@ namespace Duplicati.Library.Main.Operation.Backup if (oldHash != filekey) { if (oldHash == null) - Logging.Log.WriteVerboseMessage(FILELOGTAG, "NewFile", "New file {0}", e.Path); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "NewFile", "New file {0}", e.Entry.Path); else - Logging.Log.WriteVerboseMessage(FILELOGTAG, "ChangedFile", "File has changed {0}", e.Path); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "ChangedFile", "File has changed {0}", e.Entry.Path); if (e.OldId < 0) { await stats.AddAddedFile(filesize); if (options.Dryrun) - Logging.Log.WriteVerboseMessage(FILELOGTAG, "WouldAddNewFile", "Would add new file {0}, size {1}", e.Path, Library.Utility.Utility.FormatSizeString(filesize)); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "WouldAddNewFile", "Would add new file {0}, size {1}", e.Entry.Path, Library.Utility.Utility.FormatSizeString(filesize)); } else { await stats.AddModifiedFile(filesize); if (options.Dryrun) - Logging.Log.WriteVerboseMessage(FILELOGTAG, "WouldAddChangedFile", "Would add changed file {0}, size {1}", e.Path, Library.Utility.Utility.FormatSizeString(filesize)); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "WouldAddChangedFile", "Would add changed file {0}, size {1}", e.Entry.Path, Library.Utility.Utility.FormatSizeString(filesize)); } await database.AddFileAsync(e.PathPrefixID, e.Filename, e.LastWrite, filestreamdata.Blocksetid, metadataid); } else if (e.MetadataChanged) { - Logging.Log.WriteVerboseMessage(FILELOGTAG, "FileMetadataChanged", "File has only metadata changes {0}", e.Path); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "FileMetadataChanged", "File has only metadata changes {0}", e.Entry.Path); await database.AddFileAsync(e.PathPrefixID, e.Filename, e.LastWrite, filestreamdata.Blocksetid, metadataid); } else /*if (e.OldId >= 0)*/ { // When we write the file to output, update the last modified time - Logging.Log.WriteVerboseMessage(FILELOGTAG, "NoFileChanges", "File has not changed {0}", e.Path); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "NoFileChanges", "File has not changed {0}", e.Entry.Path); try { @@ -129,7 +128,7 @@ namespace Duplicati.Library.Main.Operation.Backup } catch (Exception ex) { - Logging.Log.WriteWarningMessage(FILELOGTAG, "FailedToAddFile", ex, "Failed while attempting to add unmodified file to database: {0}", e.Path); + Logging.Log.WriteWarningMessage(FILELOGTAG, "FailedToAddFile", ex, "Failed while attempting to add unmodified file to database: {0}", e.Entry.Path); } } } @@ -138,7 +137,7 @@ namespace Duplicati.Library.Main.Operation.Backup if (ex.IsRetiredException()) return; else - Logging.Log.WriteWarningMessage(FILELOGTAG, "PathProcessingFailed", ex, "Failed to process path: {0}", e.Path); + Logging.Log.WriteWarningMessage(FILELOGTAG, "PathProcessingFailed", ex, "Failed to process path: {0}", e.Entry.Path); } } }); diff --git a/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs b/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs index 7b7478a82..7283da48e 100644 --- a/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs @@ -27,9 +27,10 @@ using System.Collections.Generic; using System.Linq; using System.Threading; using Duplicati.Library.Main.Operation.Common; -using Duplicati.Library.Snapshots; -using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; +using System.Runtime.CompilerServices; +using Duplicati.Library.Snapshots; +using Duplicati.Library.SourceProvider; namespace Duplicati.Library.Main.Operation.Backup { @@ -47,18 +48,16 @@ namespace Duplicati.Library.Main.Operation.Backup public static Task Run( Channels channels, - IEnumerable sources, - ISnapshotService snapshot, + ISourceProvider sourceProvider, UsnJournalService journalService, - FileAttributes fileAttributes, - Library.Utility.IFilter sourcefilter, + FileAttributes fileAttributeFilter, Library.Utility.IFilter emitfilter, Options.SymlinkStrategy symlinkPolicy, Options.HardlinkStrategy hardlinkPolicy, bool excludeemptyfolders, string[] ignorenames, HashSet blacklistPaths, - string[] changedfilelist, + IEnumerable changedfilelist, ITaskReader taskreader, Action onStopRequested, CancellationToken token) @@ -73,9 +72,19 @@ namespace Duplicati.Library.Main.Operation.Backup { if (!token.IsCancellationRequested) { + // The hardlink map tracks the hardlink targets we have seen + // and avoid multiple processing of the same contents var hardlinkmap = new Dictionary(); - var mixinqueue = new Queue(); - Duplicati.Library.Utility.IFilter enumeratefilter = emitfilter; + + // The mixin queue is used to store symlinks that should be processed + // The symlinks are emitted during the enumeration process when they are found + var mixinqueue = new Queue(); + + // The enumeration filter is used to determine what paths to + // recurse into. If the emit filter only has includes, + // the enumeration filter will also include all folders, + // as nothing will match otherwise + var enumeratefilter = emitfilter; Library.Utility.FilterExpression.AnalyzeFilters(emitfilter, out var includes, out var excludes); if (includes && !excludes) @@ -86,58 +95,59 @@ namespace Duplicati.Library.Main.Operation.Backup ignorenames = null; // If we have a specific list, use that instead of enumerating the filesystem - IEnumerable worklist; - if (changedfilelist != null && changedfilelist.Length > 0) + async IAsyncEnumerable ExpandSources(IEnumerable list) { - worklist = changedfilelist.Where(x => - { - var fa = FileAttributes.Normal; - try - { - fa = snapshot.GetAttributes(x); - } - catch - { - } + foreach (var s in list) + yield return await sourceProvider.GetEntry(s, false, token).ConfigureAwait(false); + } - if (token.IsCancellationRequested) - { - return false; - } + IAsyncEnumerable worklist; + if (changedfilelist != null && changedfilelist.Any()) + { + worklist = ExpandSources(changedfilelist).WhereAwait(x => + SourceFileEntryFilter(x, blacklistPaths, hardlinkPolicy, symlinkPolicy, hardlinkmap, fileAttributeFilter, enumeratefilter, ignorenames, mixinqueue, token)); + } + else if (journalService != null) + { + if (!OperatingSystem.IsWindows()) + throw new NotSupportedException("USN is only supported on Windows"); - return AttributeFilter(x, fa, snapshot, sourcefilter, blacklistPaths, hardlinkPolicy, symlinkPolicy, hardlinkmap, fileAttributes, enumeratefilter, ignorenames, mixinqueue); - }); + var fileProviders = (sourceProvider is Combiner c ? c.Providers.AsEnumerable() : [sourceProvider]) + .OfType() + .ToList(); + + if (fileProviders.Count <= 0) + throw new InvalidOperationException("No file providers found, but USN was enabled?"); + if (fileProviders.Count > 1) + throw new InvalidOperationException("Multiple file providers found, but USN only supports one"); + + worklist = RecursiveFilterHelper.GetModifiedSources( + ExpandSources(journalService.VolumeDataList.SelectMany(x => x.Folders.Concat(x.Files))), + fileProviders.First().SnapshotService, + x => SourceFileEntryFilter(x, blacklistPaths, hardlinkPolicy, symlinkPolicy, hardlinkmap, fileAttributeFilter, enumeratefilter, ignorenames, mixinqueue, token), + token + ); } else { - Library.Utility.Utility.EnumerationFilterDelegate attributeFilter = (root, path, attr) => - AttributeFilter(path, attr, snapshot, sourcefilter, blacklistPaths, hardlinkPolicy, symlinkPolicy, hardlinkmap, fileAttributes, enumeratefilter, ignorenames, mixinqueue); - - if (journalService != null) - { - if (!OperatingSystem.IsWindows()) - throw new UserInformationException("USN journal is only supported on Windows", "USNJournalNotSupported"); - // filter sources using USN journal, to obtain a sub-set of files / folders that may have been modified - sources = journalService.GetModifiedSources(attributeFilter); - } - - worklist = snapshot.EnumerateFilesAndFolders(sources, attributeFilter, (rootpath, errorpath, ex) => - { - Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "FileAccessError", ex, "Error reported while accessing file: {0}", errorpath); - }).Distinct(Library.Utility.Utility.IsFSCaseSensitive ? StringComparer.Ordinal : StringComparer.OrdinalIgnoreCase); + worklist = RecurseEntries(sourceProvider.Enumerate(token), + x => SourceFileEntryFilter(x, blacklistPaths, hardlinkPolicy, symlinkPolicy, hardlinkmap, fileAttributeFilter, enumeratefilter, ignorenames, mixinqueue, token), + token + ); } if (token.IsCancellationRequested) - { return; - } - var source = ExpandWorkList(worklist, mixinqueue, emitfilter, enumeratefilter); + var source = ExpandWorkList(worklist, mixinqueue, emitfilter, enumeratefilter, token); + // TODO: There was a call to DistinctBy here, but this would cause all paths to be stored in memory + //.DistinctBy(x => x.Path, Library.Utility.Utility.IsFSCaseSensitive ? StringComparer.Ordinal : StringComparer.OrdinalIgnoreCase); + if (excludeemptyfolders) - source = ExcludeEmptyFolders(source); + source = ExcludeEmptyFolders(source, token); // Process each path, and dequeue the mixins with symlinks as we go - foreach (var s in source) + await foreach (var s in source.WithCancellation(token).ConfigureAwait(false)) { #if DEBUG // For testing purposes, we need exact control @@ -146,7 +156,7 @@ namespace Duplicati.Library.Main.Operation.Backup // if the process is the real file enumeration process // because the counter processe does not have a callback if (onStopRequested != null) - taskreader.TestMethodCallback?.Invoke(s); + taskreader.TestMethodCallback?.Invoke(s.Path); #endif // Stop if requested if (token.IsCancellationRequested || !await taskreader.ProgressRendevouz().ConfigureAwait(false)) @@ -167,9 +177,9 @@ namespace Duplicati.Library.Main.Operation.Backup private class DirectoryStackEntry { /// - /// The path for the folder + /// The item being tracked /// - public string Path; + public ISourceFileEntry Item; /// /// A flag indicating if any items are found in this folder /// @@ -182,17 +192,17 @@ namespace Duplicati.Library.Main.Operation.Backup /// /// The list without empty folders. /// The list with potential empty folders. - private static IEnumerable ExcludeEmptyFolders(IEnumerable source) + private static async IAsyncEnumerable ExcludeEmptyFolders(IAsyncEnumerable source, [EnumeratorCancellation] CancellationToken cancellationToken) { var pathstack = new Stack(); - foreach (var s in source) + await foreach (var s in source.WithCancellation(cancellationToken).ConfigureAwait(false)) { // Keep track of directories - var isDirectory = s[s.Length - 1] == System.IO.Path.DirectorySeparatorChar; + var isDirectory = s.Path[s.Path.Length - 1] == System.IO.Path.DirectorySeparatorChar; if (isDirectory) { - while (pathstack.Count > 0 && !s.StartsWith(pathstack.Peek().Path, Library.Utility.Utility.ClientFilenameStringComparison)) + while (pathstack.Count > 0 && !s.Path.StartsWith(pathstack.Peek().Item.Path, Library.Utility.Utility.ClientFilenameStringComparison)) { var e = pathstack.Pop(); if (e.AnyEntries || pathstack.Count == 0) @@ -201,15 +211,15 @@ namespace Duplicati.Library.Main.Operation.Backup if (pathstack.Count > 0) pathstack.Peek().AnyEntries = true; - yield return e.Path; + yield return e.Item; } else - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingEmptyFolder", "Excluding empty folder {0}", e.Path); + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingEmptyFolder", "Excluding empty folder {0}", e.Item); } - if (pathstack.Count == 0 || s.StartsWith(pathstack.Peek().Path, Library.Utility.Utility.ClientFilenameStringComparison)) + if (pathstack.Count == 0 || s.Path.StartsWith(pathstack.Peek().Item.Path, Library.Utility.Utility.ClientFilenameStringComparison)) { - pathstack.Push(new DirectoryStackEntry() { Path = s }); + pathstack.Push(new DirectoryStackEntry() { Item = s }); continue; } } @@ -231,7 +241,46 @@ namespace Duplicati.Library.Main.Operation.Backup if (pathstack.Count > 0) pathstack.Peek().AnyEntries = true; - yield return e.Path; + yield return e.Item; + } + } + } + + /// + /// Performs recursive traversal of the sources + /// + /// The entries to recurse + /// The filter to apply + /// + private static async IAsyncEnumerable RecurseEntries(IAsyncEnumerable entries, Func> filter, [EnumeratorCancellation] CancellationToken cancellationToken) + { + var work = new Stack(); + + await foreach (var e in entries.WithCancellation(cancellationToken).ConfigureAwait(false)) + if (await filter(e).ConfigureAwait(false)) + work.Push(e); + + while (work.Count > 0) + { + var e = work.Pop(); + + // Process meta entry contents, but don't emit them for processing + if (!e.IsMetaEntry) + yield return e; + + if (e.IsFolder) + { + try + { + // We only filter new items, as we assume the input is already filtered + await foreach (var r in e.Enumerate(cancellationToken).ConfigureAwait(false)) + if (await filter(r).ConfigureAwait(false)) + work.Push(r); + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingErrorEnumerate", ex, "Failed to enumerate path: {0}", e.Path); + } } } } @@ -244,16 +293,18 @@ namespace Duplicati.Library.Main.Operation.Backup /// The mix in queue. /// The emitfilter. /// The enumeratefilter. - private static IEnumerable ExpandWorkList(IEnumerable worklist, Queue mixinqueue, Library.Utility.IFilter emitfilter, Library.Utility.IFilter enumeratefilter) + private static async IAsyncEnumerable ExpandWorkList(IAsyncEnumerable worklist, Queue mixinqueue, Library.Utility.IFilter emitfilter, Library.Utility.IFilter enumeratefilter, [EnumeratorCancellation] CancellationToken cancellationToken) { // Process each path, and dequeue the mixins with symlinks as we go - foreach (var s in worklist) + await foreach (var s in worklist.WithCancellation(cancellationToken).ConfigureAwait(false)) { while (mixinqueue.Count > 0) yield return mixinqueue.Dequeue(); - Library.Utility.IFilter m; - if (emitfilter != enumeratefilter && !Library.Utility.FilterExpression.Matches(emitfilter, s, out m)) + // If there are only includes in the filter, check if the item is in the original filter + // Since the enumerate filter also includes all folders, we need to ensure we do not emit + // any entries that are filtered explicitly by the user + if (emitfilter != enumeratefilter && !Library.Utility.FilterExpression.Matches(emitfilter, s.Path, out var _)) continue; yield return s; @@ -265,65 +316,92 @@ namespace Duplicati.Library.Main.Operation.Backup } /// - /// Plugin filter for enumerating a list of files. + /// Performs a pre-filter on the source entry to see if it should be included in the backup /// - /// True if the path should be returned, false otherwise. - /// The current path. - /// The file or folder attributes. - /// The snapshot service. - /// The source filter. + /// The entry to evaluate. /// The blacklist paths. - /// The hardlink policy. - /// The symlink policy. - /// The hardlink map. - /// The file attributes to exclude. - /// The enumerate filter. - /// The ignore names. - /// The mixin queue. - private static bool AttributeFilter(string path, FileAttributes attributes, Snapshots.ISnapshotService snapshot, Library.Utility.IFilter sourcefilter, HashSet blacklistPaths, Options.HardlinkStrategy hardlinkPolicy, Options.SymlinkStrategy symlinkPolicy, Dictionary hardlinkmap, FileAttributes fileAttributes, Duplicati.Library.Utility.IFilter enumeratefilter, string[] ignorenames, Queue mixinqueue) + /// True if the path should be returned, false otherwise. + private static bool PreFilterSourceEntry(ISourceFileEntry entry, HashSet blacklistPaths) { + // Don't filter meta stuff + if (entry.IsMetaEntry) + return true; + // Exclude any blacklisted paths - if (blacklistPaths.Contains(path)) + if (blacklistPaths.Contains(entry.Path)) { - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingBlacklistedPath", "Excluding blacklisted path: {0}", path); + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingBlacklistedPath", "Excluding blacklisted path: {0}", entry.Path); return false; } // Exclude block devices try { - if (snapshot.IsBlockDevice(path)) + if (entry.IsBlockDevice) { - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingBlockDevice", "Excluding block device: {0}", path); + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingBlockDevice", "Excluding block device: {0}", entry.Path); return false; } } catch (Exception ex) { - Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingErrorBlockDevice", ex, "Failed to process path: {0}", path); + Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingErrorBlockDevice", ex, "Failed to process path: {0}", entry.Path); return false; } - // Check if we explicitly include this entry - Duplicati.Library.Utility.IFilter sourcematch; - bool sourcematches; - if (sourcefilter.Matches(path, out sourcematches, out sourcematch) && sourcematches) + // Exclude character devices + try { - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "IncludingSourcePath", "Including source path: {0}", path); - return true; + if (entry.IsCharacterDevice) + { + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingCharacterDevice", "Excluding character device: {0}", entry.Path); + return false; + } } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingErrorCharacterDevice", ex, "Failed to process path: {0}", entry.Path); + return false; + } + + return true; + } + + /// + /// Evaluates a single entry for inclusion in the backup + /// + /// The current entry. + /// The snapshot service. + /// The blacklist paths. + /// The hardlink policy. + /// The symlink policy. + /// The hardlink map. + /// The file attributes to exclude. + /// The enumerate filter. + /// The ignore names. + /// The mixin queue. + /// True if the path should be returned, false otherwise. + private static async ValueTask SourceFileEntryFilter(ISourceFileEntry entry, HashSet blacklistPaths, Options.HardlinkStrategy hardlinkPolicy, Options.SymlinkStrategy symlinkPolicy, Dictionary hardlinkmap, FileAttributes fileAttributeFilter, Duplicati.Library.Utility.IFilter enumeratefilter, string[] ignorenames, Queue mixinqueue, CancellationToken cancellationToken) + { + // Do the course pre-filtering first + if (!PreFilterSourceEntry(entry, blacklistPaths)) + return false; + + // Never exclude the root entries + if (entry.IsRootEntry) + return true; // If we have a hardlink strategy, obey it if (hardlinkPolicy != Options.HardlinkStrategy.All) { try { - var id = snapshot.HardlinkTargetID(path); + var id = entry.HardlinkTargetId; if (id != null) { if (hardlinkPolicy == Options.HardlinkStrategy.None) { - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingHardlinkByPolicy", "Excluding hardlink: {0} ({1})", path, id); + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingHardlinkByPolicy", "Excluding hardlink: {0} ({1})", entry.Path, id); return false; } else if (hardlinkPolicy == Options.HardlinkStrategy.First) @@ -331,99 +409,116 @@ namespace Duplicati.Library.Main.Operation.Backup string prevPath; if (hardlinkmap.TryGetValue(id, out prevPath)) { - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingDuplicateHardlink", "Excluding hardlink ({1}) for: {0}, previous hardlink: {2}", path, id, prevPath); + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingDuplicateHardlink", "Excluding hardlink ({1}) for: {0}, previous hardlink: {2}", entry.Path, id, prevPath); return false; } else { - hardlinkmap.Add(id, path); + hardlinkmap.Add(id, entry.Path); } } } } catch (Exception ex) { - Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingErrorHardLink", ex, "Failed to process path: {0}", path); + Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingErrorHardLink", ex, "Failed to process path: {0}", entry.Path); return false; } } - if (ignorenames != null && (attributes & FileAttributes.Directory) != 0) + // Check if there is an ignore marker file + if (ignorenames != null && entry.IsFolder) { try { foreach (var n in ignorenames) { - var ignorepath = SystemIO.IO_OS.PathCombine(path, n); - if (snapshot.FileExists(ignorepath)) + if (await entry.FileExists(n, cancellationToken).ConfigureAwait(false)) { - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingPathDueToIgnoreFile", "Excluding path because ignore file was found: {0}", ignorepath); + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingPathDueToIgnoreFile", "Excluding path because ignore file {0} was found in: {1}", n, entry.Path); return false; } } } catch (Exception ex) { - Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingErrorIgnoreFile", ex, "Failed to process path: {0}", path); + Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingErrorIgnoreFile", ex, "Failed to process path: {0}", entry.Path); } } - // If we exclude files based on attributes, filter that - if ((fileAttributes & attributes) != 0) + // Setup some basic processing attributes + var attributes = entry.IsFolder + ? FileAttributes.Directory + : FileAttributes.Normal; + + try { - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingPathFromAttributes", "Excluding path due to attribute filter: {0}", path); + attributes = entry.Attributes; + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingErrorAttributes", ex, "Failed to process path, using default attributes: {0}", entry.Path); + } + + // If we exclude files based on attributes, filter that + if ((fileAttributeFilter & attributes) != 0) + { + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingPathFromAttributes", "Excluding path due to attribute filter: {0}", entry.Path); return false; } // Then check if the filename is not explicitly excluded by a filter - Library.Utility.IFilter match; var filtermatch = false; - if (!Library.Utility.FilterExpression.Matches(enumeratefilter, path, out match)) + if (!Library.Utility.FilterExpression.Matches(enumeratefilter, entry.Path, out var match)) { - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingPathFromFilter", "Excluding path due to filter: {0} => {1}", path, match == null ? "null" : match.ToString()); + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingPathFromFilter", "Excluding path due to filter: {0} => {1}", entry.Path, match == null ? "null" : match.ToString()); return false; } else if (match != null) { filtermatch = true; - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "IncludingPathFromFilter", "Including path due to filter: {0} => {1}", path, match.ToString()); + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "IncludingPathFromFilter", "Including path due to filter: {0} => {1}", entry.Path, match.ToString()); } // If the file is a symlink, apply special handling - var isSymlink = snapshot.IsSymlink(path, attributes); string symlinkTarget = null; - if (isSymlink) - try { symlinkTarget = snapshot.GetSymlinkTarget(path); } - catch (Exception ex) { Logging.Log.WriteExplicitMessage(FILTER_LOGTAG, "SymlinkTargetReadError", ex, "Failed to read symlink target for path: {0}", path); } + try + { + symlinkTarget = entry.SymlinkTarget; + } + catch (Exception ex) + { + Logging.Log.WriteExplicitMessage(FILTER_LOGTAG, "SymlinkTargetReadError", ex, "Failed to read symlink target for path: {0}", entry.Path); + } - if (isSymlink) + if (symlinkTarget != null) { if (!string.IsNullOrWhiteSpace(symlinkTarget)) { if (symlinkPolicy == Options.SymlinkStrategy.Ignore) { - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludeSymlink", "Excluding symlink: {0}", path); + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludeSymlink", "Excluding symlink: {0}", entry.Path); return false; } if (symlinkPolicy == Options.SymlinkStrategy.Store) { - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "StoreSymlink", "Storing symlink: {0}", path); + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "StoreSymlink", "Storing symlink: {0}", entry.Path); // We return false because we do not want to recurse into the path, // but we add the symlink to the mixin so we process the symlink itself - mixinqueue.Enqueue(path); + mixinqueue.Enqueue(entry); return false; } } else { - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "FollowingEmptySymlink", "Treating empty symlink as regular path {0}", path); + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "FollowingEmptySymlink", "Treating empty symlink as regular path {0}", entry.Path); } } if (!filtermatch) - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "IncludingPath", "Including path as no filters matched: {0}", path); + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "IncludingPath", "Including path as no filters matched: {0}", entry.Path); // All the way through, yes! return true; diff --git a/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs b/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs index 4276fdefc..f65398e91 100644 --- a/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -40,7 +40,7 @@ namespace Duplicati.Library.Main.Operation.Backup /// private static readonly string FILELOGTAG = Logging.Log.LogTagFromType(typeof(FilePreFilterProcess)) + ".FileEntry"; - public static Task Run(Channels channels, Snapshots.ISnapshotService snapshot, Options options, BackupStatsCollector stats, BackupDatabase database) + public static Task Run(Channels channels, Options options, BackupStatsCollector stats, BackupDatabase database) { return AutomationExtensions.RunTask( new @@ -73,11 +73,11 @@ namespace Duplicati.Library.Main.Operation.Backup long filestatsize = -1; try { - filestatsize = snapshot.GetFileSize(e.Path); + filestatsize = e.Entry.Size; } - catch(Exception ex) + catch (Exception ex) { - Logging.Log.WriteExplicitMessage(FILELOGTAG, "FailedToReadSize", ex, "Failed to read size of file: {0}", e.Path); + Logging.Log.WriteExplicitMessage(FILELOGTAG, "FailedToReadSize", ex, "Failed to read size of file: {0}", e.Entry.Path); } await stats.AddExaminedFile(filestatsize); @@ -86,8 +86,8 @@ namespace Duplicati.Library.Main.Operation.Backup var tooLargeFile = SKIPFILESLARGERTHAN != 0 && filestatsize >= 0 && filestatsize > SKIPFILESLARGERTHAN; if (tooLargeFile) { - Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckTooLarge", "Skipped checking file, because the size exceeds limit {0}", e.Path); - await self.ProgressChannel.WriteAsync(new ProgressEvent() { Filepath = e.Path, Length = filestatsize, Type = EventType.FileSkipped }); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckTooLarge", "Skipped checking file, because the size exceeds limit {0}", e.Entry.Path); + await self.ProgressChannel.WriteAsync(new ProgressEvent() { Filepath = e.Entry.Path, Length = filestatsize, Type = EventType.FileSkipped }); continue; } @@ -102,7 +102,7 @@ namespace Duplicati.Library.Main.Operation.Backup // and we only check for timestamp changes if (CHECKFILETIMEONLY && !timestampChanged && !isNewFile) { - Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoTimestampChange", "Skipped checking file, because timestamp was not updated {0}", e.Path); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoTimestampChange", "Skipped checking file, because timestamp was not updated {0}", e.Entry.Path); try { await database.AddUnmodifiedAsync(e.OldId, e.LastWrite); @@ -111,9 +111,9 @@ namespace Duplicati.Library.Main.Operation.Backup { if (ex.IsRetiredException()) throw; - Logging.Log.WriteWarningMessage(FILELOGTAG, "FailedToAddFile", ex, "Failed while attempting to add unmodified file to database: {0}", e.Path); + Logging.Log.WriteWarningMessage(FILELOGTAG, "FailedToAddFile", ex, "Failed while attempting to add unmodified file to database: {0}", e.Entry.Path); } - await self.ProgressChannel.WriteAsync(new ProgressEvent() { Filepath = e.Path, Length = filestatsize, Type = EventType.FileSkipped }); + await self.ProgressChannel.WriteAsync(new ProgressEvent() { Filepath = e.Entry.Path, Length = filestatsize, Type = EventType.FileSkipped }); continue; } @@ -130,28 +130,28 @@ namespace Duplicati.Library.Main.Operation.Backup } // Compute current metadata - e.MetaHashAndSize = SKIPMETADATA ? EMPTY_METADATA : Utility.WrapMetadata(MetadataGenerator.GenerateMetadata(e.Path, e.Attributes, options, snapshot), options); + e.MetaHashAndSize = SKIPMETADATA ? EMPTY_METADATA : Utility.WrapMetadata(MetadataGenerator.GenerateMetadata(e.Entry, e.Attributes, options), options); e.MetadataChanged = !SKIPMETADATA && (e.MetaHashAndSize.Blob.Length != e.OldMetaSize || e.MetaHashAndSize.FileHash != e.OldMetaHash); // Check if the file is new, or something indicates a change var filesizeChanged = filestatsize < 0 || e.LastFileSize < 0 || filestatsize != e.LastFileSize; if (isNewFile || timestampChanged || filesizeChanged || e.MetadataChanged) { - Logging.Log.WriteVerboseMessage(FILELOGTAG, "CheckFileForChanges", "Checking file for changes {0}, new: {1}, timestamp changed: {2}, size changed: {3}, metadatachanged: {4}, {5} vs {6}", e.Path, isNewFile, timestampChanged, filesizeChanged, e.MetadataChanged, e.LastWrite, e.OldModified); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "CheckFileForChanges", "Checking file for changes {0}, new: {1}, timestamp changed: {2}, size changed: {3}, metadatachanged: {4}, {5} vs {6}", e.Entry.Path, isNewFile, timestampChanged, filesizeChanged, e.MetadataChanged, e.LastWrite, e.OldModified); await self.Output.WriteAsync(e); } else { - Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoMetadataChange", "Skipped checking file, because no metadata was updated {0}", e.Path); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoMetadataChange", "Skipped checking file, because no metadata was updated {0}", e.Entry.Path); try { await database.AddUnmodifiedAsync(e.OldId, e.LastWrite); } catch (Exception ex) { - Logging.Log.WriteWarningMessage(FILELOGTAG, "FailedToAddFile", ex, "Failed while attempting to add unmodified file to database: {0}", e.Path); + Logging.Log.WriteWarningMessage(FILELOGTAG, "FailedToAddFile", ex, "Failed while attempting to add unmodified file to database: {0}", e.Entry.Path); } - await self.ProgressChannel.WriteAsync(new ProgressEvent() { Filepath = e.Path, Length = filestatsize, Type = EventType.FileSkipped }); + await self.ProgressChannel.WriteAsync(new ProgressEvent() { Filepath = e.Entry.Path, Length = filestatsize, Type = EventType.FileSkipped }); } } }); diff --git a/Duplicati/Library/Main/Operation/Backup/MetadataGenerator.cs b/Duplicati/Library/Main/Operation/Backup/MetadataGenerator.cs index adb32616c..02c6f75c2 100644 --- a/Duplicati/Library/Main/Operation/Backup/MetadataGenerator.cs +++ b/Duplicati/Library/Main/Operation/Backup/MetadataGenerator.cs @@ -1,30 +1,27 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; -using System.Threading.Tasks; using System.Collections.Generic; -using CoCoL; -using Duplicati.Library.Main.Operation.Common; -using Duplicati.Library.Snapshots; +using Duplicati.Library.Interface; namespace Duplicati.Library.Main.Operation.Backup { @@ -35,7 +32,7 @@ namespace Duplicati.Library.Main.Operation.Backup { private static readonly string METALOGTAG = Logging.Log.LogTagFromType(typeof(MetadataGenerator)) + ".Metadata"; - public static Dictionary GenerateMetadata(string path, System.IO.FileAttributes attributes, Options options, Snapshots.ISnapshotService snapshot) + public static Dictionary GenerateMetadata(ISourceFileEntry entry, System.IO.FileAttributes attributes, Options options) { try { @@ -43,7 +40,7 @@ namespace Duplicati.Library.Main.Operation.Backup if (!options.SkipMetadata) { - metadata = snapshot.GetMetadata(path, snapshot.IsSymlink(path, attributes), options.SymlinkPolicy == Options.SymlinkStrategy.Follow); + metadata = entry.MinorMetadata; if (metadata == null) metadata = new Dictionary(); @@ -54,11 +51,11 @@ namespace Duplicati.Library.Main.Operation.Backup { try { - metadata["CoreLastWritetime"] = snapshot.GetLastWriteTimeUtc(path).Ticks.ToString(); + metadata["CoreLastWritetime"] = entry.LastModificationUtc.Ticks.ToString(); } catch (Exception ex) { - Logging.Log.WriteWarningMessage(METALOGTAG, "TimestampReadFailed", ex, "Failed to read timestamp on \"{0}\"", path); + Logging.Log.WriteWarningMessage(METALOGTAG, "TimestampReadFailed", ex, "Failed to read timestamp on \"{0}\"", entry.Path); } } @@ -66,11 +63,11 @@ namespace Duplicati.Library.Main.Operation.Backup { try { - metadata["CoreCreatetime"] = snapshot.GetCreationTimeUtc(path).Ticks.ToString(); + metadata["CoreCreatetime"] = entry.CreatedUtc.Ticks.ToString(); } catch (Exception ex) { - Logging.Log.WriteWarningMessage(METALOGTAG, "TimestampReadFailed", ex, "Failed to read timestamp on \"{0}\"", path); + Logging.Log.WriteWarningMessage(METALOGTAG, "TimestampReadFailed", ex, "Failed to read timestamp on \"{0}\"", entry.Path); } } } @@ -81,9 +78,9 @@ namespace Duplicati.Library.Main.Operation.Backup return metadata; } - catch(Exception ex) + catch (Exception ex) { - Logging.Log.WriteWarningMessage(METALOGTAG, "MetadataProcessFailed", ex, "Failed to process metadata for \"{0}\", storing empty metadata", path); + Logging.Log.WriteWarningMessage(METALOGTAG, "MetadataProcessFailed", ex, "Failed to process metadata for \"{0}\", storing empty metadata", entry.Path); return new Dictionary(); } } diff --git a/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs b/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs index b8554d45f..1642d1b8a 100644 --- a/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -24,11 +24,8 @@ using CoCoL; using System.Threading.Tasks; using System.IO; using System.Collections.Generic; -using System.Security.Principal; -using System.Threading; using Duplicati.Library.Interface; using Duplicati.Library.Main.Operation.Common; -using Duplicati.Library.Snapshots; namespace Duplicati.Library.Main.Operation.Backup { @@ -43,7 +40,7 @@ namespace Duplicati.Library.Main.Operation.Backup public class FileEntry { // From input - public string Path; + public ISourceFileEntry Entry; // Split public long PathPrefixID; @@ -65,7 +62,7 @@ namespace Duplicati.Library.Main.Operation.Backup public bool MetadataChanged; } - public static Task Run(Channels channels, Snapshots.ISnapshotService snapshot, Options options, BackupDatabase database, long lastfilesetid, ITaskReader taskReader) + public static Task Run(Channels channels, Options options, BackupDatabase database, long lastfilesetid, ITaskReader taskReader) { return AutomationExtensions.RunTask(new { @@ -84,37 +81,40 @@ namespace Duplicati.Library.Main.Operation.Backup while (true) { - var path = await self.Input.ReadAsync(); + var entry = await self.Input.ReadAsync(); // We ignore the stop signal, but not the pause and terminate await taskReader.ProgressRendevouz().ConfigureAwait(false); var lastwrite = new DateTime(0, DateTimeKind.Utc); - var attributes = default(FileAttributes); + var attributes = entry.IsFolder + ? FileAttributes.Directory + : FileAttributes.Normal; + try { - lastwrite = snapshot.GetLastWriteTimeUtc(path); + lastwrite = entry.LastModificationUtc; } catch (Exception ex) { - Logging.Log.WriteWarningMessage(FILELOGTAG, "TimestampReadFailed", ex, "Failed to read timestamp on \"{0}\"", path); + Logging.Log.WriteWarningMessage(FILELOGTAG, "TimestampReadFailed", ex, "Failed to read timestamp on \"{0}\"", entry.Path); } try { - attributes = snapshot.GetAttributes(path); + attributes = entry.Attributes; } catch (Exception ex) { - Logging.Log.WriteVerboseMessage(FILELOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", path, ex.Message); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", entry.Path, ex.Message); } // If we only have metadata, stop here - if (await ProcessMetadata(path, attributes, lastwrite, options, snapshot, emptymetadata, database, self.StreamBlockChannel).ConfigureAwait(false)) + if (await ProcessMetadata(entry, attributes, lastwrite, options, emptymetadata, database, self.StreamBlockChannel).ConfigureAwait(false)) { try { - var split = Database.LocalDatabase.SplitIntoPrefixAndName(path); + var split = Database.LocalDatabase.SplitIntoPrefixAndName(entry.Path); long prefixid; if (string.Equals(prevprefix.Key, split.Key, StringComparison.Ordinal)) @@ -131,7 +131,7 @@ namespace Duplicati.Library.Main.Operation.Backup await self.Output.WriteAsync(new FileEntry { OldId = tmp.Item1, - Path = path, + Entry = entry, PathPrefixID = prefixid, Filename = split.Value, Attributes = attributes, @@ -148,7 +148,7 @@ namespace Duplicati.Library.Main.Operation.Backup await self.Output.WriteAsync(new FileEntry { OldId = res == null ? -1 : res.id, - Path = path, + Entry = entry, PathPrefixID = prefixid, Filename = split.Value, Attributes = attributes, @@ -166,7 +166,7 @@ namespace Duplicati.Library.Main.Operation.Backup continue; Logging.Log.WriteWarningMessage(FILELOGTAG, "ProcessingMetadataFailed", ex, - "Failed to process entry, path: {0}", path); + "Failed to process entry, path: {0}", entry.Path); } } } @@ -179,9 +179,9 @@ namespace Duplicati.Library.Main.Operation.Backup /// Processes the metadata for the given path. /// /// True if the path should be submitted to more analysis, false if there is nothing else to do - private static async Task ProcessMetadata(string path, FileAttributes attributes, DateTime lastwrite, Options options, Snapshots.ISnapshotService snapshot, IMetahash emptymetadata, BackupDatabase database, IWriteChannel streamblockchannel) + private static async Task ProcessMetadata(ISourceFileEntry entry, FileAttributes attributes, DateTime lastwrite, Options options, IMetahash emptymetadata, BackupDatabase database, IWriteChannel streamblockchannel) { - if (snapshot.IsSymlink(path, attributes)) + if (entry.IsSymlink) { // Not all reparse points are symlinks. // For example, on Windows 10 Fall Creator's Update, the OneDrive folder (and all subfolders) @@ -190,57 +190,58 @@ namespace Duplicati.Library.Main.Operation.Backup string symlinkTarget = null; try { - symlinkTarget = snapshot.GetSymlinkTarget(path); + symlinkTarget = entry.SymlinkTarget; } catch (Exception ex) { - Logging.Log.WriteExplicitMessage(FILELOGTAG, "SymlinkTargetReadFailure", ex, "Failed to read symlink target for path: {0}", path); + Logging.Log.WriteExplicitMessage(FILELOGTAG, "SymlinkTargetReadFailure", ex, "Failed to read symlink target for path: {0}", entry.Path); } if (!string.IsNullOrWhiteSpace(symlinkTarget)) { if (options.SymlinkPolicy == Options.SymlinkStrategy.Ignore) { - Logging.Log.WriteVerboseMessage(FILELOGTAG, "IgnoreSymlink", "Ignoring symlink {0}", path); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "IgnoreSymlink", "Ignoring symlink {0}", entry.Path); return false; } if (options.SymlinkPolicy == Options.SymlinkStrategy.Store) { - var metadata = MetadataGenerator.GenerateMetadata(path, attributes, options, snapshot); + var metadata = MetadataGenerator.GenerateMetadata(entry, attributes, options); if (!metadata.ContainsKey("CoreSymlinkTarget")) metadata["CoreSymlinkTarget"] = symlinkTarget; var metahash = Utility.WrapMetadata(metadata, options); - await AddSymlinkToOutputAsync(path, DateTime.UtcNow, metahash, database, streamblockchannel).ConfigureAwait(false); + await AddSymlinkToOutputAsync(entry.Path, DateTime.UtcNow, metahash, database, streamblockchannel).ConfigureAwait(false); - Logging.Log.WriteVerboseMessage(FILELOGTAG, "StoreSymlink", "Stored symlink {0}", path); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "StoreSymlink", "Stored symlink {0}", entry.Path); // Don't process further return false; } } else { - Logging.Log.WriteVerboseMessage(FILELOGTAG, "FollowingEmptySymlink", "Treating empty symlink as regular path {0}", path); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "FollowingEmptySymlink", "Treating empty symlink as regular path {0}", entry.Path); } } + if ((attributes & FileAttributes.Directory) == FileAttributes.Directory) { IMetahash metahash; if (!options.SkipMetadata) { - metahash = Utility.WrapMetadata(MetadataGenerator.GenerateMetadata(path, attributes, options, snapshot), options); + metahash = Utility.WrapMetadata(MetadataGenerator.GenerateMetadata(entry, attributes, options), options); } else { metahash = emptymetadata; } - Logging.Log.WriteVerboseMessage(FILELOGTAG, "AddDirectory", "Adding directory {0}", path); - await AddFolderToOutputAsync(path, lastwrite, metahash, database, streamblockchannel).ConfigureAwait(false); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "AddDirectory", "Adding directory {0}", entry.Path); + await AddFolderToOutputAsync(entry.Path, lastwrite, metahash, database, streamblockchannel).ConfigureAwait(false); return false; } diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 258119897..4c3171f6c 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -34,6 +34,8 @@ using Duplicati.Library.Common.IO; using Duplicati.Library.Logging; using Duplicati.Library.Main.Operation.Common; using System.Data; +using Duplicati.Library.DynamicLoader; +using Duplicati.Library.SourceProvider; namespace Duplicati.Library.Main.Operation { @@ -74,53 +76,120 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException(Strings.Common.PassphraseChangeUnsupported, "PassphraseChangeUnsupported"); } - public static Snapshots.ISnapshotService GetSnapshot(string[] sources, Options options) + /// + /// Gets a single source provider for the given sources + /// + /// The sources to get providers for + /// The options to use + /// The source providers + public static ISourceProvider GetSourceProvider(IEnumerable sources, Options options) + => Combiner.Combine(GetSourceProviders(sources, options)); + + /// + /// Gets a snapshot service for the given sources + /// + /// The sources to get the snapshot for + /// The options to use + /// The source provider + private static ISnapshotService GetFileSnapshotService(IEnumerable sources, Options options) { try { if (options.SnapShotStrategy != Options.OptimizationStrategy.Off) - return Duplicati.Library.Snapshots.SnapshotUtility.CreateSnapshot(sources, options.RawOptions); + return SnapshotUtility.CreateSnapshot(sources, options.RawOptions, options.SymlinkPolicy == Options.SymlinkStrategy.Follow); } catch (Exception ex) { if (options.SnapShotStrategy == Options.OptimizationStrategy.Required) throw new UserInformationException(Strings.Common.SnapshotFailedError(ex.Message), "SnapshotFailed", ex); else if (options.SnapShotStrategy == Options.OptimizationStrategy.On) - Logging.Log.WriteWarningMessage(LOGTAG, "SnapshotFailed", ex, Strings.Common.SnapshotFailedError(ex.Message)); + Log.WriteWarningMessage(LOGTAG, "SnapshotFailed", ex, Strings.Common.SnapshotFailedError(ex.Message)); else if (options.SnapShotStrategy == Options.OptimizationStrategy.Auto) - Logging.Log.WriteInformationMessage(LOGTAG, "SnapshotFailed", Strings.Common.SnapshotFailedError(ex.Message)); + Log.WriteInformationMessage(LOGTAG, "SnapshotFailed", Strings.Common.SnapshotFailedError(ex.Message)); } - if (OperatingSystem.IsMacOS() || OperatingSystem.IsLinux()) - { - return new NoSnapshotLinux(options.IgnoreAdvisoryLocking); - } - else if (OperatingSystem.IsWindows()) - { - return new NoSnapshotWindows(); - } - else - { - throw new NotSupportedException("Unsupported Operating System"); - } + return SnapshotUtility.CreateNoSnapshot(sources, options.IgnoreAdvisoryLocking, options.SymlinkPolicy == Options.SymlinkStrategy.Follow); } /// - /// Create instance of USN journal service + /// Gets all source providers for the given sources /// - /// - /// - /// - /// - /// - private UsnJournalService GetJournalService(IEnumerable sources, ISnapshotService snapshot, IFilter filter, long lastfilesetid) + /// The sources to get providers for + /// The options to use + /// The filter to use + /// The last fileset id + /// The source providers + private static List GetSourceProviders(IEnumerable sources, Options options) + { + // Group the sources by their type, so we can combine all snapshot paths into a single snapshot + var sourceTypes = sources.GroupBy(x => Library.Utility.Utility.GuessScheme(x) ?? "file", StringComparer.OrdinalIgnoreCase); + + // To avoid leaking snapshot instances, we create all instances first and then dispose them if an exception occurs + // The number of instances is expected to be low, so the memory overhead is acceptable + var results = new List(); + try + { + foreach (var entry in sourceTypes) + { + if ("file".Equals(entry.Key, StringComparison.OrdinalIgnoreCase)) + results.Add(new SourceProvider.File(GetFileSnapshotService(entry, options))); + else if ("vss".Equals(entry.Key, StringComparison.OrdinalIgnoreCase) || "lvm".Equals(entry.Key, StringComparison.OrdinalIgnoreCase)) + results.Add(new SourceProvider.File(SnapshotUtility.CreateSnapshot(entry, options.RawOptions, options.SymlinkPolicy == Options.SymlinkStrategy.Follow))); + else + { + foreach (var url in entry) + { + // Source providers are preferred over backends + var provider = SourceProviderLoader.GetSourceProvider(url, options.RawOptions); + if (provider != null) + { + results.Add(provider); + continue; + } + + // See if we can implement the same with a backend + var backend = BackendLoader.GetBackend(url, options.RawOptions); + if (backend != null) + { + results.Add(new SourceProvider.BackendSourceProvider(backend, url)); + continue; + } + + throw new UserInformationException(string.Format("The source \"{0}\" is not supported", url), "SourceNotSupported"); + } + } + } + } + catch + { + foreach (var provider in results) + (provider as IDisposable)?.Dispose(); + + throw; + } + + return results; + } + + public UsnJournalService GetJournalService(ISourceProvider provider, IFilter filter, long lastfilesetid) { if (m_options.UsnStrategy == Options.OptimizationStrategy.Off) return null; if (!OperatingSystem.IsWindows()) throw new UserInformationException("USN journal is only supported on Windows", "UsnJournalNotSupported"); + var providers = (provider is Combiner c ? c.Providers : [provider]) + .Select((x, i) => new { Provider = x, Index = i }) + .Where(x => x.Provider is SourceProvider.File) + .ToList(); + + if (providers.Count == 0) + return null; + if (providers.Count > 1) + throw new UserInformationException("Multiple USN journal services are not supported", "MultipleUSNJournalServices"); + var fileProvider = providers.First().Provider as SourceProvider.File; + var journalData = m_database.GetChangeJournalData(lastfilesetid); - var service = new UsnJournalService(sources, snapshot, filter, m_options.FileAttributeFilter, m_options.SkipFilesLargerThan, + var service = new UsnJournalService(fileProvider.SnapshotService, filter, m_options.FileAttributeFilter, m_options.SkipFilesLargerThan, journalData, cancellationTokenSource.Token); foreach (var volumeData in service.VolumeDataList) @@ -288,7 +357,7 @@ namespace Duplicati.Library.Main.Operation /// /// Performs the bulk of work by starting all relevant processes /// - private static async Task RunMainOperation(Backup.Channels channels, IEnumerable sources, Snapshots.ISnapshotService snapshot, UsnJournalService journalService, Backup.BackupDatabase database, IBackendManager backendManager, Backup.BackupStatsCollector stats, Options options, IFilter sourcefilter, IFilter filter, BackupResults result, ITaskReader taskreader, long filesetid, long lastfilesetid) + private static async Task RunMainOperation(Backup.Channels channels, ISourceProvider source, UsnJournalService journalService, Backup.BackupDatabase database, IBackendManager backendManager, Backup.BackupStatsCollector stats, Options options, IFilter sourcefilter, IFilter filter, BackupResults result, ITaskReader taskreader, long filesetid, long lastfilesetid) { using (new Logging.Timer(LOGTAG, "BackupMainOperation", "BackupMainOperation")) { @@ -299,15 +368,15 @@ namespace Duplicati.Library.Main.Operation new[] { Backup.DataBlockProcessor.Run(channels, database, backendManager, options, taskreader), - Backup.FileBlockProcessor.Run(channels, snapshot, options, database, stats, taskreader), + Backup.FileBlockProcessor.Run(channels, options, database, stats, taskreader), Backup.StreamBlockSplitter.Run(channels, options, database, taskreader), - Backup.FileEnumerationProcess.Run(channels, sources, snapshot, journalService, - options.FileAttributeFilter, sourcefilter, filter, options.SymlinkPolicy, + Backup.FileEnumerationProcess.Run(channels, source, journalService, + options.FileAttributeFilter, filter, options.SymlinkPolicy, options.HardlinkPolicy, options.ExcludeEmptyFolders, options.IgnoreFilenames, GetBlacklistedPaths(options), options.ChangedFilelist, taskreader, () => result.PartialBackup = true, CancellationToken.None), - Backup.FilePreFilterProcess.Run(channels, snapshot, options, stats, database), - Backup.MetadataPreProcess.Run(channels, snapshot, options, database, lastfilesetid, taskreader), + Backup.FilePreFilterProcess.Run(channels, options, stats, database), + Backup.MetadataPreProcess.Run(channels, options, database, lastfilesetid, taskreader), Backup.SpillCollectorProcess.Run(channels, options, database, backendManager, taskreader), Backup.ProgressHandler.Run(channels, result) } @@ -324,7 +393,7 @@ namespace Duplicati.Library.Main.Operation // Spawn additional file processors .Concat( Enumerable.Range(0, options.ConcurrencyFileprocessors - 1).Select(x => - Backup.FileBlockProcessor.Run(channels, snapshot, options, database, stats, taskreader) + Backup.FileBlockProcessor.Run(channels, options, database, stats, taskreader) ) ) ); @@ -351,9 +420,8 @@ namespace Duplicati.Library.Main.Operation return true; if (fileSize >= 0) - { stats.AddExaminedFile(fileSize); - } + return false; }); @@ -528,7 +596,7 @@ namespace Duplicati.Library.Main.Operation { long filesetid; using var counterToken = CancellationTokenSource.CreateLinkedTokenSource(m_taskReader.ProgressToken); - using (var snapshot = GetSnapshot(sources, m_options)) + using (var source = GetSourceProvider(sources, m_options)) { try { @@ -558,8 +626,7 @@ namespace Duplicati.Library.Main.Operation var filesetvolumeid = await db.RegisterRemoteVolumeAsync(filesetvolume.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary); filesetid = await db.CreateFilesetAsync(filesetvolumeid, VolumeBase.ParseFilename(filesetvolume.RemoteFilename).Time); - // create USN-based scanner if enabled - var journalService = GetJournalService(sources, snapshot, filter, lastfilesetid); + var journalService = GetJournalService(source, filter, lastfilesetid); // Start parallel scan, or use the database if (m_options.DisableFileScanner) @@ -569,13 +636,13 @@ namespace Duplicati.Library.Main.Operation } else { - parallelScanner = Backup.CountFilesHandler.Run(sources, snapshot, journalService, m_result, m_options, m_sourceFilter, m_filter, GetBlacklistedPaths(), m_result.TaskControl, counterToken.Token); + parallelScanner = Backup.CountFilesHandler.Run(source, journalService, m_result, m_options, m_filter, GetBlacklistedPaths(), m_result.TaskControl, counterToken.Token); } // Run the backup operation if (await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await RunMainOperation(channels, sources, snapshot, journalService, db, backendManager, stats, m_options, m_sourceFilter, m_filter, m_result, m_result.TaskControl, filesetid, lastfilesetid).ConfigureAwait(false); + await RunMainOperation(channels, source, journalService, db, backendManager, stats, m_options, m_sourceFilter, m_filter, m_result, m_result.TaskControl, filesetid, lastfilesetid).ConfigureAwait(false); } } finally diff --git a/Duplicati/Library/Main/Operation/TestFilterHandler.cs b/Duplicati/Library/Main/Operation/TestFilterHandler.cs index c2f96ffba..4a06bbc89 100644 --- a/Duplicati/Library/Main/Operation/TestFilterHandler.cs +++ b/Duplicati/Library/Main/Operation/TestFilterHandler.cs @@ -1,27 +1,26 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.IO; -using Duplicati.Library.Snapshots; using CoCoL; using System.Threading.Tasks; @@ -45,14 +44,13 @@ namespace Duplicati.Library.Main.Operation public async Task RunAsync(string[] sources, Library.Utility.IFilter filter) { - var sourcefilter = new Library.Utility.FilterExpression(sources, true); var stopToken = m_result.TaskControl.ProgressToken; - using (var snapshot = BackupHandler.GetSnapshot(sources, m_options)) + using (var provider = BackupHandler.GetSourceProvider(sources, m_options)) { Backup.Channels channels = new(); - var source = Backup.FileEnumerationProcess.Run(channels, sources, snapshot, null, - m_options.FileAttributeFilter, sourcefilter, filter, m_options.SymlinkPolicy, + var source = Backup.FileEnumerationProcess.Run(channels, provider, null, + m_options.FileAttributeFilter, filter, m_options.SymlinkPolicy, m_options.HardlinkPolicy, m_options.ExcludeEmptyFolders, m_options.IgnoreFilenames, BackupHandler.GetBlacklistedPaths(m_options), null, m_result.TaskControl, null, stopToken); @@ -64,20 +62,20 @@ namespace Duplicati.Library.Main.Operation { while (await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - var path = await self.source.ReadAsync(); - var fa = FileAttributes.Normal; - try { fa = snapshot.GetAttributes(path); } - catch (Exception ex) { Logging.Log.WriteVerboseMessage(LOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", path, ex.Message); } + var entry = await self.source.ReadAsync(); + var fa = entry.IsFolder + ? FileAttributes.Directory + : FileAttributes.Normal; + + try { fa = entry.Attributes; } + catch (Exception ex) { Logging.Log.WriteVerboseMessage(LOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", entry.Path, ex.Message); } // Analyze symlinks - var isSymlink = snapshot.IsSymlink(path, fa); string symlinkTarget = null; + try { symlinkTarget = entry.SymlinkTarget; } + catch (Exception ex) { Logging.Log.WriteExplicitMessage(LOGTAG, "SymlinkTargetReadFailure", ex, "Failed to read symlink target for path: {0}", entry.Path); } - if (isSymlink) - try { symlinkTarget = snapshot.GetSymlinkTarget(path); } - catch (Exception ex) { Logging.Log.WriteExplicitMessage(LOGTAG, "SymlinkTargetReadFailure", ex, "Failed to read symlink target for path: {0}", path); } - - if (isSymlink && m_options.SymlinkPolicy == Options.SymlinkStrategy.Store && !string.IsNullOrWhiteSpace(symlinkTarget)) + if (m_options.SymlinkPolicy == Options.SymlinkStrategy.Store && !string.IsNullOrWhiteSpace(symlinkTarget)) { // Skip stored symlinks continue; @@ -86,33 +84,37 @@ namespace Duplicati.Library.Main.Operation // Go for the symlink target, as we know we follow symlinks if (!string.IsNullOrWhiteSpace(symlinkTarget)) { - path = symlinkTarget; + entry = await provider.GetEntry(symlinkTarget, false, stopToken).ConfigureAwait(false); fa = FileAttributes.Normal; - try { fa = snapshot.GetAttributes(path); } - catch (Exception ex) { Logging.Log.WriteVerboseMessage(LOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", path, ex.Message); } + + try { fa = entry.Attributes; } + catch (Exception ex) { Logging.Log.WriteVerboseMessage(LOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", entry.Path, ex.Message); } + + if (fa.HasFlag(FileAttributes.Directory)) + entry = await provider.GetEntry(symlinkTarget, true, stopToken).ConfigureAwait(false); } // Proceed with non-folders - if (!((fa & FileAttributes.Directory) == FileAttributes.Directory)) + if (!entry.IsFolder) { m_result.FileCount++; var size = -1L; try { - size = snapshot.GetFileSize(path); + size = entry.Size; m_result.FileSize += size; } catch (Exception ex) { - Logging.Log.WriteVerboseMessage(LOGTAG, "SizeReadFailed", "Failed to read length of file {0}: {1}", path, ex.Message); + Logging.Log.WriteVerboseMessage(LOGTAG, "SizeReadFailed", "Failed to read length of file {0}: {1}", entry.Path, ex.Message); } if (m_options.SkipFilesLargerThan == long.MaxValue || m_options.SkipFilesLargerThan == 0 || size < m_options.SkipFilesLargerThan) - Logging.Log.WriteVerboseMessage(LOGTAG, "IncludeFile", "Including file: {0} ({1})", path, size < 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(size)); + Logging.Log.WriteVerboseMessage(LOGTAG, "IncludeFile", "Including file: {0} ({1})", entry.Path, size < 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(size)); else - Logging.Log.WriteVerboseMessage(LOGTAG, "ExcludeLargeFile", "Excluding file due to size: {0} ({1})", path, size < 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(size)); + Logging.Log.WriteVerboseMessage(LOGTAG, "ExcludeLargeFile", "Excluding file due to size: {0} ({1})", entry.Path, size < 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(size)); } } } diff --git a/Duplicati/Library/Modules/Builtin/ReportHelper.cs b/Duplicati/Library/Modules/Builtin/ReportHelper.cs index 78cbfae29..e37ee17ac 100644 --- a/Duplicati/Library/Modules/Builtin/ReportHelper.cs +++ b/Duplicati/Library/Modules/Builtin/ReportHelper.cs @@ -411,7 +411,7 @@ namespace Duplicati.Library.Modules.Builtin case DESTINATION_TYPE: // Only return the url scheme, as the rest could contain sensitive information var ix = m_remoteurl?.IndexOf("://", StringComparison.OrdinalIgnoreCase) ?? -1; - return ix >= 0 && ix < 15 ? m_remoteurl.Substring(0, ix) : "file"; + return Utility.Utility.GuessScheme(m_remoteurl) ?? "file"; default: return null; } diff --git a/Duplicati/Library/Modules/Builtin/SendJabberMessage.cs b/Duplicati/Library/Modules/Builtin/SendJabberMessage.cs index 2f8dac808..bbb565017 100644 --- a/Duplicati/Library/Modules/Builtin/SendJabberMessage.cs +++ b/Duplicati/Library/Modules/Builtin/SendJabberMessage.cs @@ -195,8 +195,7 @@ namespace Duplicati.Library.Modules.Builtin protected override void SendMessage(string subject, string body) { - - var uri = new Library.Utility.Uri(m_username.Contains("://") ? m_username : "http://" + m_username); + var uri = new Utility.Uri(m_username.Contains("://") ? m_username : "http://" + m_username); var resource = uri.Path ?? ""; if (resource.StartsWith("/", StringComparison.Ordinal)) resource = resource.Substring(1); diff --git a/Duplicati/Library/Snapshots/Duplicati.Library.Snapshots.csproj b/Duplicati/Library/Snapshots/Duplicati.Library.Snapshots.csproj index 2f5c08116..a166310db 100644 --- a/Duplicati/Library/Snapshots/Duplicati.Library.Snapshots.csproj +++ b/Duplicati/Library/Snapshots/Duplicati.Library.Snapshots.csproj @@ -3,10 +3,12 @@ net8.0 Copyright © 2025 Team Duplicati, MIT license + Duplicati.Library.Snapshots + @@ -14,7 +16,7 @@ - + diff --git a/Duplicati/Library/Snapshots/ISnapshotService.cs b/Duplicati/Library/Snapshots/ISnapshotService.cs index 9f61d6886..4deea290f 100644 --- a/Duplicati/Library/Snapshots/ISnapshotService.cs +++ b/Duplicati/Library/Snapshots/ISnapshotService.cs @@ -1,27 +1,32 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable using System; using System.Collections.Generic; using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Duplicati.Library.Interface; namespace Duplicati.Library.Snapshots { @@ -31,12 +36,28 @@ namespace Duplicati.Library.Snapshots public interface ISnapshotService : IDisposable { /// - /// Enumerates all files and folders in the snapshot, restricted to sources + /// Gets the source folders /// - /// Sources to enumerate - /// The callback to invoke with each found path - /// The callback used to report errors - IEnumerable EnumerateFilesAndFolders(IEnumerable sources, Utility.Utility.EnumerationFilterDelegate callback, Utility.Utility.ReportAccessError errorCallback); + IEnumerable SourceFolders { get; } + + /// + /// Enumerates the root source files and folders + /// + /// The source files and folders + IEnumerable EnumerateFilesystemEntries(); + /// + /// Enumerates the files and folders in a given folder + /// + /// The parent folder to enumerate + /// The files and folders in the parent folder + IEnumerable EnumerateFilesystemEntries(ISourceFileEntry parent); + /// + /// Gets a filesystem entry for a given path + /// + /// The path to get + /// True if the path is a folder + /// The filesystem entry + ISourceFileEntry GetFilesystemEntry(string path, bool isFolder); /// /// Gets the last write time of a given file in UTC @@ -59,6 +80,14 @@ namespace Duplicati.Library.Snapshots /// An open filestream that can be read and seeked Stream OpenRead(string localPath); + /// + /// Opens a file for reading + /// + /// The full path to the file in non-snapshot format + /// A token that can be used to cancel the operation + /// An open filestream that can be read and seeked + Task OpenReadAsync(string localPath, CancellationToken cancellationToken); + /// /// Returns the size of a file /// @@ -79,29 +108,28 @@ namespace Duplicati.Library.Snapshots /// The file attributes /// The file or folder to examine FileAttributes GetAttributes(string localPath); - + /// /// Gets the metadata for the given file or folder /// /// The metadata for the given file or folder /// The file or folder to examine /// A flag indicating if the target is a symlink - /// A flag indicating if a symlink should be followed - Dictionary GetMetadata(string localPath, bool isSymlink, bool followSymlink); - + Dictionary GetMetadata(string localPath, bool isSymlink); + /// /// Gets a value indicating if the path points to a block device /// /// true if this instance is a block device; otherwise, false. /// The file or folder to examine bool IsBlockDevice(string localPath); - + /// /// Gets a unique hardlink target ID /// /// The hardlink ID /// The file or folder to examine - string HardlinkTargetID(string localPath); + string? HardlinkTargetID(string localPath); /// /// Converts a snapshot path to a local path @@ -130,17 +158,5 @@ namespace Duplicati.Library.Snapshots /// The local path /// True if folder exists, false otherwise bool DirectoryExists(string localFolderPath); - - /// - /// Tests if a file or folder exists in the snapshot - /// - /// The local path - /// True if file or folder exists, false otherwise - bool PathExists(string localFileOrFolderPath); - - /// - /// Is true if the actual service is a real snapshot service - /// - bool IsSnapshot { get; } } } diff --git a/Duplicati/Library/Snapshots/LinuxSnapshot.cs b/Duplicati/Library/Snapshots/LinuxSnapshot.cs index f49044f04..9d5a51832 100644 --- a/Duplicati/Library/Snapshots/LinuxSnapshot.cs +++ b/Duplicati/Library/Snapshots/LinuxSnapshot.cs @@ -1,31 +1,33 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; using System.Diagnostics; +using System.Linq; using System.Runtime.Versioning; using System.Text; using Duplicati.Library.Common.IO; +using Duplicati.Library.Interface; namespace Duplicati.Library.Snapshots { @@ -40,7 +42,7 @@ namespace Duplicati.Library.Snapshots [SupportedOSPlatform("macOS")] public sealed class LinuxSnapshot : SnapshotBase { - /// + /// /// The tag used for logging messages /// public static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(WindowsSnapshot)); @@ -59,7 +61,9 @@ namespace Duplicati.Library.Snapshots /// Constructs a new snapshot module using LVM /// /// The list of folders to create snapshots for - public LinuxSnapshot(IEnumerable sources) + /// A flag indicating if symlinks should be followed + public LinuxSnapshot(IEnumerable sources, bool followSymlinks) + : base(followSymlinks) { try { @@ -79,7 +83,7 @@ namespace Duplicati.Library.Snapshots m_entries.Add(new KeyValuePair(path, snap)); } - m_snapShots = new List(snaps.Values); + m_snapShots = [.. snaps.Values]; // We have all the snapshots that we need, lets activate them foreach (var snap in m_snapShots) @@ -94,15 +98,44 @@ namespace Duplicati.Library.Snapshots { Dispose(); } - catch (Exception ex) + catch (Exception ex) { - Logging.Log.WriteVerboseMessage(LOGTAG, "SnapshotCleanupError", ex, "Failed to clean up after error"); + Logging.Log.WriteVerboseMessage(LOGTAG, "SnapshotCleanupError", ex, "Failed to clean up after error"); } throw; } } + /// + /// Gets the source folders + /// + public override IEnumerable SourceFolders => m_snapShots.Select(e => e.LocalPath); + + /// + /// Gets a filesystem entry for a given path + /// + /// The path to get the entry for + /// A flag indicating if the path is a folder + /// The filesystem entry + public override ISourceFileEntry GetFilesystemEntry(string path, bool isFolder) + => new SnapshotSourceFileEntry(this, path, isFolder, false); + + /// + /// Enumerates the root source files and folders + /// + /// The source files and folders + public override IEnumerable EnumerateFilesystemEntries() + { + foreach (var snap in m_snapShots) + { + if (DirectoryExists(snap.LocalPath) || snap.LocalPath.EndsWith(System.IO.Path.DirectorySeparatorChar)) + yield return new SnapshotSourceFileEntry(this, snap.LocalPath, true, true); + else + yield return new SnapshotSourceFileEntry(this, snap.LocalPath, false, true); + } + } + /// protected override void Dispose(bool disposing) { @@ -111,10 +144,10 @@ namespace Duplicati.Library.Snapshots if (disposing) { // Attempt to clean out as many as possible - foreach(var s in m_snapShots) + foreach (var s in m_snapShots) { try { s.Dispose(); } - catch (Exception ex) { Logging.Log.WriteVerboseMessage(LOGTAG, "SnapshotCloseError", ex, "Failed to close a snapshot"); } + catch (Exception ex) { Logging.Log.WriteVerboseMessage(LOGTAG, "SnapshotCloseError", ex, "Failed to close a snapshot"); } } } @@ -475,10 +508,9 @@ namespace Duplicati.Library.Snapshots /// The metadata for the given file or folder /// The file or folder to examine /// A flag indicating if the target is a symlink - /// A flag indicating if a symlink should be followed - public override Dictionary GetMetadata(string localPath, bool isSymlink, bool followSymlink) + public override Dictionary GetMetadata(string localPath, bool isSymlink) { - return SystemIO.IO_SYS.GetMetadata(ConvertToSnapshotPath(localPath), isSymlink, followSymlink); + return SystemIO.IO_SYS.GetMetadata(ConvertToSnapshotPath(localPath), isSymlink, FollowSymlinks); } /// @@ -500,8 +532,8 @@ namespace Duplicati.Library.Snapshots default: return true; } - } - catch + } + catch { if (!System.IO.File.Exists(SystemIOLinux.NormalizePath(localPath))) return false; @@ -509,7 +541,7 @@ namespace Duplicati.Library.Snapshots throw; } } - + /// /// Gets a unique hardlink target ID /// @@ -518,10 +550,10 @@ namespace Duplicati.Library.Snapshots public override string HardlinkTargetID(string localPath) { var snapshotPath = ConvertToSnapshotPath(localPath); - + if (PosixFile.GetHardlinkCount(snapshotPath) <= 1) return null; - + return PosixFile.GetInodeTargetID(snapshotPath); } @@ -537,9 +569,6 @@ namespace Duplicati.Library.Snapshots return FindSnapshotByLocalPath(localPath).ConvertToSnapshotPath(localPath); } - /// - public override bool IsSnapshot => true; - #endregion } } diff --git a/Duplicati/Library/Snapshots/NoSnapshotLinux.cs b/Duplicati/Library/Snapshots/NoSnapshotLinux.cs index f1601169b..3bad638eb 100644 --- a/Duplicati/Library/Snapshots/NoSnapshotLinux.cs +++ b/Duplicati/Library/Snapshots/NoSnapshotLinux.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -25,6 +25,7 @@ using System.IO; using System.Runtime.InteropServices; using System.Runtime.Versioning; using Duplicati.Library.Common.IO; +using Duplicati.Library.Interface; using Microsoft.Win32.SafeHandles; namespace Duplicati.Library.Snapshots @@ -65,15 +66,43 @@ namespace Duplicati.Library.Snapshots /// private readonly bool m_ignoreAdvisoryLocks; + /// + /// The list of folders to create snapshots of + /// + private readonly IEnumerable m_folders; + /// /// Initializes a new instance of the class. /// + /// The list of folders to create snapshots of /// A flag indicating if advisory locks should be ignored - public NoSnapshotLinux(bool ignoreAdvisoryLocks) + /// A flag indicating if symlinks should be followed + public NoSnapshotLinux(IEnumerable folders, bool ignoreAdvisoryLocks, bool followSymlinks) + : base(followSymlinks) { + m_folders = folders; m_ignoreAdvisoryLocks = ignoreAdvisoryLocks; } + /// + /// Gets the source folders + /// + public override IEnumerable SourceFolders => m_folders; + + /// + /// Enumerates the root source files and folders + /// + /// The source files and folders + public override IEnumerable EnumerateFilesystemEntries() + { + foreach (var folder in m_folders) + { + if (DirectoryExists(folder) || folder.EndsWith(Path.DirectorySeparatorChar)) + yield return new SnapshotSourceFileEntry(this, folder, true, true); + else + yield return new SnapshotSourceFileEntry(this, folder, false, true); + } + } /// /// Returns the symlink target if the entry is a symlink, and null otherwise @@ -91,10 +120,9 @@ namespace Duplicati.Library.Snapshots /// The metadata for the given file or folder /// The file or folder to examine /// A flag indicating if the target is a symlink - /// A flag indicating if a symlink should be followed - public override Dictionary GetMetadata(string localPath, bool isSymlink, bool followSymlink) + public override Dictionary GetMetadata(string localPath, bool isSymlink) { - return SystemIO.IO_SYS.GetMetadata(localPath, isSymlink, followSymlink); + return SystemIO.IO_SYS.GetMetadata(localPath, isSymlink, FollowSymlinks); } /// diff --git a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs index f3a3c7184..706554cfd 100644 --- a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs +++ b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -24,6 +24,7 @@ using System.Collections.Generic; using System.Linq; using System.Runtime.Versioning; using Duplicati.Library.Common.IO; +using Duplicati.Library.Interface; namespace Duplicati.Library.Snapshots { @@ -33,100 +34,114 @@ namespace Duplicati.Library.Snapshots [SupportedOSPlatform("windows")] public sealed class NoSnapshotWindows : SnapshotBase { + /// + /// The list of folders to create snapshots of + /// + private readonly IEnumerable m_folders; + + /// + /// Initializes a new instance of the class. + /// + /// The list of folders to create snapshots of + /// A flag indicating if symlinks should be followed + public NoSnapshotWindows(IEnumerable folders, bool followSymlinks) + : base(followSymlinks) + { + m_folders = folders; + } + /// /// Returns the symlink target if the entry is a symlink, and null otherwise /// /// The file or folder to examine /// The symlink target public override string GetSymlinkTarget(string localPath) - { - return SystemIO.IO_WIN.GetSymlinkTarget(localPath); - } + => SystemIO.IO_WIN.GetSymlinkTarget(localPath); /// /// Gets the attributes for the given file or folder /// /// The file attributes /// The file or folder to examine - public override System.IO.FileAttributes GetAttributes (string localPath) - { - return SystemIO.IO_WIN.GetFileAttributes(localPath); - } + public override System.IO.FileAttributes GetAttributes(string localPath) + => SystemIO.IO_WIN.GetFileAttributes(localPath); /// /// Returns the size of a file /// /// The full path to the file in non-snapshot format /// The length of the file - public override long GetFileSize (string localPath) - { - return SystemIO.IO_WIN.FileLength(localPath); - } + public override long GetFileSize(string localPath) + => SystemIO.IO_WIN.FileLength(localPath); /// - /// Enumerates all files and folders in the snapshot, restricted to sources + /// Gets the source folders /// - /// Sources to enumerate - /// The callback to invoke with each found path - /// The callback used to report errors - public override IEnumerable EnumerateFilesAndFolders(IEnumerable sources, Utility.Utility.EnumerationFilterDelegate callback, Utility.Utility.ReportAccessError errorCallback) + public override IEnumerable SourceFolders + => m_folders; + + /// + /// Enumerates the root source files and folders + /// + /// The source files and folders + public override IEnumerable EnumerateFilesystemEntries() { - // For Windows, ensure we don't store paths with extended device path prefixes (i.e., @"\\?\" or @"\\?\UNC\") - return base.EnumerateFilesAndFolders(sources.Select(SystemIOWindows.RemoveExtendedDevicePathPrefix), callback, errorCallback); + foreach (var folder in m_folders.Select(SystemIOWindows.AddExtendedDevicePathPrefix)) + { + if (DirectoryExists(folder) || folder.EndsWith(System.IO.Path.DirectorySeparatorChar)) + yield return new SnapshotSourceFileEntry(this, folder, true, true); + else + yield return new SnapshotSourceFileEntry(this, folder, false, true); + } } + /// /// Gets the last write time of a given file in UTC /// /// The full path to the file in non-snapshot format /// The last write time of the file - public override DateTime GetLastWriteTimeUtc (string localPath) - { - return SystemIO.IO_WIN.FileGetLastWriteTimeUtc(localPath); - } + public override DateTime GetLastWriteTimeUtc(string localPath) + => SystemIO.IO_WIN.FileGetLastWriteTimeUtc(localPath); /// /// Gets the last write time of a given file in UTC /// /// The full path to the file in non-snapshot format /// The last write time of the file - public override DateTime GetCreationTimeUtc (string localPath) - { - return SystemIO.IO_WIN.FileGetCreationTimeUtc(localPath); - } + public override DateTime GetCreationTimeUtc(string localPath) + => SystemIO.IO_WIN.FileGetCreationTimeUtc(localPath); /// /// Opens a file for reading /// /// The full path to the file in non-snapshot format /// An open filestream that can be read - public override System.IO.Stream OpenRead (string localPath) - { - return SystemIO.IO_WIN.FileOpenRead(localPath); - } + public override System.IO.Stream OpenRead(string localPath) + => SystemIO.IO_WIN.FileOpenRead(localPath); /// /// Lists all files in the given folder /// /// All folders found in the folder /// The folder to examinate - protected override string[] ListFiles (string localFolderPath) + protected override string[] ListFiles(string localFolderPath) { string[] tmp = SystemIO.IO_WIN.GetFiles(localFolderPath); string[] res = new string[tmp.Length]; - for(int i = 0; i < tmp.Length; i++) + for (int i = 0; i < tmp.Length; i++) res[i] = SystemIOWindows.RemoveExtendedDevicePathPrefix(tmp[i]); return res; } - + /// /// Lists all folders in the given folder /// /// All folders found in the folder /// The folder to examinate - protected override string[] ListFolders (string localFolderPath) + protected override string[] ListFolders(string localFolderPath) { string[] tmp = SystemIO.IO_WIN.GetDirectories(SystemIOWindows.AddExtendedDevicePathPrefix(localFolderPath)); string[] res = new string[tmp.Length]; @@ -142,11 +157,8 @@ namespace Duplicati.Library.Snapshots /// The metadata for the given file or folder /// The file or folder to examine /// A flag indicating if the target is a symlink - /// A flag indicating if a symlink should be followed - public override Dictionary GetMetadata(string localPath, bool isSymlink, bool followSymlink) - { - return SystemIO.IO_WIN.GetMetadata(localPath, isSymlink, followSymlink); - } + public override Dictionary GetMetadata(string localPath, bool isSymlink) + => SystemIO.IO_WIN.GetMetadata(localPath, isSymlink, FollowSymlinks); /// public override string ConvertToLocalPath(string snapshotPath) @@ -155,11 +167,9 @@ namespace Duplicati.Library.Snapshots } /// - public override string ConvertToSnapshotPath(string localPath) - { + public override string ConvertToSnapshotPath(string localPath) => // For Windows, ensure we don't store paths with extended device path prefixes (i.e., @"\\?\" or @"\\?\UNC\") - return SystemIOWindows.RemoveExtendedDevicePathPrefix(localPath); - } + SystemIOWindows.RemoveExtendedDevicePathPrefix(localPath); } } diff --git a/Duplicati/Library/Snapshots/Program.cs b/Duplicati/Library/Snapshots/Program.cs index 5f39417e9..6afd2647d 100644 --- a/Duplicati/Library/Snapshots/Program.cs +++ b/Duplicati/Library/Snapshots/Program.cs @@ -110,7 +110,7 @@ Where is the folder where files will be locked/created etc"); Console.WriteLine("Creating snapshot for folder: {0}", args[0]); Console.WriteLine("If this fails, try to run as " + ((OperatingSystem.IsMacOS() || OperatingSystem.IsLinux()) ? "root" : "Administrator")); - using (ISnapshotService snapshot = SnapshotUtility.CreateSnapshot(new[] { args[0] }, options)) + using (ISnapshotService snapshot = SnapshotUtility.CreateSnapshot(new[] { args[0] }, options, false)) { Console.WriteLine("Attempting to read locked file via snapshot"); try diff --git a/Duplicati/Library/Snapshots/RecursiveFilterHelper.cs b/Duplicati/Library/Snapshots/RecursiveFilterHelper.cs new file mode 100644 index 000000000..1e8fb9f02 --- /dev/null +++ b/Duplicati/Library/Snapshots/RecursiveFilterHelper.cs @@ -0,0 +1,229 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using Duplicati.Library.Interface; +using System.Threading.Tasks; +using System.Runtime.CompilerServices; + +namespace Duplicati.Library.Snapshots; + +public static class RecursiveFilterHelper +{ + /// + /// The log tag to use + /// + private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(RecursiveFilterHelper)); + + /// + /// Filters sources, returning sub-set having been modified since last + /// change, as specified by journalData. + /// + /// List of sources + /// Snapshot service + /// Filter callback to exclude filtered items + /// Cancellation token + /// Filtered sources + public static async IAsyncEnumerable GetModifiedSources(IAsyncEnumerable entries, ISnapshotService snapshot, Func> filter, [EnumeratorCancellation] CancellationToken token) + { + // prepare cache for includes (value = true) and excludes (value = false, will be populated + // on-demand) + var cache = new Dictionary(); + await foreach (var source in entries) + { + if (token.IsCancellationRequested) + break; + + cache[source.Path] = true; + } + + // Check the simplified folders, and their parent folders against the exclusion filter. + // This is needed because the filter may exclude "C:\A\", but this won't match the more + // specific "C:\A\B\" in our list, even though it's meant to be excluded. + // The reason why the filter doesn't exclude it is because during a regular (non-USN) full scan, + // FilterHandler.EnumerateFilesAndFolders() works top-down, and won't even enumerate child + // folders. + // The sources are needed to stop evaluating parent folders above the specified source folders + await foreach (var folder in FilterExcludedFolders(entries.Where(x => x.IsFolder), snapshot, filter, cache, token)) + { + if (token.IsCancellationRequested) + break; + + if (!snapshot.DirectoryExists(folder.Path)) + continue; + + yield return folder; + } + + // The simplified file list also needs to be checked against the exclusion filter, as it + // may contain entries excluded due to attributes, but also because they are below excluded + // folders, which themselves aren't in the folder list from step 1. + // Note that the simplified file list may contain entries that have been deleted! They need to + // be kept in the list (unless excluded by the filter) in order for the backup handler to record their + // deletion. + await foreach (var files in FilterExcludedFiles(entries.Where(x => !x.IsFolder), snapshot, filter, cache, token)) + { + if (token.IsCancellationRequested) + break; + + if (!snapshot.FileExists(files.Path)) + continue; + + yield return files; + } + } + + /// + /// Filter supplied files, removing any files which itself, or one + /// of its parent folders, is excluded by the filter. + /// + /// Files to filter + /// Snapshot service + /// Exclusion filter + /// Cache of included and excluded files / folders + /// Cancellation token + /// Filtered files + private static async IAsyncEnumerable FilterExcludedFiles( + IAsyncEnumerable files, + ISnapshotService snapshot, + Func> filter, + IDictionary cache, + [EnumeratorCancellation] CancellationToken token) + { + await foreach (var file in files.WithCancellation(token).ConfigureAwait(false)) + { + try + { + if (!await filter(file)) + continue; + + var parentPath = Utility.Utility.GetParent(file.Path, true); + if (!string.IsNullOrWhiteSpace(parentPath)) + { + var parent = snapshot.GetFilesystemEntry(parentPath, true); + if (await IsFolderOrAncestorsExcluded(parent, snapshot, filter, cache, token)) + continue; + } + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "FilterExcludedFilesError", ex, "Error while filtering file: {0}", ex.Message); + continue; + } + + yield return file; + } + } + + /// + /// Filter supplied folders, removing any folder which itself, or one + /// of its ancestors, is excluded by the filter. + /// + /// Folder to filter + /// Snapshot service + /// Exclusion filter + /// Cache of excluded folders + /// Cancellation token + /// Filtered folders + private static async IAsyncEnumerable FilterExcludedFolders( + IAsyncEnumerable folders, + ISnapshotService snapshot, + Func> filter, + IDictionary cache, + [EnumeratorCancellation] CancellationToken token) + { + await foreach (var folder in folders.WithCancellation(token).ConfigureAwait(false)) + { + try + { + if (await IsFolderOrAncestorsExcluded(folder, snapshot, filter, cache, token)) + continue; + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "FilterExcludedFoldersError", ex, "Error while filtering folder: {0}", ex.Message); + continue; + } + + yield return folder; + } + } + + /// + /// Tests if specified folder, or any of its ancestors, is excluded by the filter + /// + /// Folder to test + /// Filter + /// Cache of excluded folders (optional) + /// True if excluded, false otherwise + private static async ValueTask IsFolderOrAncestorsExcluded( + ISourceFileEntry folder, + ISnapshotService snapshot, + Func> filter, + IDictionary cache, + CancellationToken token) + { + List? parents = null; + while (folder != null) + { + if (token.IsCancellationRequested) + break; + + // first check cache + if (cache.TryGetValue(folder.Path, out var include)) + { + if (include) + return false; + + break; // hit! + } + + parents ??= []; // create on-demand + + // remember folder for cache + parents.Add(folder.Path); + + + if (!await filter(folder)) + break; // excluded + + var parentPath = Utility.Utility.GetParent(folder.Path, true); + if (string.IsNullOrWhiteSpace(parentPath)) + break; + + folder = snapshot.GetFilesystemEntry(parentPath, true); + } + + if (folder != null) + { + // update cache + parents?.ForEach(p => cache[p] = false); + } + + return folder != null; + } +} + diff --git a/Duplicati/Library/Snapshots/SnapshotBase.cs b/Duplicati/Library/Snapshots/SnapshotBase.cs index f5d6da147..1c6151137 100644 --- a/Duplicati/Library/Snapshots/SnapshotBase.cs +++ b/Duplicati/Library/Snapshots/SnapshotBase.cs @@ -20,10 +20,11 @@ // DEALINGS IN THE SOFTWARE. using System; -using System.Linq; using System.Collections.Generic; using System.IO; -using Duplicati.Library.Common.IO; +using Duplicati.Library.Interface; +using System.Threading.Tasks; +using System.Threading; namespace Duplicati.Library.Snapshots { @@ -40,19 +41,56 @@ namespace Duplicati.Library.Snapshots Dispose(false); } - #region ISnapshotService - /// - /// Enumerates all files and folders in the shadow copy + /// Initializes a new instance of the class. /// - /// Source paths to enumerate - /// The callback to invoke with each found path - /// The callback used to report errors - public virtual IEnumerable EnumerateFilesAndFolders(IEnumerable sources, Utility.Utility.EnumerationFilterDelegate callback, Utility.Utility.ReportAccessError errorCallback) + /// A flag indicating if symlinks should be followed + protected SnapshotBase(bool followSymlinks) { - return sources.SelectMany(s => Utility.Utility.EnumerateFileSystemEntries(s, callback, ListFolders, ListFiles, GetAttributes, errorCallback)); + FollowSymlinks = followSymlinks; } - + + #region ISnapshotService + + protected bool FollowSymlinks { get; } + + /// + /// Gets the source folders + /// + public abstract IEnumerable SourceFolders { get; } + + /// + /// Enuemrates the root source files and folders + /// + /// The source files and folders + public abstract IEnumerable EnumerateFilesystemEntries(); + + /// + /// Gets a filesystem entry for a given path + /// + /// The path to get the entry for + /// A flag indicating if the path is a folder + /// The filesystem entry + public virtual ISourceFileEntry GetFilesystemEntry(string path, bool isFolder) + => new SnapshotSourceFileEntry(this, path, isFolder, false); + + /// + /// Enumerates files and folders in the given folder + /// + /// Source to enumerate + /// The files and folders in the given folder + public virtual IEnumerable EnumerateFilesystemEntries(ISourceFileEntry source) + { + if (source.IsFolder) + { + foreach (var f in ListFiles(source.Path)) + yield return new SnapshotSourceFileEntry(this, f, false, false); + + foreach (var d in ListFolders(source.Path)) + yield return new SnapshotSourceFileEntry(this, d, true, false); + } + } + /// /// Gets the last write time of a given file in UTC /// @@ -72,7 +110,7 @@ namespace Duplicati.Library.Snapshots { return File.GetCreationTimeUtc(localPath); } - + /// /// Opens a file for reading /// @@ -83,6 +121,15 @@ namespace Duplicati.Library.Snapshots return File.Open(localPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); } + /// + /// Opens a file for reading + /// + /// The full path to the file in non-snapshot format + /// A token that can be used to cancel the operation + /// An open filestream that can be read + public virtual Task OpenReadAsync(string localPath, CancellationToken cancellationToken) + => Task.FromResult(OpenRead(localPath)); + /// /// Returns the size of a file /// @@ -116,9 +163,8 @@ namespace Duplicati.Library.Snapshots /// The metadata for the given file or folder /// The file or folder to examine /// A flag indicating if the target is a symlink - /// A flag indicating if a symlink should be followed - public abstract Dictionary GetMetadata(string localPath, bool isSymlink, bool followSymlink); - + public abstract Dictionary GetMetadata(string localPath, bool isSymlink); + /// /// Gets a value indicating if the path points to a block device /// @@ -128,7 +174,7 @@ namespace Duplicati.Library.Snapshots { return false; } - + /// /// Gets a unique hardlink target ID /// @@ -157,15 +203,6 @@ namespace Duplicati.Library.Snapshots return Directory.Exists(ConvertToSnapshotPath(localFolderPath)); } - /// - public virtual bool PathExists(string localFileOrFolderPath) - { - return FileExists(localFileOrFolderPath) || DirectoryExists(localFileOrFolderPath); - } - - /// - public virtual bool IsSnapshot => false; - #endregion /// @@ -198,7 +235,7 @@ namespace Duplicati.Library.Snapshots // Now suppress finalization for this object, since we've already handled our resource cleanup tasks GC.SuppressFinalize(this); - } + } #endregion diff --git a/Duplicati/Library/Snapshots/SnapshotSourceFileEntry.cs b/Duplicati/Library/Snapshots/SnapshotSourceFileEntry.cs new file mode 100644 index 000000000..932963611 --- /dev/null +++ b/Duplicati/Library/Snapshots/SnapshotSourceFileEntry.cs @@ -0,0 +1,143 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Duplicati.Library.Interface; + +namespace Duplicati.Library.Snapshots; + + +/// +/// A snapshot-based source file entry +/// +/// The snapshot service +/// The path of the file +/// True if the entry is a folder, false otherwise +/// True if the entry is the root entry, false otherwise +public class SnapshotSourceFileEntry(ISnapshotService service, string path, bool isFolder, bool isRoot) : ISourceFileEntry +{ + /// + /// The symlink target + /// + private string? symlinkTarget; + /// + /// The file attributes + /// + private FileAttributes? attributes; + /// + /// Gets the symlink target or null if the entry is not a symlink + /// + public string? SymlinkTarget + { + get + { + if (symlinkTarget != null) + return symlinkTarget; + + if (!service.IsSymlink(path, Attributes)) + return null; + + return symlinkTarget = service.GetSymlinkTarget(path); + } + } + + /// + /// True if the entry is a symlink, false otherwise + /// + public bool IsSymlink => SymlinkTarget != null; + + /// + /// Gets the entry attributes + /// + public FileAttributes Attributes + { + get + { + if (attributes != null) + return attributes.Value; + + attributes = service.GetAttributes(path); + return attributes.Value; + } + } + + /// + /// Gets the minor metadata for the file + /// + public Dictionary MinorMetadata => service.GetMetadata(path, IsSymlink); + + /// + /// Gets a value indicating if the entry is a block device + /// + public bool IsBlockDevice => service.IsBlockDevice(path); + + /// + /// Gets a value indicating if the entry is a character device + /// + public bool IsCharacterDevice => false; + + /// + /// Gets a value indicating if the entry is an alternate stream + /// + public bool IsAlternateStream => false; + + /// + /// Gets the hardlink target ID, or null if the entry is not a hardlink + /// + public string? HardlinkTargetId => service.HardlinkTargetID(path); + + /// + public bool IsFolder => isFolder; + + /// + public bool IsMetaEntry => false; + /// + public bool IsRootEntry => isRoot; + + /// + public DateTime CreatedUtc => service.GetCreationTimeUtc(path); + + /// + public DateTime LastModificationUtc => service.GetLastWriteTimeUtc(path); + + /// + public string Path => path; + + /// + public long Size => IsFolder ? -1 : service.GetFileSize(path); + + /// + public Task OpenRead(CancellationToken cancellationToken) => service.OpenReadAsync(path, cancellationToken); + + /// + public Task OpenMetadataRead(CancellationToken cancellationToken) => Task.FromResult(new MemoryStream(System.Text.Json.JsonSerializer.SerializeToUtf8Bytes(MinorMetadata))); + /// + public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) => service.EnumerateFilesystemEntries(this).ToAsyncEnumerable(); + /// + public Task FileExists(string filename, CancellationToken cancellationToken) => Task.FromResult(service.FileExists(System.IO.Path.Combine(path, filename))); +} \ No newline at end of file diff --git a/Duplicati/Library/Snapshots/SnapshotUtility.cs b/Duplicati/Library/Snapshots/SnapshotUtility.cs index 7ea314b14..10e9d6c6e 100644 --- a/Duplicati/Library/Snapshots/SnapshotUtility.cs +++ b/Duplicati/Library/Snapshots/SnapshotUtility.cs @@ -1,30 +1,29 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; using System.IO; using System.Runtime.Versioning; -using Duplicati.Library.Common; using Duplicati.Library.Common.IO; namespace Duplicati.Library.Snapshots @@ -37,24 +36,42 @@ namespace Duplicati.Library.Snapshots /// /// Loads a snapshot implementation for the current OS /// - /// The list of folders to create snapshots of + /// The list of paths to create snapshots of /// A set of commandline options + /// Whether to follow symlinks /// The ISnapshotService implementation - public static ISnapshotService CreateSnapshot(IEnumerable folders, Dictionary options) + public static ISnapshotService CreateSnapshot(IEnumerable paths, Dictionary options, bool followSymlinks) { - if (OperatingSystem.IsMacOS() || OperatingSystem.IsLinux()) + if (OperatingSystem.IsLinux()) { - return CreateLinuxSnapshot(folders); + return CreateLinuxSnapshot(paths, followSymlinks); } else if (OperatingSystem.IsWindows()) { - return CreateWindowsSnapshot(folders, options); + return CreateWindowsSnapshot(paths, options, followSymlinks); } else { throw new NotSupportedException("Unsupported Operating System"); } - + } + + /// + /// Creates a snapshot implementation that does not use snapshots (i.e., regular file access) + /// + /// The list of paths to create snapshots of + /// Flag to ignore advisory locking + /// Whether to follow symlinks + /// The ISnapshotService implementation + public static ISnapshotService CreateNoSnapshot(IEnumerable paths, bool ignoreAdvisoryLocking, bool followSymlinks) + { + if (OperatingSystem.IsMacOS() || OperatingSystem.IsLinux()) + return new NoSnapshotLinux(paths, ignoreAdvisoryLocking, followSymlinks); + else if (OperatingSystem.IsWindows()) + return new NoSnapshotWindows(paths, followSymlinks); + else + throw new NotSupportedException("Unsupported Operating System"); + } // The two loader methods below guard against the type system attempting to load types @@ -65,13 +82,14 @@ namespace Duplicati.Library.Snapshots /// Loads a snapshot implementation for Linux /// /// The list of folders to create snapshots of + /// Whether to follow symlinks /// The ISnapshotService implementation [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] [SupportedOSPlatform("linux")] [SupportedOSPlatform("macOS")] - private static ISnapshotService CreateLinuxSnapshot(IEnumerable folders) + private static ISnapshotService CreateLinuxSnapshot(IEnumerable folders, bool followSymlinks) { - return new LinuxSnapshot(folders); + return new LinuxSnapshot(folders, followSymlinks); } /// @@ -79,23 +97,13 @@ namespace Duplicati.Library.Snapshots /// /// The list of folders to create snapshots of /// A set of commandline options + /// Whether to follow symlinks /// The ISnapshotService implementation [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] [SupportedOSPlatform("windows")] - private static ISnapshotService CreateWindowsSnapshot(IEnumerable folders, Dictionary options) + private static ISnapshotService CreateWindowsSnapshot(IEnumerable folders, Dictionary options, bool followSymlinks) { - return new WindowsSnapshot(folders, options); - } - - /// - /// Extension method for ISnapshotService which determines whether the given path is a symlink. - /// - /// ISnapshotService implementation - /// File or folder path - /// Whether the path is a symlink - public static bool IsSymlink(this ISnapshotService snapshot, string path) - { - return snapshot.IsSymlink(path, snapshot.GetAttributes(path)); + return new WindowsSnapshot(folders, options, followSymlinks); } /// diff --git a/Duplicati/Library/Snapshots/UsnJournalService.cs b/Duplicati/Library/Snapshots/UsnJournalService.cs index 419dc802e..47cd08baa 100644 --- a/Duplicati/Library/Snapshots/UsnJournalService.cs +++ b/Duplicati/Library/Snapshots/UsnJournalService.cs @@ -40,25 +40,31 @@ namespace Duplicati.Library.Snapshots /// private static readonly string FILTER_LOGTAG = Logging.Log.LogTagFromType(typeof(UsnJournalService)); + /// + /// The snapshot service + /// private readonly ISnapshotService m_snapshot; - private readonly IEnumerable m_sources; + /// + /// The volume data dictionary + /// private readonly Dictionary m_volumeDataDict; + /// + /// The cancellation token + /// private readonly CancellationToken m_token; /// /// Constructor. /// - /// Sources to filter /// /// Emit filter /// /// /// Journal-data of previous fileset /// - public UsnJournalService(IEnumerable sources, ISnapshotService snapshot, IFilter emitFilter, FileAttributes fileAttributeFilter, + public UsnJournalService(ISnapshotService snapshot, IFilter emitFilter, FileAttributes fileAttributeFilter, long skipFilesLargerThan, IEnumerable prevJournalData, CancellationToken token) { - m_sources = sources; m_snapshot = snapshot; m_volumeDataDict = Initialize(emitFilter, fileAttributeFilter, skipFilesLargerThan, prevJournalData); m_token = token; @@ -85,7 +91,7 @@ namespace Duplicati.Library.Snapshots // get hash identifying current source filter / sources configuration var configHash = Utility.Utility.ByteArrayAsHexString(MD5HashHelper.GetHash(new string[] { emitFilter == null ? string.Empty : emitFilter.ToString(), - string.Join("; ", m_sources), + string.Join("; ", m_snapshot.SourceFolders), fileAttributeFilter.ToString(), skipFilesLargerThan.ToString() })); @@ -94,7 +100,7 @@ namespace Duplicati.Library.Snapshots var journalDataDict = prevJournalData.ToDictionary(data => data.Volume); // iterate over volumes - foreach (var sourcesPerVolume in SortByVolume(m_sources)) + foreach (var sourcesPerVolume in SortByVolume(m_snapshot.SourceFolders)) { Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "UsnInitialize", "Reading USN journal for volume: {0}", sourcesPerVolume.Key); @@ -180,8 +186,7 @@ namespace Duplicati.Library.Snapshots // Since that folder is going to be fully scanned, those files can be removed. // Note: it would be wrong to use the result from step 2. as the folder list! The entries removed // between 1. and 2. are *excluded* folders, and files below them are to be *excluded*, too. - volumeData.Files = - new HashSet(Utility.Utility.GetFilesNotInFolders(changedFiles, volumeData.Folders)); + volumeData.Files = [.. Utility.Utility.GetFilesNotInFolders(changedFiles, volumeData.Folders)]; // Record success for volume volumeData.IsFullScan = false; @@ -213,216 +218,11 @@ namespace Duplicati.Library.Snapshots return result; } - /// - /// Filters sources, returning sub-set having been modified since last - /// change, as specified by journalData. - /// - /// Filter callback to exclude filtered items - /// Filtered sources - public IEnumerable GetModifiedSources(Utility.Utility.EnumerationFilterDelegate filter) - { - // iterate over volumes - foreach (var volumeData in m_volumeDataDict) - { - // prepare cache for includes (value = true) and excludes (value = false, will be populated - // on-demand) - var cache = new Dictionary(); - foreach (var source in m_sources) - { - if (m_token.IsCancellationRequested) - { - break; - } - - cache[source] = true; - } - - // Check the simplified folders, and their parent folders against the exclusion filter. - // This is needed because the filter may exclude "C:\A\", but this won't match the more - // specific "C:\A\B\" in our list, even though it's meant to be excluded. - // The reason why the filter doesn't exclude it is because during a regular (non-USN) full scan, - // FilterHandler.EnumerateFilesAndFolders() works top-down, and won't even enumerate child - // folders. - // The sources are needed to stop evaluating parent folders above the specified source folders - if (volumeData.Value.Folders != null) - { - foreach (var folder in FilterExcludedFolders(volumeData.Value.Folders, filter, cache).Where(m_snapshot.DirectoryExists)) - { - if (m_token.IsCancellationRequested) - { - break; - } - - yield return folder; - } - } - - // The simplified file list also needs to be checked against the exclusion filter, as it - // may contain entries excluded due to attributes, but also because they are below excluded - // folders, which themselves aren't in the folder list from step 1. - // Note that the simplified file list may contain entries that have been deleted! They need to - // be kept in the list (unless excluded by the filter) in order for the backup handler to record their - // deletion. - if (volumeData.Value.Files != null) - { - foreach (var files in FilterExcludedFiles(volumeData.Value.Files, filter, cache).Where(m_snapshot.FileExists)) - { - if (m_token.IsCancellationRequested) - { - break; - } - - yield return files; - } - } - } - } - - /// - /// Filter supplied files, removing any files which itself, or one - /// of its parent folders, is excluded by the filter. - /// - /// Files to filter - /// Exclusion filter - /// Cache of included and excluded files / folders - /// - /// Filtered files - private IEnumerable FilterExcludedFiles(IEnumerable files, - Utility.Utility.EnumerationFilterDelegate filter, IDictionary cache, Utility.Utility.ReportAccessError errorCallback = null) - { - var result = new List(); - - foreach (var file in files) - { - if (m_token.IsCancellationRequested) - { - break; - } - - var attr = m_snapshot.FileExists(file) ? m_snapshot.GetAttributes(file) : FileAttributes.Normal; - try - { - if (!filter(file, file, attr)) - continue; - - if (!IsFolderOrAncestorsExcluded(Utility.Utility.GetParent(file, true), filter, cache)) - { - result.Add(file); - } - } - catch (System.Threading.ThreadAbortException) - { - throw; - } - catch (Exception ex) - { - errorCallback?.Invoke(file, file, ex); - filter(file, file, attr | Utility.Utility.ATTRIBUTE_ERROR); - } - } - - return result; - } - - /// - /// Filter supplied folders, removing any folder which itself, or one - /// of its ancestors, is excluded by the filter. - /// - /// Folder to filter - /// Exclusion filter - /// Cache of excluded folders (optional) - /// - /// Filtered folders - private IEnumerable FilterExcludedFolders(IEnumerable folders, - Utility.Utility.EnumerationFilterDelegate filter, IDictionary cache, Utility.Utility.ReportAccessError errorCallback = null) - { - var result = new List(); - - foreach (var folder in folders) - { - if (m_token.IsCancellationRequested) - { - break; - } - - try - { - if (!IsFolderOrAncestorsExcluded(folder, filter, cache)) - { - result.Add(folder); - } - } - catch (System.Threading.ThreadAbortException) - { - throw; - } - catch (Exception ex) - { - errorCallback?.Invoke(folder, folder, ex); - filter(folder, folder, FileAttributes.Directory | Utility.Utility.ATTRIBUTE_ERROR); - } - } - - return result; - } - - /// - /// Tests if specified folder, or any of its ancestors, is excluded by the filter - /// - /// Folder to test - /// Filter - /// Cache of excluded folders (optional) - /// True if excluded, false otherwise - private bool IsFolderOrAncestorsExcluded(string folder, Utility.Utility.EnumerationFilterDelegate filter, IDictionary cache) - { - List parents = null; - while (folder != null) - { - if (m_token.IsCancellationRequested) - { - break; - } - - // first check cache - if (cache.TryGetValue(folder, out var include)) - { - if (include) - return false; - - break; // hit! - } - - // remember folder for cache - if (parents == null) - { - parents = new List(); // create on-demand - } - parents.Add(folder); - - - var attr = m_snapshot.DirectoryExists(folder) ? m_snapshot.GetAttributes(folder) : FileAttributes.Directory; - - if (!filter(folder, folder, attr)) - break; // excluded - - folder = Utility.Utility.GetParent(folder, true); - } - - if (folder != null) - { - // update cache - parents?.ForEach(p => cache[p] = false); - } - - return folder != null; - } - /// /// Sort sources by root volume /// /// List of sources /// Dictionary of volumes, with list of sources as values - [SupportedOSPlatform("windows")] private static Dictionary> SortByVolume(IEnumerable sources) { var sourcesByVolume = new Dictionary>(); @@ -448,7 +248,6 @@ namespace Duplicati.Library.Snapshots /// /// /// - [SupportedOSPlatform("windows")] public bool IsPathEnumerated(string path) { // get NTFS volume root diff --git a/Duplicati/Library/Snapshots/WindowsSnapshot.cs b/Duplicati/Library/Snapshots/WindowsSnapshot.cs index 65df03a2c..ae36027ec 100644 --- a/Duplicati/Library/Snapshots/WindowsSnapshot.cs +++ b/Duplicati/Library/Snapshots/WindowsSnapshot.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. @@ -26,6 +26,7 @@ using System.IO; using System.Linq; using System.Runtime.Versioning; using Duplicati.Library.Common.IO; +using Duplicati.Library.Interface; namespace Duplicati.Library.Snapshots { @@ -54,7 +55,9 @@ namespace Duplicati.Library.Snapshots /// /// Sources to determine which volumes to include in snapshot /// A set of commandline options - public WindowsSnapshot(IEnumerable sources, IDictionary options) + /// A flag indicating if symlinks should be followed + public WindowsSnapshot(IEnumerable sources, IDictionary options, bool followSymlinks) + : base(followSymlinks) { // For Windows, ensure we don't store paths with extended device path prefixes (i.e., @"\\?\" or @"\\?\UNC\") sources = sources.Select(SystemIOWindows.RemoveExtendedDevicePathPrefix); @@ -103,6 +106,26 @@ namespace Duplicati.Library.Snapshots } } + /// + /// Gets the source folders + /// + public override IEnumerable SourceFolders => _vssBackupComponents.SnapshotLocalPaths; + + /// + /// Enumerates the root source files and folders + /// + /// The source files and folders + public override IEnumerable EnumerateFilesystemEntries() + { + foreach (var folder in _vssBackupComponents.SnapshotLocalPaths) + { + if (folder.EndsWith(Path.DirectorySeparatorChar) || DirectoryExists(folder)) + yield return new SnapshotSourceFileEntry(this, folder, true, true); + else + yield return new SnapshotSourceFileEntry(this, folder, false, true); + } + } + #region Private functions /// @@ -158,18 +181,6 @@ namespace Duplicati.Library.Snapshots #region ISnapshotService Members - /// - /// Enumerates all files and folders in the snapshot, restricted to sources - /// - /// Sources to enumerate - /// The callback to invoke with each found path - /// The callback used to report errors - public override IEnumerable EnumerateFilesAndFolders(IEnumerable sources, Utility.Utility.EnumerationFilterDelegate callback, Utility.Utility.ReportAccessError errorCallback) - { - // For Windows, ensure we don't store paths with extended device path prefixes (i.e., @"\\?\" or @"\\?\UNC\") - return base.EnumerateFilesAndFolders(sources.Select(SystemIOWindows.RemoveExtendedDevicePathPrefix), callback, errorCallback); - } - /// /// Gets the last write time of a given file in UTC /// @@ -241,23 +252,16 @@ namespace Duplicati.Library.Snapshots /// The metadata for the given file or folder /// The file or folder to examine /// A flag indicating if the target is a symlink - /// A flag indicating if a symlink should be followed - public override Dictionary GetMetadata(string localPath, bool isSymlink, bool followSymlink) + public override Dictionary GetMetadata(string localPath, bool isSymlink) { - return SystemIO.IO_WIN.GetMetadata(ConvertToSnapshotPath(localPath), isSymlink, followSymlink); + return SystemIO.IO_WIN.GetMetadata(ConvertToSnapshotPath(localPath), isSymlink, FollowSymlinks); } /// - public override bool IsBlockDevice(string localPath) - { - return false; - } + public override bool IsBlockDevice(string localPath) => false; /// - public override string HardlinkTargetID(string localPath) - { - return null; - } + public override string HardlinkTargetID(string localPath) => null; /// public override string ConvertToLocalPath(string snapshotPath) @@ -312,9 +316,6 @@ namespace Duplicati.Library.Snapshots return SystemIO.IO_WIN.DirectoryExists(ConvertToSnapshotPath(localFolderPath)); } - /// - public override bool IsSnapshot => true; - #endregion /// diff --git a/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs new file mode 100644 index 000000000..54ad99034 --- /dev/null +++ b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs @@ -0,0 +1,211 @@ +// Copyright (C) 2024, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using Duplicati.Library.Interface; +using Duplicati.Library.Utility; + +namespace Duplicati.Library.SourceProvider; + +/// +/// A source provider that wraps a backend +/// +/// The backend to wrap +public class BackendSourceProvider : ISourceProvider +{ + /// + /// The wrapped backend + /// + public IBackend WrappedBackend { get; init; } + + /// + /// The path key unique for the destination + /// + public readonly string PathKey; + + /// + /// Creates a new backend source provider + /// + /// The backend to wrap + /// The URL of the backend + public BackendSourceProvider(IBackend backend, string url) + { + WrappedBackend = backend; + var uri = new Utility.Uri(url); + // TODO: The username may be provided in other ways to the backend... + PathKey = $"{uri.Scheme}://{uri.Host}:{uri.Port}/~{uri.Username}/{uri.Path}/"; + } + + /// + /// Creates a new backend source entry + /// + /// The parent backend + /// The path of the entry + /// True if the entry is a folder + /// True if the entry is a meta entry + /// The creation time of the entry + /// The last modification time of the entry + /// The size of the entry + private class BackendSourceFileEntry(BackendSourceProvider parent, string path, bool isFolder, bool isMetaEntry, DateTime createdUtc, DateTime lastModificationUtc, long size) + : ISourceFileEntry + { + /// + public bool IsFolder => isFolder; + + /// + public bool IsMetaEntry => isMetaEntry; + + /// + public bool IsRootEntry => isMetaEntry; + + /// + public DateTime CreatedUtc => createdUtc; + + /// + public DateTime LastModificationUtc => lastModificationUtc; + + /// + public string Path => $"{parent.PathKey}{path}"; + + /// + public long Size => size; + /// + public bool IsSymlink => false; + + /// + public string? SymlinkTarget => null; + + /// + public FileAttributes Attributes => IsFolder + ? FileAttributes.Directory + : FileAttributes.Normal; + + /// + public Dictionary MinorMetadata => new Dictionary(); + + /// + public bool IsBlockDevice => false; + + /// + public bool IsCharacterDevice => false; + + /// + public bool IsAlternateStream => false; + + /// + public string? HardlinkTargetId => null; + + /// + public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) + { + if (!isFolder) + throw new InvalidOperationException("Enumerate can only be called on folders"); + + if (parent.WrappedBackend is IFolderEnabledBackend folderEnabledBackend) + { + return folderEnabledBackend.ListAsync(path, cancellationToken) + .Select(x => new BackendSourceFileEntry(parent, x.Path, x.IsFolder, false, new DateTime(0), x.LastModificationUtc, x.Size)); + } + else + { + + // If we do not support folders, we can only list the root + if (this.IsMetaEntry) + return parent.WrappedBackend.List() + .Select(x => new BackendSourceFileEntry(parent, x.Name, x.IsFolder, false, new DateTime(0), x.LastModification, x.Size)) + .ToAsyncEnumerable(); + + return Array.Empty().ToAsyncEnumerable(); + } + } + + /// + public async Task FileExists(string path, CancellationToken cancellationToken) + { + if (!isFolder) + throw new InvalidOperationException("FileExists can only be called on folders"); + + if (parent.WrappedBackend is IFolderEnabledBackend folderEnabledBackend) + { + try + { + var entry = await folderEnabledBackend.GetEntryAsync(path, cancellationToken); + return entry != null && !entry.IsFolder; + } + catch (FileNotFoundException) + { + return false; + } + } + + return false; + } + + /// + public Task OpenMetadataRead(CancellationToken cancellationToken) + => Task.FromResult(null); + + /// + public async Task OpenRead(CancellationToken cancellationToken) + { + if (isFolder) + throw new InvalidOperationException("OpenRead can only be called on files"); + + if (parent.WrappedBackend is IStreamingBackend streamingBackend) + { + var mapStream = new MapStream(); + mapStream.CopyTask = streamingBackend.GetAsync(this.Path, mapStream, cancellationToken); + return mapStream; + } + else + { + TempFileStream? file = null; + try + { + file = TempFileStream.Create(); + await parent.WrappedBackend.GetAsync(this.Path, file.Path, cancellationToken); + file.Position = 0; + return file; + } + catch + { + file?.Dispose(); + throw; + } + } + } + } + + /// + public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) + => new[] { new BackendSourceFileEntry(this, "", true, true, new DateTime(0), new DateTime(0), 0) }.ToAsyncEnumerable(); + + /// + public Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) + // TODO: If we cache the enumerate result, we could check if the path exists in the cache and return it + => Task.FromResult(new BackendSourceFileEntry(this, path, isFolder, false, new DateTime(0), new DateTime(0), -1)); + + /// + public void Dispose() + { + WrappedBackend.Dispose(); + GC.SuppressFinalize(this); + } +} diff --git a/Duplicati/Library/SourceProvider/Builtin/Combiner.cs b/Duplicati/Library/SourceProvider/Builtin/Combiner.cs new file mode 100644 index 000000000..5ef6496ba --- /dev/null +++ b/Duplicati/Library/SourceProvider/Builtin/Combiner.cs @@ -0,0 +1,82 @@ +// Copyright (C) 2024, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using System.Runtime.CompilerServices; +using Duplicati.Library.Interface; + +namespace Duplicati.Library.SourceProvider; + +/// +/// Helper class that wraps multiple source providers into a single source provider +/// +public class Combiner(IEnumerable providers) : ISourceProvider +{ + /// + /// The providers to combine + /// + private readonly List providers = providers.SelectMany(x => x is Combiner c ? c.providers.AsEnumerable() : [x]).ToList(); + + /// + public async IAsyncEnumerable Enumerate([EnumeratorCancellation] CancellationToken cancellationToken) + { + foreach (var provider in providers) + { + await foreach (var entry in provider.Enumerate(cancellationToken).ConfigureAwait(false)) + yield return entry; + } + } + + /// + /// Provides access to the combined providers + /// + public IEnumerable Providers => providers; + + /// + /// Gets a filesystem entry for a given path + /// + /// The path to get + /// True if the path is a folder + /// The cancellation token + /// The filesystem entry + public Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) + { + // TODO: Map roots to the paths so we get the right provider + throw new NotImplementedException(); + } + + /// + /// Creates a combined source provider from a list of providers + /// + /// The providers to combine + /// The combined provider + public static ISourceProvider Combine(IEnumerable providers) + { + var providerTotal = providers.SelectMany(x => x is Combiner c ? c.providers.AsEnumerable() : [x]).ToList(); + return providerTotal.Count == 1 ? providerTotal[0] : new Combiner(providers); + } + + /// + public void Dispose() + { + foreach (var provider in providers) + provider.Dispose(); + } +} diff --git a/Duplicati/Library/SourceProvider/Builtin/Duplicati.Library.SourceProvider.Builtin.csproj b/Duplicati/Library/SourceProvider/Builtin/Duplicati.Library.SourceProvider.Builtin.csproj new file mode 100644 index 000000000..4ba5377b7 --- /dev/null +++ b/Duplicati/Library/SourceProvider/Builtin/Duplicati.Library.SourceProvider.Builtin.csproj @@ -0,0 +1,27 @@ + + + + net8.0 + Library + Duplicati.Library.SourceProvider + Copyright © 2024 Team Duplicati, MIT license + enable + enable + Duplicati.Library.SourceProvider + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + diff --git a/Duplicati/Library/SourceProvider/Builtin/File.cs b/Duplicati/Library/SourceProvider/Builtin/File.cs new file mode 100644 index 000000000..82e43997a --- /dev/null +++ b/Duplicati/Library/SourceProvider/Builtin/File.cs @@ -0,0 +1,52 @@ +// Copyright (C) 2024, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using Duplicati.Library.Interface; +using Duplicati.Library.Snapshots; + +namespace Duplicati.Library.SourceProvider; + +/// +/// A source provider that wraps a file system through a snapshot service +/// +/// The snapshot service to use +public class File(ISnapshotService snapshotService) : ISourceProvider +{ + /// + public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) + => snapshotService.EnumerateFilesystemEntries().ToAsyncEnumerable(); + + /// + /// The snapshot service being used + /// + public ISnapshotService SnapshotService => snapshotService; + + /// + public Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) + => Task.FromResult(snapshotService.GetFilesystemEntry(path, isFolder)); + + /// + public void Dispose() + { + snapshotService.Dispose(); + GC.SuppressFinalize(this); + } +} diff --git a/Duplicati/Library/SourceProvider/Builtin/Strings.cs b/Duplicati/Library/SourceProvider/Builtin/Strings.cs new file mode 100644 index 000000000..5707fccd5 --- /dev/null +++ b/Duplicati/Library/SourceProvider/Builtin/Strings.cs @@ -0,0 +1,29 @@ +// Copyright (C) 2024, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +using Duplicati.Library.Localization.Short; +namespace Duplicati.Library.SourceProvider.Strings +{ + internal static class File + { + public static string Description { get { return LC.L(@"This backend can read and write data to an file based backend. Allowed formats are ""file://hostname/folder"" and ""file://username:password@hostname/folder"". You may supply UNC paths (e.g.: ""file://\\server\folder"") or local paths (e.g.: (win) ""file://c:\folder"", (linux) ""file:///usr/pub/files""). Use the prefix ""vss://"" or ""lvm://"" to create snapshot based folders."); } } + public static string DisplayName { get { return LC.L(@"Local folder or drive"); } } + } +} diff --git a/Duplicati/Library/SourceProviders/Duplicati.Library.SourceProviders.csproj b/Duplicati/Library/SourceProviders/Duplicati.Library.SourceProviders.csproj new file mode 100644 index 000000000..64dae938d --- /dev/null +++ b/Duplicati/Library/SourceProviders/Duplicati.Library.SourceProviders.csproj @@ -0,0 +1,14 @@ + + + net8.0 + Central reference to all source providers + Duplicati.Library.Backend + Copyright © 2024 Team Duplicati, MIT license + enable + enable + + + + + + diff --git a/Duplicati/Library/SourceProviders/SourceProviderModules.cs b/Duplicati/Library/SourceProviders/SourceProviderModules.cs new file mode 100644 index 000000000..1bf09d104 --- /dev/null +++ b/Duplicati/Library/SourceProviders/SourceProviderModules.cs @@ -0,0 +1,45 @@ +// Copyright (C) 2024, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using Duplicati.Library.Interface; + +namespace Duplicati.Library.SourceProviders; + +/// +/// Shared list of statically linked source-provider modules +/// +public static class SourceProviderModules +{ + /// + /// The list of all built-in source-provider modules + /// + public static IReadOnlyList BuiltInSourceProviderModules => SupportedSourceProviders; + + /// + /// Calculate list once and cache it + /// + private static readonly IReadOnlyList SupportedSourceProviders = new ISourceProviderModule[] { + // ... No providers currently implemented + } + .Where(x => x != null) + .ToList(); +} + diff --git a/Duplicati/Library/Utility/MapStream.cs b/Duplicati/Library/Utility/MapStream.cs new file mode 100644 index 000000000..b2c717739 --- /dev/null +++ b/Duplicati/Library/Utility/MapStream.cs @@ -0,0 +1,176 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +using System; +using System.IO; +using System.Threading; +using System.Threading.Channels; +using System.Threading.Tasks; + +namespace Duplicati.Library.Utility; + +/// +/// A stream that maps data with an internal buffer. +/// This allows a reader and a writer to share a stream without blocking. +/// +public class MapStream : Stream +{ + /// + /// The channel used to communicate between the reader and writer. + /// + private readonly Channel channel = Channel.CreateBounded(0); + + /// + /// The buffer used to store data that has been written but not consumed by the reader. + /// + private Memory readBuffer = Array.Empty(); + + /// + /// The task that is using the map stream. + /// + public Task? CopyTask { get; set; } + + /// + public override bool CanRead => true; + + /// + public override bool CanSeek => false; + + /// + public override bool CanWrite => true; + + /// + public override long Length => throw new NotImplementedException(); + + /// + public override long Position { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } + + /// + public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + int read = 0; + // If we have left-over data from the previous read, return it + if (readBuffer.Length > 0) + { + // Return as much as possible + var requested = Math.Min(readBuffer.Length, count); + readBuffer.Span.Slice(0, requested).CopyTo(buffer.AsSpan(offset, count)); + // Adjust the remaining buffer + if (readBuffer.Length > requested) + readBuffer = readBuffer.Slice(requested); + else + readBuffer = Array.Empty(); + + // Adjust the counters + offset += requested; + count -= requested; + read += requested; + + // Return if we have read all that was requested + if (count == 0) + return read; + } + + // Handle external task + var ct = CopyTask; + + // If the task is completed, clear it + if (ct != null && ct.IsCompletedSuccessfully) + CopyTask = ct = null; + // If the task failed, stop reading + else if (ct != null && (ct.IsCanceled || ct.IsFaulted)) + { + // If we have any data in the buffer, return it + if (read > 0) + return read; + + // Otherwise, throw the exception + await ct; + } + + if (!await channel.Reader.WaitToReadAsync(cancellationToken)) + { + // If we have any data in the buffer, return it + if (read > 0) + return read; + + // If the copy crashed, throw the exception + ct = CopyTask; + if (ct != null && (ct.IsCanceled || ct.IsFaulted)) + await ct; + + // Otherwise, return 0 + return read; + } + + // Read the data from the channel + var res = await channel.Reader.ReadAsync(cancellationToken); + + // Consume as much as was requested + var consumed = Math.Min(res.Length, count); + + // Keep the rest in the buffer + if (res.Length > count) + readBuffer = res.AsMemory(count); + + res.AsSpan(0, consumed).CopyTo(buffer.AsSpan(offset, count)); + read += consumed; + return read; + } + + /// + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + var ct = CopyTask; + // Throw if the task failed + if (ct != null && (ct.IsCanceled || ct.IsFaulted)) + await ct; + await channel.Writer.WriteAsync(buffer.AsSpan(offset, count).ToArray(), cancellationToken); + } + + public override void Close() + { + base.Close(); + channel.Writer.Complete(); + } + + protected override void Dispose(bool disposing) + { + channel.Writer.Complete(); + base.Dispose(disposing); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) + => ReadAsync(buffer, offset, count, CancellationToken.None).Await(); + + public override long Seek(long offset, SeekOrigin origin) + => throw new NotImplementedException(); + + public override void SetLength(long value) + => throw new NotImplementedException(); + + public override void Write(byte[] buffer, int offset, int count) + => WriteAsync(buffer, offset, count, CancellationToken.None).Await(); +} diff --git a/Duplicati/Library/Utility/TempFileStream.cs b/Duplicati/Library/Utility/TempFileStream.cs new file mode 100644 index 000000000..104eab261 --- /dev/null +++ b/Duplicati/Library/Utility/TempFileStream.cs @@ -0,0 +1,87 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace Duplicati.Library.Utility; + +/// +/// A stream that wraps a temporary file. +/// The temporary file is automatically deleted when the stream is disposed. +/// +public class TempFileStream : StreamUtil.WrappingAsyncStream +{ + /// + /// The temporary file instance. + /// + private readonly TempFile tempFile; + /// + /// The stream for the temporary file. + /// + private readonly Stream stream; + + /// + /// The path to the temporary file. + /// + public string Path => tempFile.Name; + + /// + /// Creates a new TempFileStream instance. + /// + /// The temporary file instance. + /// The stream for the temporary file. + private TempFileStream(TempFile tempFile, Stream stream) + : base(stream) + { + this.tempFile = tempFile; + this.stream = stream; + } + + /// + /// Creates a new TempFileStream instance. + /// + /// The new TempFileStream instance. + public static TempFileStream Create() + { + var tempFile = new TempFile(); + return new TempFileStream(tempFile, File.Open(tempFile.Name, FileMode.Open, FileAccess.ReadWrite, FileShare.None)); + } + + /// + protected override Task ReadImplAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + => stream.ReadAsync(buffer, offset, count, cancellationToken); + + /// + protected override Task WriteImplAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + => stream.WriteAsync(buffer, offset, count, cancellationToken); + + /// + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + stream.Dispose(); + tempFile.Dispose(); + } +} diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index 4d2d8e423..c764f3165 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -246,7 +246,7 @@ namespace Duplicati.Library.Utility /// A list of the full filenames and foldernames. Foldernames ends with the directoryseparator char public static IEnumerable EnumerateFileSystemEntries(string basepath) { - return EnumerateFileSystemEntries(basepath, (rootpath, path, attributes) => true, SystemIO.IO_OS.GetDirectories, Directory.GetFiles, null); + return EnumerateFileSystemEntries(basepath, SystemIO.IO_OS.GetDirectories, Directory.GetFiles, null); } /// @@ -276,13 +276,12 @@ namespace Duplicati.Library.Utility /// The search is recursive. /// /// The folder to look in - /// The function to call with the filenames /// A function to call that lists all folders in the supplied folder /// A function to call that lists all files in the supplied folder /// A function to call that obtains the attributes for an element, set to null to avoid reading attributes /// An optional function to call with error messages. /// A list of the full filenames - public static IEnumerable EnumerateFileSystemEntries(string rootpath, EnumerationFilterDelegate callback, FileSystemInteraction folderList, FileSystemInteraction fileList, ExtractFileAttributes? attributeReader, ReportAccessError? errorCallback = null) + public static IEnumerable EnumerateFileSystemEntries(string rootpath, FileSystemInteraction folderList, FileSystemInteraction fileList, ExtractFileAttributes? attributeReader, ReportAccessError? errorCallback = null) { var lst = new Stack(); @@ -292,8 +291,7 @@ namespace Duplicati.Library.Utility try { var attr = attributeReader?.Invoke(rootpath) ?? FileAttributes.Directory; - if (callback(rootpath, rootpath, attr)) - lst.Push(rootpath); + lst.Push(rootpath); } catch (System.Threading.ThreadAbortException) { @@ -302,7 +300,6 @@ namespace Duplicati.Library.Utility catch (Exception ex) { errorCallback?.Invoke(rootpath, rootpath, ex); - callback(rootpath, rootpath, FileAttributes.Directory | ATTRIBUTE_ERROR); } while (lst.Count > 0) @@ -319,8 +316,7 @@ namespace Duplicati.Library.Utility try { var attr = attributeReader?.Invoke(sf) ?? FileAttributes.Directory; - if (callback(rootpath, sf, attr)) - lst.Push(sf); + lst.Push(sf); } catch (System.Threading.ThreadAbortException) { @@ -329,7 +325,6 @@ namespace Duplicati.Library.Utility catch (Exception ex) { errorCallback?.Invoke(rootpath, sf, ex); - callback(rootpath, sf, FileAttributes.Directory | ATTRIBUTE_ERROR); } } } @@ -340,7 +335,6 @@ namespace Duplicati.Library.Utility catch (Exception ex) { errorCallback?.Invoke(rootpath, f, ex); - callback(rootpath, f, FileAttributes.Directory | ATTRIBUTE_ERROR); } string[]? files = null; @@ -357,54 +351,18 @@ namespace Duplicati.Library.Utility catch (Exception ex) { errorCallback?.Invoke(rootpath, f, ex); - callback(rootpath, f, FileAttributes.Directory | ATTRIBUTE_ERROR); } } if (files != null) { foreach (var s in files) - { - try - { - var attr = attributeReader?.Invoke(s) ?? FileAttributes.Normal; - if (!callback(rootpath, s, attr)) - continue; - } - catch (System.Threading.ThreadAbortException) - { - throw; - } - catch (Exception ex) - { - errorCallback?.Invoke(rootpath, s, ex); - callback(rootpath, s, ATTRIBUTE_ERROR); - continue; - } yield return s; - } } } } else { - try - { - var attr = attributeReader?.Invoke(rootpath) ?? FileAttributes.Normal; - if (!callback(rootpath, rootpath, attr)) - yield break; - } - catch (System.Threading.ThreadAbortException) - { - throw; - } - catch (Exception ex) - { - errorCallback?.Invoke(rootpath, rootpath, ex); - callback(rootpath, rootpath, ATTRIBUTE_ERROR); - yield break; - } - yield return rootpath; } } @@ -1591,5 +1549,20 @@ namespace Duplicati.Library.Utility foreach (var iex in FlattenException(ex.InnerException)) yield return iex; } + + /// + /// Guesses the URL scheme and returns it + /// + /// The URL to guess the scheme for + /// The guessed scheme, or null if no scheme was found + public static string? GuessScheme(string url) + { + var idx = url.IndexOf("://"); + if (idx < 0 && idx < 15 && idx + "://".Length < url.Length) + return null; + + return url.Substring(0, idx); + + } } } diff --git a/Duplicati/WebserverCore/Endpoints/V1/Filesystem.cs b/Duplicati/WebserverCore/Endpoints/V1/Filesystem.cs index 9a7625a0d..bb1e9509d 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/Filesystem.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/Filesystem.cs @@ -254,11 +254,11 @@ public class Filesystem : IEndpointV1 { var attr = SystemIO.IO_OS.GetFileAttributes(s); var isSymlink = SystemIO.IO_OS.IsSymlink(s, attr); - var isFolder = (attr & FileAttributes.Directory) != 0; + var isFolder = attr.HasFlag(FileAttributes.Directory); var isFile = !isFolder; - var isHidden = (attr & FileAttributes.Hidden) != 0; - bool isSystem = (attr & FileAttributes.System) != 0; - bool isTemporary = (attr & FileAttributes.Temporary) != 0; + var isHidden = attr.HasFlag(FileAttributes.Hidden); + bool isSystem = attr.HasFlag(FileAttributes.System); + bool isTemporary = attr.HasFlag(FileAttributes.Temporary); long fileSize = -1; var accessible = isFile || canAccess(s); From 68d11a2a8bdb1832550dbc0aa98928e844d7af80 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 10 Feb 2025 10:22:16 +0100 Subject: [PATCH 141/362] Improved support for disabled signin tokens and trayicon with a hosted server --- .../Duplicati.GUI.TrayIcon/HttpServerConnection.cs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/HttpServerConnection.cs b/Duplicati/GUI/Duplicati.GUI.TrayIcon/HttpServerConnection.cs index 6a489ac09..481744afc 100644 --- a/Duplicati/GUI/Duplicati.GUI.TrayIcon/HttpServerConnection.cs +++ b/Duplicati/GUI/Duplicati.GUI.TrayIcon/HttpServerConnection.cs @@ -345,16 +345,27 @@ namespace Duplicati.GUI.TrayIcon private async Task ObtainAccessTokenAsync() { + // If we host the server, issue the access token from the service + if (FIXMEGlobal.IsServerStarted && m_passwordSource == Program.PasswordSource.HostedServer) + { + var provider = FIXMEGlobal.Provider.GetRequiredService(); + m_accesstoken = provider.CreateAccessToken("trayicon", provider.TemporaryFamilyId); + return; + } + + // If we know the password, issue a token from the API if (!string.IsNullOrWhiteSpace(m_password)) { m_accesstoken = (await PerformRequestInternalAsync("POST", "/auth/login", JsonSerializer.Serialize(new { Password = m_password }), null).ConfigureAwait(false)).AccessToken; return; } + // Otherwise, try to get a signin token var token = await ObtainSignInTokenAsync().ConfigureAwait(false); if (string.IsNullOrWhiteSpace(token)) return; + // Use the signin token to get an access token m_accesstoken = (await PerformRequestInternalAsync("POST", "/auth/signin", JsonSerializer.Serialize(new { SigninToken = token }), null).ConfigureAwait(false)).AccessToken; } From dbd3520d9c7cbf53e7c0e80b7fce034260dbdc20 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 10 Feb 2025 12:18:16 +0100 Subject: [PATCH 142/362] Formatting --- .../Main/Operation/Restore/FileProcessor.cs | 73 ++++++++++--------- 1 file changed, 37 insertions(+), 36 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs index 8b61eea9c..b92c186e2 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -66,12 +66,12 @@ namespace Duplicati.Library.Main.Operation.Restore }, async self => { - Stopwatch sw_file = options.InternalProfiling ? new () : null; - Stopwatch sw_block = options.InternalProfiling ? new () : null; - Stopwatch sw_meta = options.InternalProfiling ? new () : null; - Stopwatch sw_req = options.InternalProfiling ? new () : null; - Stopwatch sw_resp = options.InternalProfiling ? new () : null; - Stopwatch sw_work = options.InternalProfiling ? new () : null; + Stopwatch sw_file = options.InternalProfiling ? new() : null; + Stopwatch sw_block = options.InternalProfiling ? new() : null; + Stopwatch sw_meta = options.InternalProfiling ? new() : null; + Stopwatch sw_req = options.InternalProfiling ? new() : null; + Stopwatch sw_resp = options.InternalProfiling ? new() : null; + Stopwatch sw_work = options.InternalProfiling ? new() : null; try { @@ -206,7 +206,7 @@ namespace Duplicati.Library.Main.Operation.Restore // Burst the block requests to speed up the restore int burst = Channels.BufferSize; int j = 0; - for (int i = 0; i < (int) Math.Min(missing_blocks.Count, burst); i++) + for (int i = 0; i < (int)Math.Min(missing_blocks.Count, burst); i++) { await block_request.WriteAsync(missing_blocks[i]); } @@ -303,7 +303,8 @@ namespace Duplicati.Library.Main.Operation.Restore } } - if (!options.SkipMetadata) { + if (!options.SkipMetadata) + { empty_file_or_symlink |= await RestoreMetadata(db, file, block_request, block_response, options, sw_meta, sw_work, sw_req, sw_resp); } @@ -476,7 +477,7 @@ namespace Duplicati.Library.Main.Operation.Restore /// The restoration results. /// The channel to request blocks from the block manager. Used to inform the block manager which blocks are already present. /// An awaitable `Task`, which returns a collection of data blocks that are missing. - private static async Task<(long,List,List)> VerifyTargetBlocks(FileRequest file, BlockRequest[] blocks, System.Security.Cryptography.HashAlgorithm filehasher, System.Security.Cryptography.HashAlgorithm blockhasher, Options options, RestoreResults results, IChannel block_request) + private static async Task<(long, List, List)> VerifyTargetBlocks(FileRequest file, BlockRequest[] blocks, System.Security.Cryptography.HashAlgorithm filehasher, System.Security.Cryptography.HashAlgorithm blockhasher, Options options, RestoreResults results, IChannel block_request) { long bytes_read = 0; List missing_blocks = []; @@ -541,7 +542,7 @@ namespace Duplicati.Library.Main.Operation.Restore if (Convert.ToBase64String(filehasher.Hash) == file.Hash) { // Truncate the file if it is larger than the expected size. - FileInfo fi = new (file.TargetPath); + FileInfo fi = new(file.TargetPath); if (file.Length < fi.Length) { if (options.Dryrun) @@ -611,7 +612,7 @@ namespace Duplicati.Library.Main.Operation.Restore try { f_original.Seek(i * options.Blocksize, SeekOrigin.Begin); - read = await f_original.ReadAsync(buffer, 0, (int) blocks[j].BlockSize); + read = await f_original.ReadAsync(buffer, 0, (int)blocks[j].BlockSize); } catch (Exception) { @@ -632,19 +633,19 @@ namespace Duplicati.Library.Main.Operation.Restore else { if (!options.Dryrun) - { - try { - f_target.Seek(blocks[j].BlockOffset * options.Blocksize, SeekOrigin.Begin); - await f_target.WriteAsync(buffer, 0, read); - } - catch (Exception) - { - lock (results) + try { - results.BrokenLocalFiles.Add(file.TargetPath); + f_target.Seek(blocks[j].BlockOffset * options.Blocksize, SeekOrigin.Begin); + await f_target.WriteAsync(buffer, 0, read); } - throw; + catch (Exception) + { + lock (results) + { + results.BrokenLocalFiles.Add(file.TargetPath); + } + throw; } } bytes_read += read; From 6e2f8156f810b6027ea1d62d9373b590ad1a2959 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 10 Feb 2025 12:33:45 +0100 Subject: [PATCH 143/362] VerifyLocalBlocks should only perform checks when the target differs from the original. --- Duplicati/Library/Main/Operation/Restore/FileProcessor.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs index b92c186e2..323806665 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs @@ -583,6 +583,12 @@ namespace Duplicati.Library.Main.Operation.Restore /// An awaitable `Task`, which returns a collection of data blocks that are missing. private static async Task<(long, List)> VerifyLocalBlocks(FileRequest file, List blocks, long total_blocks, System.Security.Cryptography.HashAlgorithm filehasher, System.Security.Cryptography.HashAlgorithm blockhasher, Options options, RestoreResults results, IChannel block_request) { + if (file.TargetPath == file.OriginalPath) + { + // The original file is the same as the target file, so no new blocks can be used. + return (0, blocks); + } + List missing_blocks = []; List verified_blocks = []; From 259322f0d92ad3b51cfce68a05b26e04a36f275e Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 10 Feb 2025 12:34:16 +0100 Subject: [PATCH 144/362] CopyOldTargetBlocksToNewTarget was made async --- Duplicati/Library/Main/Operation/Restore/FileProcessor.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs index 323806665..797e7243f 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs @@ -124,7 +124,7 @@ namespace Duplicati.Library.Main.Operation.Restore } else { - CopyOldTargetBlocksToNewTarget(file, new_file, verified_blocks); + await CopyOldTargetBlocksToNewTarget(file, new_file, verified_blocks); } } file = new_file; @@ -351,7 +351,7 @@ namespace Duplicati.Library.Main.Operation.Restore /// The old target file. /// The new target file. /// The blocks in the old file that were verified. - private static void CopyOldTargetBlocksToNewTarget(FileRequest old_file, FileRequest new_file, List verified_blocks) + private static async Task CopyOldTargetBlocksToNewTarget(FileRequest old_file, FileRequest new_file, List verified_blocks) { using var fs_old = SystemIO.IO_OS.FileOpenRead(old_file.TargetPath); using var fs_new = SystemIO.IO_OS.FileOpenWrite(new_file.TargetPath); @@ -362,8 +362,8 @@ namespace Duplicati.Library.Main.Operation.Restore fs_new.Seek(block.BlockOffset * block.BlockSize, SeekOrigin.Begin); var buffer = new byte[block.BlockSize]; - fs_old.Read(buffer, 0, buffer.Length); - fs_new.Write(buffer, 0, buffer.Length); + await fs_old.ReadAsync(buffer, 0, buffer.Length); + await fs_new.WriteAsync(buffer, 0, buffer.Length); } } From 3b9fc188e118958ad7bfe3918cecceec671a3079 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 10 Feb 2025 12:34:44 +0100 Subject: [PATCH 145/362] Retire block request and response on exception in FileProcessor --- Duplicati/Library/Main/Operation/Restore/FileProcessor.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs index 797e7243f..0c3a22a15 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs @@ -295,6 +295,8 @@ namespace Duplicati.Library.Main.Operation.Restore { results.BrokenLocalFiles.Add(file.TargetPath); } + block_request.Retire(); + block_response.Retire(); throw; } finally From 0b3ff33716ec9f5142b00d047b6aa2ae0db11f2a Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 10 Feb 2025 12:38:24 +0100 Subject: [PATCH 146/362] Moved retire on exception to final, so regardless of how the FileProcessor exits, it retires the channels. --- Duplicati/Library/Main/Operation/Restore/FileProcessor.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs index 0c3a22a15..5ef00bb0f 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs @@ -328,8 +328,6 @@ namespace Duplicati.Library.Main.Operation.Restore catch (RetiredException) { Logging.Log.WriteVerboseMessage(LOGTAG, "RetiredProcess", null, "File processor retired"); - block_request.Retire(); - block_response.Retire(); if (options.InternalProfiling) { @@ -340,9 +338,12 @@ namespace Duplicati.Library.Main.Operation.Restore catch (Exception ex) { Logging.Log.WriteErrorMessage(LOGTAG, "FileProcessingError", ex, "Error during file processing"); + throw; + } + finally + { block_request.Retire(); block_response.Retire(); - throw; } }); } From 82019ea28e1198cc8ca68295c0eb9919f9846741 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 10 Feb 2025 13:42:27 +0100 Subject: [PATCH 147/362] Modified VerifyTargetBlocks to not evict block from cache. It is now handled outside, in case of file name retargeting when overwrite is false. --- .../Main/Operation/Restore/FileProcessor.cs | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs index 5ef00bb0f..9e315d799 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs @@ -93,7 +93,7 @@ namespace Duplicati.Library.Main.Operation.Restore sw_work?.Start(); // Verify the target file blocks that may already exist. - var (bytes_verified, missing_blocks, verified_blocks) = await VerifyTargetBlocks(file, blocks, filehasher, blockhasher, options, results, block_request); + var (bytes_verified, missing_blocks, verified_blocks) = await VerifyTargetBlocks(file, blocks, filehasher, blockhasher, options, results); long bytes_written = 0; if (blocks.Length != missing_blocks.Count + verified_blocks.Count) { @@ -127,10 +127,24 @@ namespace Duplicati.Library.Main.Operation.Restore await CopyOldTargetBlocksToNewTarget(file, new_file, verified_blocks); } } + else + { + verified_blocks.Clear(); + missing_blocks = [.. blocks.Select(x => { x.CacheDecrEvict = false; return x; })]; + } file = new_file; } } + // Notify to the cache that we use local blocks. + foreach (var block in verified_blocks) + { + block.CacheDecrEvict = true; + sw_req?.Start(); + await block_request.WriteAsync(block); + sw_req?.Stop(); + } + if (missing_blocks.Count > 0 && options.UseLocalBlocks) { // Verify the local blocks at the original restore path that may be used to restore the file. @@ -478,9 +492,8 @@ namespace Duplicati.Library.Main.Operation.Restore /// A hasher for a data block. /// The Duplicati configuration options. /// The restoration results. - /// The channel to request blocks from the block manager. Used to inform the block manager which blocks are already present. /// An awaitable `Task`, which returns a collection of data blocks that are missing. - private static async Task<(long, List, List)> VerifyTargetBlocks(FileRequest file, BlockRequest[] blocks, System.Security.Cryptography.HashAlgorithm filehasher, System.Security.Cryptography.HashAlgorithm blockhasher, Options options, RestoreResults results, IChannel block_request) + private static async Task<(long, List, List)> VerifyTargetBlocks(FileRequest file, BlockRequest[] blocks, System.Security.Cryptography.HashAlgorithm filehasher, System.Security.Cryptography.HashAlgorithm blockhasher, Options options, RestoreResults results) { long bytes_read = 0; List missing_blocks = []; @@ -506,8 +519,6 @@ namespace Duplicati.Library.Main.Operation.Restore { // Block matches bytes_read += read; - blocks[i].CacheDecrEvict = true; - await block_request.WriteAsync(blocks[i]); verified_blocks.Add(blocks[i]); } else From 043b9230b5f74cede96f99eff6cc299cffa3e31c Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 10 Feb 2025 13:44:02 +0100 Subject: [PATCH 148/362] VerifyLocalBlocks opened file in incorrect mode, and the target file may not exist. --- Duplicati/Library/Main/Operation/Restore/FileProcessor.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs index 9e315d799..48d79df0e 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs @@ -617,7 +617,7 @@ namespace Duplicati.Library.Main.Operation.Restore (SystemIO.IO_OS.FileExists(file.TargetPath) ? SystemIO.IO_OS.FileOpenRead(file.TargetPath) : null) : - SystemIO.IO_OS.FileOpenWrite(file.TargetPath); + SystemIO.IO_OS.FileOpenReadWrite(file.TargetPath); var buffer = new byte[options.Blocksize]; long bytes_read = 0; long bytes_written = 0; @@ -656,8 +656,8 @@ namespace Duplicati.Library.Main.Operation.Restore { try { - f_target.Seek(blocks[j].BlockOffset * options.Blocksize, SeekOrigin.Begin); - await f_target.WriteAsync(buffer, 0, read); + f_target?.Seek(blocks[j].BlockOffset * options.Blocksize, SeekOrigin.Begin); + await f_target?.WriteAsync(buffer, 0, read); } catch (Exception) { From 3962cc12972405393350a9796b23c62e91f7dd6d Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 10 Feb 2025 13:44:21 +0100 Subject: [PATCH 149/362] Added a test to cover Issue #5957 --- Duplicati/UnitTest/IssueTests.cs | 93 ++++++++++++++++++++++++++------ 1 file changed, 78 insertions(+), 15 deletions(-) diff --git a/Duplicati/UnitTest/IssueTests.cs b/Duplicati/UnitTest/IssueTests.cs index bd706946a..4809a9376 100644 --- a/Duplicati/UnitTest/IssueTests.cs +++ b/Duplicati/UnitTest/IssueTests.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.DynamicLoader; using Duplicati.Library.Interface; @@ -311,6 +311,7 @@ namespace Duplicati.UnitTest [Test] + [Category("Restore"), Category("Bug")] public void Issue5886RestoreModifiedMiddleBlock() { var blocksize = 1024; @@ -374,6 +375,68 @@ namespace Duplicati.UnitTest } } + [Test] + [Category("Restore"), Category("Bug")] + public void Issue5957RestoreModifiedBlockUseLocalBlocks([Values] bool use_local, [Values] bool overwrite) + { + var blocksize = 1024; + var testopts = new Dictionary(TestOptions) + { + ["blocksize"] = $"{blocksize}b", + ["restore-with-local-blocks"] = use_local.ToString().ToLower(), + ["overwrite"] = overwrite.ToString().ToLower() + }; + + var original_dir = Path.Combine(DATAFOLDER, "some_original_dir"); + Directory.CreateDirectory(original_dir); + string f = Path.Combine(original_dir, "some_file"); + TestUtils.WriteTestFile(f, 3 * blocksize); + + // Backup the files + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IBackupResults backupResults = c.Backup([DATAFOLDER]); + TestUtils.AssertResults(backupResults); + } + + var original_contents = File.ReadAllBytes(f); + + using (var stream = File.Open(f, FileMode.Open, FileAccess.ReadWrite)) + { + stream.Seek(0, SeekOrigin.Begin); + byte[] buffer = new byte[1]; + stream.Read(buffer, 0, 1); + buffer[0] = (byte)~buffer[0]; + stream.Seek(0, SeekOrigin.Begin); + stream.Write(buffer, 0, 1); + } + + var restored_contents = File.ReadAllBytes(f); + Assert.That(restored_contents, Is.Not.EqualTo(original_contents), "Restored file should not be equal to original file"); + + // Attempt to restore the file + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + var restoreResults = c.Restore([f]); + Assert.That(restoreResults.RestoredFiles, Is.EqualTo(1), "File should have been restored."); + } + + // Verify that the files are equal + // List files in folder + if (overwrite) + { + restored_contents = File.ReadAllBytes(f); + } + else + { + var files = Directory.GetFiles(original_dir, "*", SearchOption.TopDirectoryOnly); + Assert.That(files.Length, Is.EqualTo(2), "There should be one file in the folder"); + var f2 = files.FirstOrDefault(v => v != f); + restored_contents = File.ReadAllBytes(f2); + } + Assert.That(restored_contents, Is.EqualTo(original_contents), "Restored file should be equal to original file"); + } + [Test] [Category("Disruption"), Category("Bug")] From d33fa0caf9df6c329486767af69c94bbe693d54f Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 10 Feb 2025 14:02:36 +0100 Subject: [PATCH 150/362] Fixed a typo in a unit test assertion --- Duplicati/UnitTest/IssueTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/UnitTest/IssueTests.cs b/Duplicati/UnitTest/IssueTests.cs index 4809a9376..62ddc787b 100644 --- a/Duplicati/UnitTest/IssueTests.cs +++ b/Duplicati/UnitTest/IssueTests.cs @@ -430,7 +430,7 @@ namespace Duplicati.UnitTest else { var files = Directory.GetFiles(original_dir, "*", SearchOption.TopDirectoryOnly); - Assert.That(files.Length, Is.EqualTo(2), "There should be one file in the folder"); + Assert.That(files.Length, Is.EqualTo(2), "There should be two files in the folder"); var f2 = files.FirstOrDefault(v => v != f); restored_contents = File.ReadAllBytes(f2); } From 0260a061bb68384f78b7289c9bda6cfdf63e7233 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 10 Feb 2025 21:37:52 +0100 Subject: [PATCH 151/362] Made the synctool an executable --- Duplicati.sln | 7 +++++ .../Library/AutoUpdater/PackageHelper.cs | 5 +++ .../Duplicati.CommandLine.SyncTool.csproj | 22 +++++++++++++ .../Duplicati.CommandLine.SyncTool/Program.cs | 31 +++++++++++++++++++ ReleaseBuilder/Build/Command.cs | 1 + .../RemoteSynchronization.csproj | 2 +- 6 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 Executables/net8/Duplicati.CommandLine.SyncTool/Duplicati.CommandLine.SyncTool.csproj create mode 100644 Executables/net8/Duplicati.CommandLine.SyncTool/Program.cs diff --git a/Duplicati.sln b/Duplicati.sln index 47c8cfa25..0c665b4b9 100644 --- a/Duplicati.sln +++ b/Duplicati.sln @@ -170,6 +170,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tools", "Tools", "{15388C37 EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RemoteSynchronization", "Tools\RemoteSynchronization\RemoteSynchronization.csproj", "{D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.CommandLine.SyncTool", "Executables\net8\Duplicati.CommandLine.SyncTool\Duplicati.CommandLine.SyncTool.csproj", "{48D32674-0FE9-4407-B102-4AE46D93595F}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -480,6 +482,10 @@ Global {D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D}.Debug|Any CPU.Build.0 = Debug|Any CPU {D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D}.Release|Any CPU.ActiveCfg = Release|Any CPU {D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D}.Release|Any CPU.Build.0 = Release|Any CPU + {48D32674-0FE9-4407-B102-4AE46D93595F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {48D32674-0FE9-4407-B102-4AE46D93595F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {48D32674-0FE9-4407-B102-4AE46D93595F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {48D32674-0FE9-4407-B102-4AE46D93595F}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -541,6 +547,7 @@ Global {8ACA2736-3C69-4FA5-BCF9-5EDF50CAF332} = {566EBBDA-19A4-4056-A615-D901D57D2439} {836E0557-B40C-4DC7-9A2A-5C062F9ACC6B} = {E1A9B303-F281-45C5-A4F6-CADD9DE3F3C4} {D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D} = {15388C37-9218-4818-972E-738EEA8F1602} + {48D32674-0FE9-4407-B102-4AE46D93595F} = {6B46F6B1-1898-49B8-ADA7-5CAF68EB77E3} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {8B40BAFE-D862-4397-9495-8F5EAF5CE80C} diff --git a/Duplicati/Library/AutoUpdater/PackageHelper.cs b/Duplicati/Library/AutoUpdater/PackageHelper.cs index 5f1ee0341..314facb04 100644 --- a/Duplicati/Library/AutoUpdater/PackageHelper.cs +++ b/Duplicati/Library/AutoUpdater/PackageHelper.cs @@ -89,6 +89,10 @@ namespace Duplicati.Library.AutoUpdater /// The secret tool /// SecretTool, + /// + /// The sync tool + /// + SyncTool } @@ -115,6 +119,7 @@ namespace Duplicati.Library.AutoUpdater NamedExecutable.Service => OperatingSystem.IsWindows() ? "Duplicati.Service.exe" : "duplicati-service", NamedExecutable.Agent => OperatingSystem.IsWindows() ? "Duplicati.Agent.exe" : "duplicati-agent", NamedExecutable.SecretTool => OperatingSystem.IsWindows() ? "Duplicati.CommandLine.SecretTool.exe" : "duplicati-secret-tool", + NamedExecutable.SyncTool => OperatingSystem.IsWindows() ? "Duplicati.CommandLine.SyncTool.exe" : "duplicati-sync-tool", _ => throw new ArgumentException($"Named executable not known: {exe}", nameof(exe)) }; diff --git a/Executables/net8/Duplicati.CommandLine.SyncTool/Duplicati.CommandLine.SyncTool.csproj b/Executables/net8/Duplicati.CommandLine.SyncTool/Duplicati.CommandLine.SyncTool.csproj new file mode 100644 index 000000000..2ce4cf329 --- /dev/null +++ b/Executables/net8/Duplicati.CommandLine.SyncTool/Duplicati.CommandLine.SyncTool.csproj @@ -0,0 +1,22 @@ + + + + Exe + net8.0 + SyncTool for Duplicati + Copyright © 2025 Team Duplicati, MIT license + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + diff --git a/Executables/net8/Duplicati.CommandLine.SyncTool/Program.cs b/Executables/net8/Duplicati.CommandLine.SyncTool/Program.cs new file mode 100644 index 000000000..c0e13737d --- /dev/null +++ b/Executables/net8/Duplicati.CommandLine.SyncTool/Program.cs @@ -0,0 +1,31 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +using Duplicati.Library.Crashlog; + +namespace Duplicati.CommandLine.SyncTool.Net8 +{ + // Wrapper class to keep code independent + public static class Program + { + public static int Main(string[] args) + => CrashlogHelper.WrapWithCrashLog(() => RemoteSynchronization.Program.Main(args).ConfigureAwait(false).GetAwaiter().GetResult()); + } +} \ No newline at end of file diff --git a/ReleaseBuilder/Build/Command.cs b/ReleaseBuilder/Build/Command.cs index d0de3a4b9..46ea9f935 100644 --- a/ReleaseBuilder/Build/Command.cs +++ b/ReleaseBuilder/Build/Command.cs @@ -86,6 +86,7 @@ public static partial class Command { "Duplicati.CommandLine.Snapshots", "duplicati-snapshots" }, { "Duplicati.CommandLine.ServerUtil", "duplicati-server-util" }, { "Duplicati.CommandLine.SecretTool", "duplicati-secret-tool" }, + { "Duplicati.CommandLine.SyncTool", "duplicati-sync-tool" }, { "Duplicati.Service", "duplicati-service" }, { "Duplicati.Agent", "duplicati-agent" }, { "Duplicati.CommandLine", "duplicati-cli" }, diff --git a/Tools/RemoteSynchronization/RemoteSynchronization.csproj b/Tools/RemoteSynchronization/RemoteSynchronization.csproj index 6cd7b2704..9c800af07 100644 --- a/Tools/RemoteSynchronization/RemoteSynchronization.csproj +++ b/Tools/RemoteSynchronization/RemoteSynchronization.csproj @@ -1,7 +1,7 @@  - Exe + Library net8.0 disable enable From 162f09743a498669d54376e9951d8b5d792be643 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 10 Feb 2025 21:52:20 +0100 Subject: [PATCH 152/362] Make ngclient the default client --- Duplicati/Server/webroot/index.html | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Duplicati/Server/webroot/index.html b/Duplicati/Server/webroot/index.html index 5502698e9..f5367732e 100644 --- a/Duplicati/Server/webroot/index.html +++ b/Duplicati/Server/webroot/index.html @@ -40,9 +40,7 @@ var theme = parseCookies()['default-theme']; if ((theme || '').trim().length == 0) - theme = 'ngax'; - if (theme == 'greeno') // Make sure users are redirected automatically - theme = 'ngax'; + theme = 'ngclient'; setCookieAndRun(theme); // Refresh From 782ce219177af01ec74fb159edf769bc138abea1 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 10 Feb 2025 22:04:11 +0100 Subject: [PATCH 153/362] Disable selenium tests --- .github/workflows/tests.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 67e1244ea..0f0d4b5ed 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -29,9 +29,10 @@ jobs: - name: Run unit tests run: dotnet test --no-build --verbosity minimal Duplicati.sln - selenium: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Selenium - run: pipeline/selenium/test.sh + # Disabled, as a new test needs to be written for the new UI + # selenium: + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # - name: Selenium + # run: pipeline/selenium/test.sh From 070b36dc29a72b92da373dd4f7d9e4cbf3c640bc Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 11 Feb 2025 09:24:51 +0100 Subject: [PATCH 154/362] Bumped ngclient --- Duplicati/Server/webroot/ngclient/package-lock.json | 8 ++++---- Duplicati/Server/webroot/ngclient/package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Duplicati/Server/webroot/ngclient/package-lock.json b/Duplicati/Server/webroot/ngclient/package-lock.json index 01c099cab..f8f458e12 100644 --- a/Duplicati/Server/webroot/ngclient/package-lock.json +++ b/Duplicati/Server/webroot/ngclient/package-lock.json @@ -9,13 +9,13 @@ "version": "1.0.0", "license": "MIT", "dependencies": { - "@duplicati/ngclient": "^0.0.73" + "@duplicati/ngclient": "^0.0.79" } }, "node_modules/@duplicati/ngclient": { - "version": "0.0.73", - "resolved": "https://registry.npmjs.org/@duplicati/ngclient/-/ngclient-0.0.73.tgz", - "integrity": "sha512-ucVBNCkMW/Zc4k8ypbYc4t0y75BFTiYo9pybyaiF8UpFCJtswBqBsyr21rR6S/W9Z/qI16WHnvzvEz359KfxNQ==", + "version": "0.0.79", + "resolved": "https://registry.npmjs.org/@duplicati/ngclient/-/ngclient-0.0.79.tgz", + "integrity": "sha512-wbTGr9AsekhzA0AOg5yJXEe+sfT+ivFDAmCo5+jwzp+3zc4LswVap0D4yMX2uIUB/zU4VJlPZoPZftENqdeZpw==", "license": "MIT" } } diff --git a/Duplicati/Server/webroot/ngclient/package.json b/Duplicati/Server/webroot/ngclient/package.json index 1fdc16e31..8029540e9 100644 --- a/Duplicati/Server/webroot/ngclient/package.json +++ b/Duplicati/Server/webroot/ngclient/package.json @@ -4,6 +4,6 @@ "description": "Modern Angular Client for Duplicati", "license": "MIT", "dependencies": { - "@duplicati/ngclient": "^0.0.73" + "@duplicati/ngclient": "^0.0.79" } } From 850dbd82dac0866afd99694786e0b507a0bf78e1 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 11 Feb 2025 09:59:40 +0100 Subject: [PATCH 155/362] Bump ngclient again --- Duplicati/Server/webroot/ngclient/package-lock.json | 8 ++++---- Duplicati/Server/webroot/ngclient/package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Duplicati/Server/webroot/ngclient/package-lock.json b/Duplicati/Server/webroot/ngclient/package-lock.json index f8f458e12..78716466e 100644 --- a/Duplicati/Server/webroot/ngclient/package-lock.json +++ b/Duplicati/Server/webroot/ngclient/package-lock.json @@ -9,13 +9,13 @@ "version": "1.0.0", "license": "MIT", "dependencies": { - "@duplicati/ngclient": "^0.0.79" + "@duplicati/ngclient": "^0.0.80" } }, "node_modules/@duplicati/ngclient": { - "version": "0.0.79", - "resolved": "https://registry.npmjs.org/@duplicati/ngclient/-/ngclient-0.0.79.tgz", - "integrity": "sha512-wbTGr9AsekhzA0AOg5yJXEe+sfT+ivFDAmCo5+jwzp+3zc4LswVap0D4yMX2uIUB/zU4VJlPZoPZftENqdeZpw==", + "version": "0.0.80", + "resolved": "https://registry.npmjs.org/@duplicati/ngclient/-/ngclient-0.0.80.tgz", + "integrity": "sha512-x8n8q3iFy9uh5tixtdn80M+95E1P81D0WKuCGnIv4mL1xlOKyod8vngfmabQrm8Vd2YcgB653sUc5gYxTKdtVA==", "license": "MIT" } } diff --git a/Duplicati/Server/webroot/ngclient/package.json b/Duplicati/Server/webroot/ngclient/package.json index 8029540e9..5c5f364cf 100644 --- a/Duplicati/Server/webroot/ngclient/package.json +++ b/Duplicati/Server/webroot/ngclient/package.json @@ -4,6 +4,6 @@ "description": "Modern Angular Client for Duplicati", "license": "MIT", "dependencies": { - "@duplicati/ngclient": "^0.0.79" + "@duplicati/ngclient": "^0.0.80" } } From 969f06cb9186929573ac192c6f2f16ce237dc0f7 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 11 Feb 2025 23:23:12 +0100 Subject: [PATCH 156/362] Version bump to v2.1.0.109-2.1.0.109_canary_2025-02-11 You can download this build from: Binaries: https://updates.duplicati.com/canary/?version=2.1.0.109 Signature file: https://updates.duplicati.com/canary/duplicati-2.1.0.109_canary_2025-02-11.signatures.zip --- ReleaseBuilder/build_version.txt | 2 +- changelog.txt | 53 ++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/ReleaseBuilder/build_version.txt b/ReleaseBuilder/build_version.txt index 81633e73f..60deb0e83 100644 --- a/ReleaseBuilder/build_version.txt +++ b/ReleaseBuilder/build_version.txt @@ -1 +1 @@ -2.1.0.108 \ No newline at end of file +2.1.0.109 \ No newline at end of file diff --git a/changelog.txt b/changelog.txt index a1cedc446..165df0823 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,56 @@ +2025-02-11 - 2.1.0.109_canary_2025-02-11 +========== +This release is a canary release intended to be used for testing. + +# Important change for Windows Service +This build changes the folder locations for the `Duplicati-server.sqlite` file on Windows and Linux when running as a service. +There is support for backwards compatibility, so existing installations should not be affected. + +The detailed changes are: +Windows: Avoid storing data in `C:\Windows\System32\config\systemprofile\AppData\Local\Duplicati` and prefer `{CSIDL_COMMON_APPDATA}\Duplicati` +Linux: Avoid storing data in `/Duplicati` and prefer `/var/lib/Duplicati`. + +If you are not running Duplicati as a service, this has no effect. +If you are using `--server-datafolder` or `DUPLICATI_HOME`, this has no effect on the database, but may cause your machineid and installid to change. + +The `machineid.txt` and `installid.txt` would previously be stored in the local app data folder, even when using portable mode or choosing a specific data folder. +This has been fixed, so the files will now follow the database. +If you are using the Duplicati console or otherwise depend on these values, you need to move them into the folder where the database is stored. + +## Important for users running 2.1.0.108 +For **2.1.0.108 version only**, the service folder was `{SYSTEMDRIVE}\Users\LocalService\Duplicati` which was dropped. +If you have only installed 2.1.0.108 and ended up using the previous folder, you must manually move it to `{CSIDL_COMMON_APPDATA}\Duplicati`. + +# Permissions on database and data folder +This update sets permissions on the data folder and the databases to prevent unauthorized access from local accounts. +To opt out of setting permissions on each startup, place a file named `insecure-permissions.txt` inside the data folder. + +# New user interface +This version uses the `ngclient` as the default interface. If some feature is not working as expected, it is possible to switch back to the `ngax` client. +To switch client, replace `ngclient` with `ngax` in the url, or use the `theme.html` page to choose the UI (example: `http://localhost:8200/theme.html`). +Please report any issues with the new UI. + +# Removed timeout guard +Due to many reports of failures with the timeout mechanism this has been removed in the current version. It will be added back into each of the backends later. + +# Added a new tool for offsite backup +This release features a new tool: `Duplicati.CommandLine.SyncTool.exe` / `duplicati-sync-tool` which can be used to create copies of backup files in multiple locations. The tool is currently documented with the built-in help, and more documentation is on the way. The tool is intended to be more integrated into Duplicati later and enables a straightforward way to achieve 3-2-1 backups, including copies in immutable storage. + +## Detailed list of changes: +- Support topics on Telegram reports +- Added remote syncronization tool +- Removed high-level backend timeout +- Added guard to prevent dataloss on shared destinations +- Added option to disable automatic update checks +- Fixed a bug with the new restore that could cause incorrect files to be restored if files alredy exists +- Fixed a case where updated timestamps were discarded if no data was changed +- Added permission updates to data folder +- Fixed automatic fallback on unsupported Zip files +- Fixed multiple issues with TrayIcon state +- Mark Mega.nz backend as deprecated +- Switched to ngclient as the default UI +- Added support for 429 responses on B2 + 2025-01-31 - 2.1.0.108_canary_2025-01-31 ========== This release is a canary release intended to be used for testing. From f3a8b2554b3f404728c88b242d88a6e9ee3e45b5 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 12 Feb 2025 00:13:25 +0100 Subject: [PATCH 157/362] Fixed links from wiki to docs --- README.ja-JP.md | 74 +++++++++++++++++++++++-------------------------- README.md | 44 +++++++++++++---------------- README.zh-CN.md | 43 +++++++++++++--------------- 3 files changed, 72 insertions(+), 89 deletions(-) diff --git a/README.ja-JP.md b/README.ja-JP.md index 2f21bc824..1d24d280c 100644 --- a/README.ja-JP.md +++ b/README.ja-JP.md @@ -19,87 +19,81 @@ Gitterは削除済 [![カバレッジの状況](https://coveralls.io/repos/github/duplicati/duplicati/badge.svg?branch=HEAD)](https://coveralls.io/github/duplicati/duplicati?branch=HEAD) [![ライセンス](https://img.shields.io/badge/License-MIT-yellow.svg)](https://github.com/duplicati/duplicati/blob/master/LICENSE.txt) +Duplicati は、フリー（自由）でオープンソースのバックアップ用クライアントです。圧縮し、暗号化した増分バックアップを、クラウドストレージサービスや遠隔のファイルサーバー上に安全に保存できます。Duplicati は、主に以下のサービスやソフトウェアで使うことができます。 -Duplicatiは、フリー（自由）でオープンソースのバックアップ用クライアントです。圧縮し、暗号化した増分バックアップを、クラウドストレージサービスや遠隔のファイルサーバー上に安全に保存できます。Duplicatiは、主に以下のサービスやソフトウェアで使うことができます。 +   _Amazon S3、[IDrive e2](https://www.idrive.com/e2/duplicati "Using Duplicati with IDrive e2")、[Backblaze (B2)](https://www.backblaze.com/blog/duplicati-backups-cloud-storage/ "Duplicati with Backblaze B2 Cloud Storage")、Box、Dropbox、FTP、Google クラウド、Google ドライブ、MEGA、Microsoft Azure、Microsoft OneDrive、Rackspace Cloud Files、OpenStack Storage (Swift)、Sia、Storj DCS、SSH (SFTP)、WebDAV、Tencent クラウドオブジェクトストレージ（COS）、Aliyun OSS、[その他にも対応しています！](https://docs.duplicati.com/backup-destinations/destination-overview)_ -   *Amazon S3、[IDrive e2](https://www.idrive.com/e2/duplicati "Using Duplicati with IDrive e2")、[Backblaze (B2)](https://www.backblaze.com/blog/duplicati-backups-cloud-storage/ "Duplicati with Backblaze B2 Cloud Storage")、Box、Dropbox、FTP、Googleクラウド、Googleドライブ、MEGA、Microsoft Azure、Microsoft OneDrive、Rackspace Cloud Files、OpenStack Storage (Swift)、Sia、Storj DCS、SSH (SFTP)、WebDAV、Tencentクラウドオブジェクトストレージ（COS）、Aliyun OSS、[その他にも対応しています！](https://docs.duplicati.com/backup-destinations/destination-overview)* +Duplicati は MIT ライセンスで公開されており、Windows、OSX、Linux で利用できます（.NET 4.7.1 以上、または Mono 5.10.0 以上が必要です）。 -DuplicatiはMITライセンスで公開されており、Windows、OSX、Linuxで利用できます（.NET 4.7.1以上、またはMono 5.10.0以上が必要です）。 +# ダウンロード -ダウンロード -======== +Duplicati 2.0 のベータ版が Duplicati の最新バージョンとなります。 -Duplicati 2.0のベータ版がDuplicatiの最新バージョンとなります。 +[ここをクリックすると、Duplicati 2.0 のベータ版をダウンロードできます。](https://duplicati.com/download) -[ここをクリックすると、Duplicati 2.0のベータ版をダウンロードできます。](https://duplicati.com/download) - -ベータ版では、アップデートがある場合に自動的に通知を行い、1クリック（またはターミナルでのコマンド入力）でアップグレードできます。 +ベータ版では、アップデートがある場合に自動的に通知を行い、1 クリック（またはターミナルでのコマンド入力）でアップグレードできます。 より新しい[テスト版に関しては、最新のリリースを確認](https://github.com/duplicati/duplicati/releases)するか、ソフトウェア上の画面またはコマンドラインで、別のアップデートチャンネルを選択してください。 -全てのリリースは、GPGで署名されます。署名に使われる公開鍵は[3DAC703D](https://keys.openpgp.org/search?q=0xC20E90473DAC703D)となります。最新の署名ファイル（バイナリー版とASCII版）については、[Duplicatiのダウンロード用ページ](https://github.com/duplicati/duplicati/releases)から入手できます。 +全てのリリースは、GPG で署名されます。署名に使われる公開鍵は[3DAC703D](https://keys.openpgp.org/search?q=0xC20E90473DAC703D)となります。最新の署名ファイル（バイナリー版と ASCII 版）については、[Duplicati のダウンロード用ページ](https://github.com/duplicati/duplicati/releases)から入手できます。 -サポート -======= +# サポート -Duplicatiは、活発なコミュニティーによってサポートが行われています。コミュニティーには[フォーラム](https://forum.duplicati.com)からご参加ください。 +Duplicati は、活発なコミュニティーによってサポートが行われています。コミュニティーには[フォーラム](https://forum.duplicati.com)からご参加ください。 -[Duplicatiのマニュアル](https://docs.duplicati.com)もあります。マニュアルの作成や維持にぜひ[ご参加](https://github.com/kees-z/DuplicatiDocs)ください。 +[Duplicati のマニュアル](https://docs.duplicati.com)もあります。マニュアルの作成や維持にぜひ[ご参加](https://github.com/kees-z/DuplicatiDocs)ください。 -機能 -======== +# 機能 - * 全てのデータについて、アップロードする前にAES-256（またはGNU Privacy Guard）による暗号化を行い、データの安全性を確保します。 - * 最初に全体のフルバックアップを行い、その後、小さな増分のバックアップを送信することにより、回線の帯域幅と保存領域の使用量を節約します。 - * スケジュールの設定機能により、バックアップを最新のものに自動的に維持します。 - * 新しいリリースが公開された際に、通知を行います。 - * 暗号化したバックアップのファイルを、FTP、 Cloudfiles、WebDAV、SSH (SFTP)、Amazon S3などのサービスに送信します。 - * フォルダー、ドキュメントや画像などファイルの種類、ユーザー定義のフィルターを指定して、バックアップを実行できます。 - * 簡単に使える操作画面と、コマンドラインのツールを備えています。 - * Windowsのボリュームシャドウコピーサービス（VSS）や、Linuxの論理ボリューム管理（LVM）によって、プログラムによって開かれているファイルや、ロックされているファイルを適切にバックアップできます。これにより、Microsoft Outlookを使っている際に、OutlookのPSTファイルをバックアップできます。 - * フィルター、削除に関するルール、転送や帯域幅に関する設定などを行えます。 +- 全てのデータについて、アップロードする前に AES-256（または GNU Privacy Guard）による暗号化を行い、データの安全性を確保します。 +- 最初に全体のフルバックアップを行い、その後、小さな増分のバックアップを送信することにより、回線の帯域幅と保存領域の使用量を節約します。 +- スケジュールの設定機能により、バックアップを最新のものに自動的に維持します。 +- 新しいリリースが公開された際に、通知を行います。 +- 暗号化したバックアップのファイルを、FTP、 Cloudfiles、WebDAV、SSH (SFTP)、Amazon S3 などのサービスに送信します。 +- フォルダー、ドキュメントや画像などファイルの種類、ユーザー定義のフィルターを指定して、バックアップを実行できます。 +- 簡単に使える操作画面と、コマンドラインのツールを備えています。 +- Windows のボリュームシャドウコピーサービス（VSS）や、Linux の論理ボリューム管理（LVM）によって、プログラムによって開かれているファイルや、ロックされているファイルを適切にバックアップできます。これにより、Microsoft Outlook を使っている際に、Outlook の PST ファイルをバックアップできます。 +- フィルター、削除に関するルール、転送や帯域幅に関する設定などを行えます。 -Duplicatiの利点 -================== +# Duplicati の利点 データを安全に保つこと。離れたところに保管すること。バックアップを定期的に更新すること。 とてもシンプルなルールですが、今日の多くのバックアップ用サービスやソフトウェアは、これを達成していません。 -一方、Duplicatiでは、このルールを実践しています！ +一方、Duplicati では、このルールを実践しています！ -データを安全に保ちましょう！　悪意をもったインターネット上の人々は、興味を引くデータをあらゆるところで探し回っているようです。しかしユーザーは、自らのプライベートなデータが第三者に暴かれてもよいとは誰も思っていません。Duplicatiでは、強力な暗号を使うことで、あなたのデータが、自分以外には全く意味不明なものになっていることを保証します。よく検討されたパスワードを使うと、あなたのバックアップファイルは、公開されているウェブサーバー上に保管されている場合でも、あなたの自宅にあり、しかし暗号化されずに保管されているファイルと比べて、より安全なものとなります。 +データを安全に保ちましょう！　悪意をもったインターネット上の人々は、興味を引くデータをあらゆるところで探し回っているようです。しかしユーザーは、自らのプライベートなデータが第三者に暴かれてもよいとは誰も思っていません。Duplicati では、強力な暗号を使うことで、あなたのデータが、自分以外には全く意味不明なものになっていることを保証します。よく検討されたパスワードを使うと、あなたのバックアップファイルは、公開されているウェブサーバー上に保管されている場合でも、あなたの自宅にあり、しかし暗号化されずに保管されているファイルと比べて、より安全なものとなります。 -バックアップは、離れたところに保管しましょう！　たとえバックアップが完璧だったとしても、それがバックアップ元のデータもろとも失われてしまっては何の意味もありません。職場で火事があった場合を想像してみてください。… バックアップは火事にも負けず生き残りますか？　Duplicatiはバックアップを多様な遠隔のファイルサーバーに保存し、データの更新が必要な部分だけが転送されるよう、増分バックアップをサポートしています。これによって、バックアップ元のデータから遠く離れたところにバックアップを保管しやすくなっています。 +バックアップは、離れたところに保管しましょう！　たとえバックアップが完璧だったとしても、それがバックアップ元のデータもろとも失われてしまっては何の意味もありません。職場で火事があった場合を想像してみてください。… バックアップは火事にも負けず生き残りますか？　 Duplicati はバックアップを多様な遠隔のファイルサーバーに保存し、データの更新が必要な部分だけが転送されるよう、増分バックアップをサポートしています。これによって、バックアップ元のデータから遠く離れたところにバックアップを保管しやすくなっています。 -定期的にバックアップを行いましょう！　最悪のケースは、しかるべきときにバックアップを行うことをうっかり忘れていたために、バックアップが古くなってしまっていることです。Duplicatiにはスケジュールの設定機能が備わっているので、簡単に、最新の状態のバックアップを定期的に作成できます。また、Duplicatiはファイルの圧縮を実行し、増分バックアップを行えるため、保存領域と帯域幅を節約できます。 +定期的にバックアップを行いましょう！　最悪のケースは、しかるべきときにバックアップを行うことをうっかり忘れていたために、バックアップが古くなってしまっていることです。Duplicati にはスケジュールの設定機能が備わっているので、簡単に、最新の状態のバックアップを定期的に作成できます。また、Duplicati はファイルの圧縮を実行し、増分バックアップを行えるため、保存領域と帯域幅を節約できます。 -開発に参加 -================== +# 開発に参加 ## 不具合を報告 -バグの管理にはGitHubを使っています。不具合を発見した場合は https://github.com/duplicati/duplicati/issues で既存のIssueがないか検索して、もしまだ報告されていないようであれば、新しいIssueを作成してください。 + +バグの管理には GitHub を使っています。不具合を発見した場合は https://github.com/duplicati/duplicati/issues で既存の Issue がないか検索して、もしまだ報告されていないようであれば、新しい Issue を作成してください。 ## 翻訳に参加 -Duplicatiの翻訳に興味がある場合は、[Transifex](https://www.transifex.com/duplicati/duplicati/dashboard/)で翻訳作業にご参加ください。 + +Duplicati の翻訳に興味がある場合は、[Transifex](https://www.transifex.com/duplicati/duplicati/dashboard/)で翻訳作業にご参加ください。 ## 開発作業に参加 -開発環境を設定してDuplicatiをビルドする方法については、[ウィキ](https://github.com/duplicati/duplicati/wiki/How-to-build-from-source)をご覧ください。不具合を修正したり、Duplicatiを改善したりするプルリクエストについては、いつでも歓迎します。 -修正すべき問題を探している場合は、[minor change](https://github.com/duplicati/duplicati/issues?q=is%3Aissue+is%3Aopen+label%3A%22minor+change%22)のIssueを確認してみてください。ウェブUIの開発に慣れている場合は、 [「UI」でタグ付けされたIssue](https://github.com/duplicati/duplicati/issues?q=is%3Aissue+is%3Aopen+label%3A%22UI%22)を見てみてください。 +開発環境を設定して Duplicati をビルドする方法については、[ウィキ](https://docs.duplicati.com/installation-details/developer)をご覧ください。不具合を修正したり、Duplicati を改善したりするプルリクエストについては、いつでも歓迎します。 +修正すべき問題を探している場合は、[minor change](https://github.com/duplicati/duplicati/issues?q=is%3Aissue+is%3Aopen+label%3A%22minor+change%22)の Issue を確認してみてください。ウェブ UI の開発に慣れている場合は、 [「UI」でタグ付けされた Issue](https://github.com/duplicati/duplicati/issues?q=is%3Aissue+is%3Aopen+label%3A%22UI%22)を見てみてください。 貢献していただいた皆様に感謝いたします！ - ## 後援 後援いただいている皆様に感謝いたします！🙏 - ## スポンサー -以下に、Duplicatiに寄付していただいたスポンサーを一覧でご紹介します。 +以下に、Duplicati に寄付していただいたスポンサーを一覧でご紹介します。 diff --git a/README.md b/README.md index 94240146a..93d4d54be 100644 --- a/README.md +++ b/README.md @@ -20,15 +20,13 @@ Removed Gitter [![License](https://img.shields.io/badge/License-MIT-yellow.svg)](https://github.com/duplicati/duplicati/blob/master/LICENSE.txt) [![Gurubase](https://img.shields.io/badge/Gurubase-Ask%20Duplicati%20Guru-006BFF)](https://gurubase.io/g/duplicati) - Duplicati is a free, open source, backup client that securely stores encrypted, incremental, compressed backups on cloud storage services and remote file servers. It works with: -   *Amazon S3, [IDrive e2](https://www.idrive.com/e2/duplicati "Using Duplicati with IDrive e2"), [Backblaze (B2)](https://www.backblaze.com/blog/duplicati-backups-cloud-storage/ "Duplicati with Backblaze B2 Cloud Storage"), Box, Dropbox, FTP, Google Cloud and Drive, MEGA, Microsoft Azure and OneDrive, Rackspace Cloud Files, OpenStack Storage (Swift), Sia, Storj DCS, SSH (SFTP), WebDAV, Tencent Cloud Object Storage (COS), Aliyun OSS, [and more!](https://docs.duplicati.com/backup-destinations/destination-overview)* +   _Amazon S3, [IDrive e2](https://www.idrive.com/e2/duplicati "Using Duplicati with IDrive e2"), [Backblaze (B2)](https://www.backblaze.com/blog/duplicati-backups-cloud-storage/ "Duplicati with Backblaze B2 Cloud Storage"), Box, Dropbox, FTP, Google Cloud and Drive, MEGA, Microsoft Azure and OneDrive, Rackspace Cloud Files, OpenStack Storage (Swift), Sia, Storj DCS, SSH (SFTP), WebDAV, Tencent Cloud Object Storage (COS), Aliyun OSS, [and more!](https://docs.duplicati.com/backup-destinations/destination-overview)_ Duplicati is licensed under the MIT license and available for Windows, OSX and Linux (.NET 4.7.1+ or Mono 5.10.0+ required). -Download -======== +# Download The latest version of Duplicati is a beta version for the Duplicati 2.0 release. @@ -39,28 +37,25 @@ For even more [bleeding edge access, check the latest releases](https://github.c All releases are GPG signed with the public key [3DAC703D](https://keys.openpgp.org/search?q=0xC20E90473DAC703D). The latest signature file and latest ASCII signature file are also available from [the Duplicati download page](https://github.com/duplicati/duplicati/releases). -Support -======= +# Support Duplicati is supported by an [active community and you can reach them via our forum](https://forum.duplicati.com). We have a great [Duplicati manual](https://docs.duplicati.com) that you can also [contribute to](https://github.com/kees-z/DuplicatiDocs). -Features -======== +# Features - * Duplicati uses AES-256 encryption (or GNU Privacy Guard) to secure all data before it is uploaded. - * Duplicati uploads a full backup initially and stores smaller, incremental updates afterwards to save bandwidth and storage space. - * A scheduler keeps backups up-to-date automatically. - * Integrated updater notifies you when a new release is out - * Encrypted backup files are transferred to targets like FTP, Cloudfiles, WebDAV, SSH (SFTP), Amazon S3 and others. - * Duplicati allows backups of folders, document types like e.g. documents or images, or custom filter rules. - * Duplicati is available as application with an easy-to-use user interface and as command line tool. - * Duplicati can make proper backups of opened or locked files using the Volume Snapshot Service (VSS) under Windows or the Logical Volume Manager (LVM) under Linux. This allows Duplicati to back up the Microsoft Outlook PST file while Outlook is running. - * Filters, deletion rules, transfer and bandwidth options, etc +- Duplicati uses AES-256 encryption (or GNU Privacy Guard) to secure all data before it is uploaded. +- Duplicati uploads a full backup initially and stores smaller, incremental updates afterwards to save bandwidth and storage space. +- A scheduler keeps backups up-to-date automatically. +- Integrated updater notifies you when a new release is out +- Encrypted backup files are transferred to targets like FTP, Cloudfiles, WebDAV, SSH (SFTP), Amazon S3 and others. +- Duplicati allows backups of folders, document types like e.g. documents or images, or custom filter rules. +- Duplicati is available as application with an easy-to-use user interface and as command line tool. +- Duplicati can make proper backups of opened or locked files using the Volume Snapshot Service (VSS) under Windows or the Logical Volume Manager (LVM) under Linux. This allows Duplicati to back up the Microsoft Outlook PST file while Outlook is running. +- Filters, deletion rules, transfer and bandwidth options, etc -Why use Duplicati? -================== +# Why use Duplicati? Keep your data safe, store it far away, update your backup regularly! This is a simple rule but many backup solutions do not achieve that today. @@ -72,34 +67,33 @@ Store your backup far away! The best backup is useless when it is destroyed toge Backup regularly! The worst case is that your backup is outdated simply because someone forgot to make a backup at the right time. Duplicati has a built-in scheduler, so that it's easy to have a regular, up-to-date backup. Furthermore, Duplicati uses file compression and is able to store incremental backups to save storage space and bandwidth. -Contributing -================== +# Contributing ## Contributing Bug reports + We use GitHub for bug tracking. Please search the existing issues for your bug and create a new one if the issue is not yet tracked: https://github.com/duplicati/duplicati/issues ## Contributing Translations + Interested in helping to translate duplicati? Help is always welcome at transifex: https://www.transifex.com/duplicati/duplicati/dashboard/ ## Contributing Code -Instructions on how to setup your development environment and build duplicati can be found in the [wiki](https://github.com/duplicati/duplicati/wiki/How-to-build-from-source). Pull requests fixing bugs or improving duplicati in another way are appreciated. + +Instructions on how to setup your development environment and build duplicati can be found in the [documentation](https://docs.duplicati.com/installation-details/developer). Pull requests fixing bugs or improving duplicati in another way are appreciated. If you are looking for an issue to fix, try looking at one of the [minor change](https://github.com/duplicati/duplicati/issues?q=is%3Aissue+is%3Aopen+label%3A%22minor+change%22) issues. If you are most used to web development, have a look at the [issues marked UI](https://github.com/duplicati/duplicati/issues?q=is%3Aissue+is%3Aopen+label%3A%22UI%22). - Thank you to all our existing contributors: - ## Backers Thank you to all our backers! 🙏 - ## Sponsors The list below reflects the sponsors who donated to the open-source project. diff --git a/README.zh-CN.md b/README.zh-CN.md index 9c81224bb..d7aa179c0 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -23,15 +23,13 @@ [![覆盖率状态](https://coveralls.io/repos/github/duplicati/duplicati/badge.svg?branch=HEAD)](https://coveralls.io/github/duplicati/duplicati?branch=HEAD) [![许可](https://img.shields.io/github/license/duplicati/duplicati.svg)](https://github.com/duplicati/duplicati/blob/master/LICENSE.txt) - Duplicati 是一个免费、开源的备份客户端，可以安全地将加密、增量、压缩的备份存储在云存储服务和远程文件服务器上。它与以下服务兼容： -   *亚马逊 S3、[IDrive e2](https://www.idrive.com/e2/duplicati "使用 Duplicati 与 IDrive e2")、[Backblaze (B2)](https://www.backblaze.com/blog/duplicati-backups-cloud-storage/ "Duplicati 与 Backblaze B2 云存储")、Box、Dropbox、FTP、Google Cloud 和 Drive、MEGA、Microsoft Azure 和 OneDrive、Rackspace Cloud Files、OpenStack Storage (Swift)、Sia、Storj DCS、SSH (SFTP)、WebDAV、阿里云对象存储(OSS)、百度网盘、阿里云盘、腾讯云对象存储 (COS)、[以及更多！](https://docs.duplicati.com/backup-destinations/destination-overview)* +   _亚马逊 S3、[IDrive e2](https://www.idrive.com/e2/duplicati "使用 Duplicati 与 IDrive e2")、[Backblaze (B2)](https://www.backblaze.com/blog/duplicati-backups-cloud-storage/ "Duplicati 与 Backblaze B2 云存储")、Box、Dropbox、FTP、Google Cloud 和 Drive、MEGA、Microsoft Azure 和 OneDrive、Rackspace Cloud Files、OpenStack Storage (Swift)、Sia、Storj DCS、SSH (SFTP)、WebDAV、阿里云对象存储(OSS)、百度网盘、阿里云盘、腾讯云对象存储 (COS)、[以及更多！](https://docs.duplicati.com/backup-destinations/destination-overview)_ Duplicati 根据 LGPL 许可证授权，并可用于 Windows、OSX 和 Linux (.NET 4.7.1+ 或 Mono 5.10.0+ 需要)。 -下载 -======== +# 下载 Duplicati 的最新版本是 Duplicati 2.0 发布的测试版。 @@ -42,28 +40,25 @@ Duplicati 的最新版本是 Duplicati 2.0 发布的测试版。 所有发布版本都使用公钥 [3DAC703D](https://keys.openpgp.org/search?q=0xC20E90473DAC703D) 进行 GPG 签名。最新的签名文件和最新的 ASCII 签名文件也可以在 [Duplicati 下载页面](https://github.com/duplicati/duplicati/releases) 获取。 -支持 -======= +# 支持 Duplicati 由一个[活跃的社区支持，您可以通过我们的论坛与他们联系](https://forum.duplicati.com)。 我们有一个很棒的 [Duplicati 手册](https://docs.duplicati.com)，您也可以[为其做出贡献](https://github.com/kees-z/DuplicatiDocs)。 -功能 -======== +# 功能 - * Duplicati 使用 AES-256 加密（或 GNU Privacy Guard）在上传之前加密所有数据。 - * Duplicati 最初上传完整备份，之后存储较小的增量更新，以节省带宽和存储空间。 - * 计划程序自动保持备份的最新状态。 - * 集成的更新器会通知您新版本发布 - * 加密的备份文件被传输到像 FTP、Cloudfiles、WebDAV、SSH (SFTP)、Amazon S3 等目标。 - * Duplicati 允许备份文件夹、文档类型（如文档或图像）或自定义过滤规则。 - * Duplicati 有易于使用的用户界面和命令行工具。 - * Duplicati 可以使用 Windows 下的卷快照服务 (VSS) 或 Linux 下的逻辑卷管理器 (LVM) 对打开或锁定的文件进行正确备份。这允许 Duplicati 在 Outlook 运行时备份 Microsoft Outlook PST 文件。 - * 过滤器、删除规则、传输和带宽选项等 +- Duplicati 使用 AES-256 加密（或 GNU Privacy Guard）在上传之前加密所有数据。 +- Duplicati 最初上传完整备份，之后存储较小的增量更新，以节省带宽和存储空间。 +- 计划程序自动保持备份的最新状态。 +- 集成的更新器会通知您新版本发布 +- 加密的备份文件被传输到像 FTP、Cloudfiles、WebDAV、SSH (SFTP)、Amazon S3 等目标。 +- Duplicati 允许备份文件夹、文档类型（如文档或图像）或自定义过滤规则。 +- Duplicati 有易于使用的用户界面和命令行工具。 +- Duplicati 可以使用 Windows 下的卷快照服务 (VSS) 或 Linux 下的逻辑卷管理器 (LVM) 对打开或锁定的文件进行正确备份。这允许 Duplicati 在 Outlook 运行时备份 Microsoft Outlook PST 文件。 +- 过滤器、删除规则、传输和带宽选项等 -为什么使用 Duplicati？ -================== +# 为什么使用 Duplicati？ 保护您的数据安全，将其存储在远处，定期更新您的备份！ 这是一个简单的规则，但许多备份解决方案今天都无法做到。 @@ -75,33 +70,33 @@ Duplicati 由一个[活跃的社区支持，您可以通过我们的论坛与他 定期备份！最糟糕的情况是，您的备份过时了，仅仅是因为有人忘了在正确的时间备份。Duplicati 有内置的计划程序，因此很容易拥有定期、最新的备份。此外，Duplicati 使用文件压缩，并能够存储增量备份以节省存储空间和带宽。 -贡献 -================== +# 贡献 ## 贡献错误报告 + 我们使用 GitHub 进行错误跟踪。请在创建新问题前先搜索已有的问题，看看您的错误是否已被记录： https://github.com/duplicati/duplicati/issues ## 贡献翻译 + 对帮助翻译 duplicati 感兴趣吗？欢迎在 transifex 提供帮助： https://www.transifex.com/duplicati/duplicati/dashboard/ ## 贡献代码 -关于如何设置您的开发环境以及如何构建 duplicati 的说明可以在 [wiki](https://github.com/duplicati/duplicati/wiki/How-to-build-from-source) 中找到。我们欣赏任何修复错误或以其他方式改进 duplicati 的拉取请求。 + +关于如何设置您的开发环境以及如何构建 duplicati 的说明可以在 [docs](https://docs.duplicati.com/installation-details/developer) 中找到。我们欣赏任何修复错误或以其他方式改进 duplicati 的拉取请求。 如果您正在寻找一个问题来解决，请尝试查看其中一个标记为 [小改动](https://github.com/duplicati/duplicati/issues?q=is%3Aissue+is%3Aopen+label%3A%22minor+change%22) 的问题。如果您最熟悉的是 Web 开发，请查看标记为 [UI](https://github.com/duplicati/duplicati/issues?q=is%3Aissue+is%3Aopen+label%3A%22UI%22) 的问题。 感谢我们所有的现有贡献者： - ## 赞助者 感谢所有的赞助者！🙏 [[成为赞助者](https://opencollective.com/duplicati#backer)] - ## 赞助商 通过成为赞助商来支持这个项目。您的标志将会出现在这里，附带链接到您的网站。[[成为赞助商](https://opencollective.com/duplicati#sponsor)] From 439d10b53d95ced9aaa41fd3b47aff29bbe88aab Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 12 Feb 2025 09:23:07 +0100 Subject: [PATCH 158/362] Improve shell-script arguments --- Duplicati/Library/Modules/Builtin/run-script-example.bat | 7 ++++++- Duplicati/Library/Modules/Builtin/run-script-example.sh | 6 +++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Modules/Builtin/run-script-example.bat b/Duplicati/Library/Modules/Builtin/run-script-example.bat index e858bbc2b..a1797c568 100644 --- a/Duplicati/Library/Modules/Builtin/run-script-example.bat +++ b/Duplicati/Library/Modules/Builtin/run-script-example.bat @@ -50,7 +50,12 @@ REM --run-script-with-arguments = REM If set to true, the script path will be parsed as a command line, and the REM arguments will be passed to the script. If set to false (default), REM the script path will used as a single path. -REM Use double- or single-quotes to pass arguments with spaces. +REM If you do not have spaces in your script path or arguments, simply enter +REM it as a string: +REM Example: --run-script-before="C:\path\to\script.bat arg1 arg2 --option1=a" +REM If you have spaces in the path or arguements, use double- or single-quotes +REM around the elements that have spaces, similar to how you would do +REM on the command line: REM Example: --run-script-before="\"C:\path to\script.bat\" \"arg1 \" arg2" diff --git a/Duplicati/Library/Modules/Builtin/run-script-example.sh b/Duplicati/Library/Modules/Builtin/run-script-example.sh index 8ce9ef965..4743a3a98 100755 --- a/Duplicati/Library/Modules/Builtin/run-script-example.sh +++ b/Duplicati/Library/Modules/Builtin/run-script-example.sh @@ -50,7 +50,11 @@ # If set to true, the script path will be parsed as a command line, and the # arguments will be passed to the script. If set to false (default), # the script path will used as a single path. -# Use double- or single-quotes to pass arguments with spaces. +# If you do not have spaces in your script path or arguments, simply enter +# it as a string: +# Example: --run-script-before="/path/to/script.sh arg1 arg2 --option=a" +# If you have spaces in the path or arguements, use double- or single-quotes +# around the elements that have spaces, similar to how you would do in a shell: # Example: --run-script-before="\"/path to/script.sh\" \"arg1 \" arg2" From 6058d0706f8edcbf021ebcd4ed061064c0e768f1 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 12 Feb 2025 10:01:22 +0100 Subject: [PATCH 159/362] Clear theme cookies --- Duplicati/Server/webroot/index.html | 4 ++-- Duplicati/Server/webroot/theme.html | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Duplicati/Server/webroot/index.html b/Duplicati/Server/webroot/index.html index f5367732e..dec254937 100644 --- a/Duplicati/Server/webroot/index.html +++ b/Duplicati/Server/webroot/index.html @@ -23,7 +23,7 @@ var d = new Date(); d.setTime(d.getTime() + (90*24*60*60*1000)); - document.cookie = 'default-theme=' + theme + '; expires=' + d.toUTCString() + '; path=/'; + document.cookie = 'default-theme-v2=' + theme + '; expires=' + d.toUTCString() + '; path=/'; // Make an absolute path redirect, as relative redirects break if the port is non-standard var path = window.location.pathname; @@ -38,7 +38,7 @@ window.location.replace(path); } - var theme = parseCookies()['default-theme']; + var theme = parseCookies()['default-theme-v2']; if ((theme || '').trim().length == 0) theme = 'ngclient'; diff --git a/Duplicati/Server/webroot/theme.html b/Duplicati/Server/webroot/theme.html index f3dc3e60a..b389e4d6d 100644 --- a/Duplicati/Server/webroot/theme.html +++ b/Duplicati/Server/webroot/theme.html @@ -50,7 +50,7 @@ } function updateToCurrentTheme() { - var theme = parseCookies()["default-theme"]; + var theme = parseCookies()["default-theme-v2"]; var themes = ["ngax", "ngclient"]; for (var k in themes) { From 4b70a2e0ae092245c36db0b0b07e25e4ab978d09 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 12 Feb 2025 10:05:12 +0100 Subject: [PATCH 160/362] Rename to `default-client` --- Duplicati/Server/webroot/index.html | 4 ++-- Duplicati/Server/webroot/theme.html | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Duplicati/Server/webroot/index.html b/Duplicati/Server/webroot/index.html index dec254937..d979ecc54 100644 --- a/Duplicati/Server/webroot/index.html +++ b/Duplicati/Server/webroot/index.html @@ -23,7 +23,7 @@ var d = new Date(); d.setTime(d.getTime() + (90*24*60*60*1000)); - document.cookie = 'default-theme-v2=' + theme + '; expires=' + d.toUTCString() + '; path=/'; + document.cookie = 'default-client=' + theme + '; expires=' + d.toUTCString() + '; path=/'; // Make an absolute path redirect, as relative redirects break if the port is non-standard var path = window.location.pathname; @@ -38,7 +38,7 @@ window.location.replace(path); } - var theme = parseCookies()['default-theme-v2']; + var theme = parseCookies()['default-client']; if ((theme || '').trim().length == 0) theme = 'ngclient'; diff --git a/Duplicati/Server/webroot/theme.html b/Duplicati/Server/webroot/theme.html index b389e4d6d..04500960c 100644 --- a/Duplicati/Server/webroot/theme.html +++ b/Duplicati/Server/webroot/theme.html @@ -50,7 +50,7 @@ } function updateToCurrentTheme() { - var theme = parseCookies()["default-theme-v2"]; + var theme = parseCookies()["default-client"]; var themes = ["ngax", "ngclient"]; for (var k in themes) { From 36ec595fd526c2119a08a37a0c8a85fe206864c0 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 12 Feb 2025 10:06:05 +0100 Subject: [PATCH 161/362] Renamed method to fit naming scheme --- Duplicati/Server/webroot/theme.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Duplicati/Server/webroot/theme.html b/Duplicati/Server/webroot/theme.html index 04500960c..08bbb54bd 100644 --- a/Duplicati/Server/webroot/theme.html +++ b/Duplicati/Server/webroot/theme.html @@ -49,7 +49,7 @@ return cookies; } - function updateToCurrentTheme() { + function updateToCurrentClient() { var theme = parseCookies()["default-client"]; var themes = ["ngax", "ngclient"]; @@ -68,7 +68,7 @@ } // Lazy hack for DOM timing issues... - window.setTimeout(updateToCurrentTheme, 1000); + window.setTimeout(updateToCurrentClient, 1000); @@ -99,7 +99,7 @@ From 8653a5d79a323c6f5abd9e879f320ea36a3dfdb3 Mon Sep 17 00:00:00 2001 From: "Marcelo C." Date: Thu, 13 Feb 2025 12:59:48 -0300 Subject: [PATCH 162/362] Adding longer timeout to CI for AzureBlobs --- .github/workflows/backendtests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/backendtests.yml b/.github/workflows/backendtests.yml index 059854010..78f67df04 100644 --- a/.github/workflows/backendtests.yml +++ b/.github/workflows/backendtests.yml @@ -257,6 +257,7 @@ jobs: LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln - name: Run Azure tests env: + READ_WRITE_TIMEOUT_SECONDS: 18000 TESTCREDENTIAL_AZURE_ACCOUNTNAME: "${{ secrets.TESTCREDENTIAL_AZURE_ACCOUNTNAME }}" TESTCREDENTIAL_AZURE_ACCESSKEY: "${{ secrets.TESTCREDENTIAL_AZURE_ACCESSKEY }}" TESTCREDENTIAL_AZURE_CONTAINERNAME: "${{ secrets.TESTCREDENTIAL_AZURE_CONTAINERNAME }}" From c05ca31d6b4a76facf877739f9c1f1192c7ee960 Mon Sep 17 00:00:00 2001 From: "Marcelo C." Date: Thu, 13 Feb 2025 15:51:39 -0300 Subject: [PATCH 163/362] Adding backend CI status on readme --- LiveTests/Duplicati.Backend.Tests/README.md | 32 +++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/LiveTests/Duplicati.Backend.Tests/README.md b/LiveTests/Duplicati.Backend.Tests/README.md index d1d9561eb..06002ab40 100644 --- a/LiveTests/Duplicati.Backend.Tests/README.md +++ b/LiveTests/Duplicati.Backend.Tests/README.md @@ -12,6 +12,38 @@ MAX_FILE_SIZE default is 1000kb NUMBER_OF_FILES default is 20 ``` +## Backends CI Status + +| Backend | CI Planned | CI Status | +|----------|------------|------------| +| pCloud | Planned | Live | +| WebDav | Planned | Live | +| Shhv2 | Planned | Live | +| S3 | Planned | Live | +| Ftp | Planned | Live | +| Planned | Planned | Live | +| Dropbox | Planned | Live | +| AlternativeFTP | Planned | Live | +| AzureBlob | Planned | Live | +| CIFS | Planned | Live | +| TahoeLAFS | Pending Decision | | +| TescentCOS | Pending Decision | | +| Storj | Pending Decision | | +| Sia | Pending Decision | | +| SharePoint | Pending Decision | | +| Rclone | Pending Decision | | +| OneDrive | Pending Decision | | +| OpenStack | Pending Decision | | +| Mega | Pending Decision | | +| Jottacloud | Pending Decision | | +| IDrive2 | Planned | | +| File | Planned | | +| Box | Pending Decision | | +| CloudFiles | Pending Decision | | +| backBlaze | Planned | | +| AliyunOSS | Pending Decision | | + + ## Backends that do not require Environment variables * FTP _(TestContainers required)_ From 24828d633b13b17d739f4417c840b6d2c57fde4b Mon Sep 17 00:00:00 2001 From: "Marcelo C." Date: Thu, 13 Feb 2025 15:53:46 -0300 Subject: [PATCH 164/362] CIFS CI is not live, its in development --- LiveTests/Duplicati.Backend.Tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LiveTests/Duplicati.Backend.Tests/README.md b/LiveTests/Duplicati.Backend.Tests/README.md index 06002ab40..824d8e70a 100644 --- a/LiveTests/Duplicati.Backend.Tests/README.md +++ b/LiveTests/Duplicati.Backend.Tests/README.md @@ -25,7 +25,7 @@ NUMBER_OF_FILES default is 20 | Dropbox | Planned | Live | | AlternativeFTP | Planned | Live | | AzureBlob | Planned | Live | -| CIFS | Planned | Live | +| CIFS | Planned | | | TahoeLAFS | Pending Decision | | | TescentCOS | Pending Decision | | | Storj | Pending Decision | | From f38d9a3d7ef927e035f673e35d4105cd3fe0f78e Mon Sep 17 00:00:00 2001 From: marceloduplicati Date: Thu, 13 Feb 2025 17:01:58 -0300 Subject: [PATCH 165/362] Terminal Adding CIFS to CI (#5971) Adding CIFS to CI --- .github/workflows/backendtests.yml | 37 +++++++++++++++++++ .../Duplicati.Backend.Tests/CIFS/CIFSTests.cs | 3 ++ LiveTests/Duplicati.Backend.Tests/README.md | 3 +- 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/.github/workflows/backendtests.yml b/.github/workflows/backendtests.yml index 78f67df04..7d380f725 100644 --- a/.github/workflows/backendtests.yml +++ b/.github/workflows/backendtests.yml @@ -416,3 +416,40 @@ jobs: dotnet test --no-build --filter="ClassName~GoogleDriveTests" --logger:"console;verbosity=detailed" LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln + test_cifs: + needs: check_secrets + if: needs.check_secrets.outputs.testcontainers_secrets_available == 'true' + name: CIFS Tests + runs-on: "${{ matrix.os }}" + strategy: + max-parallel: 1 + fail-fast: false + matrix: + os: + - ubuntu-latest + - windows-latest + - macos-latest + steps: + - name: Set up .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 8.x + - name: Checkout source + uses: actions/checkout@v4 + - name: Restore NuGet dependencies + run: >- + dotnet restore + LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln + - name: Build project + run: >- + dotnet build --no-restore + LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln + - name: Setup Testcontainers Cloud Client + uses: atomicjar/testcontainers-cloud-setup-action@v1 + with: + token: "${{ secrets.TC_CLOUD_TOKEN }}" + - name: Run CIFS tests + run: >- + dotnet test --no-build --filter="ClassName~CIFSTests" + --logger:"console;verbosity=detailed" + LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln \ No newline at end of file diff --git a/LiveTests/Duplicati.Backend.Tests/CIFS/CIFSTests.cs b/LiveTests/Duplicati.Backend.Tests/CIFS/CIFSTests.cs index cedf8ac0b..d7547f52b 100644 --- a/LiveTests/Duplicati.Backend.Tests/CIFS/CIFSTests.cs +++ b/LiveTests/Duplicati.Backend.Tests/CIFS/CIFSTests.cs @@ -19,6 +19,8 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +using DotNet.Testcontainers.Images; + namespace Duplicati.Backend.Tests.CIFS; /// @@ -71,6 +73,7 @@ smbd --foreground --no-process-group --debug-stdout"; var container = new ContainerBuilder() .WithImage("ubuntu:22.04") + .WithImagePullPolicy(PullPolicy.Missing) .WithCommand("/bin/bash", "-c", "apt-get update && " + "DEBIAN_FRONTEND=noninteractive apt-get install -y samba && " + "bash /etc/samba/entrypoint.sh") diff --git a/LiveTests/Duplicati.Backend.Tests/README.md b/LiveTests/Duplicati.Backend.Tests/README.md index 824d8e70a..4b1d21ed1 100644 --- a/LiveTests/Duplicati.Backend.Tests/README.md +++ b/LiveTests/Duplicati.Backend.Tests/README.md @@ -25,7 +25,8 @@ NUMBER_OF_FILES default is 20 | Dropbox | Planned | Live | | AlternativeFTP | Planned | Live | | AzureBlob | Planned | Live | -| CIFS | Planned | | +| CIFS | Planned | Live | +| Google Drive | Planned | Live | | TahoeLAFS | Pending Decision | | | TescentCOS | Pending Decision | | | Storj | Pending Decision | | From 3fbdeb65fa76f599487abdb1fe74069b502f29f0 Mon Sep 17 00:00:00 2001 From: "Marcelo C." Date: Thu, 13 Feb 2025 17:41:05 -0300 Subject: [PATCH 166/362] Disabling unsupported platforms for CI Currently the CIFS Tests on macos and windows require it to run in cloud mode, which means the port binding would not match default netbios/directtcp protocol standards as the container port allocation is random and the backend does not support specifying explicit ports. The remaining tests on linux nevertheless already grants some degree of assurance the backend is working. --- .github/workflows/backendtests.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/backendtests.yml b/.github/workflows/backendtests.yml index 7d380f725..9d84de249 100644 --- a/.github/workflows/backendtests.yml +++ b/.github/workflows/backendtests.yml @@ -427,8 +427,6 @@ jobs: matrix: os: - ubuntu-latest - - windows-latest - - macos-latest steps: - name: Set up .NET uses: actions/setup-dotnet@v4 @@ -444,10 +442,6 @@ jobs: run: >- dotnet build --no-restore LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln - - name: Setup Testcontainers Cloud Client - uses: atomicjar/testcontainers-cloud-setup-action@v1 - with: - token: "${{ secrets.TC_CLOUD_TOKEN }}" - name: Run CIFS tests run: >- dotnet test --no-build --filter="ClassName~CIFSTests" From 9501470d0d4a35ef2a031d3e77a9527ce4add983 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 14 Feb 2025 11:28:22 +0100 Subject: [PATCH 167/362] Added listen interface to startup message --- Duplicati/Library/RestAPI/Strings.cs | 2 +- Duplicati/Server/Program.cs | 4 ++-- Duplicati/WebserverCore/DuplicatiWebserver.cs | 15 ++++++++++++++- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/Duplicati/Library/RestAPI/Strings.cs b/Duplicati/Library/RestAPI/Strings.cs index e7321da48..eed6a0ef4 100644 --- a/Duplicati/Library/RestAPI/Strings.cs +++ b/Duplicati/Library/RestAPI/Strings.cs @@ -87,7 +87,7 @@ Error message: {0}", error); } public static string WindowsEventLogSourceNotFound(string source) { return LC.L(@"The Windows event log source {0} was not found. Attempting to create.", source); } public static string WindowsEventLogSourceNotCreated(string source) { return LC.L(@"The Windows Event Log was not created for: {0}, not logging to eventlog.", source); } public static string WindowsEventLogNotSupported { get { return LC.L(@"The Windows event log is not supported on this platform"); } } - public static string ServerStarted(int port) { return LC.L(@"Server has started and is listening on port {0}", port); } + public static string ServerStarted(string @interface, int port) { return LC.L(@"Server has started and is listening on {0}, port {1}", @interface, port); } public static string ServerStartedSignin(string url) { return LC.L(@"Use the following link to sign in: {0}", url); } public static string ServerCrashed(string message) { return LC.L(@"The server crashed: {0}", message); } public static string ServerStopping { get { return LC.L(@"Server is stopping, tearing down handlers"); } } diff --git a/Duplicati/Server/Program.cs b/Duplicati/Server/Program.cs index 33aedb0bf..5af27775d 100644 --- a/Duplicati/Server/Program.cs +++ b/Duplicati/Server/Program.cs @@ -288,8 +288,8 @@ namespace Duplicati.Server DataConnection.SetPreloadSettingsIfChanged(preloadDbSettings); EmitWarningsForConfigurationIssues(commandlineOptions); - Library.Logging.Log.WriteInformationMessage(LOGTAG, "ServerStarted", Strings.Program.ServerStarted(DuplicatiWebserver.Port)); - logMessageToConsole(Strings.Program.ServerStarted(DuplicatiWebserver.Port)); + Library.Logging.Log.WriteInformationMessage(LOGTAG, "ServerStarted", Strings.Program.ServerStarted(DuplicatiWebserver.Interface, DuplicatiWebserver.Port)); + logMessageToConsole(Strings.Program.ServerStarted(DuplicatiWebserver.Interface, DuplicatiWebserver.Port)); if (FIXMEGlobal.Origin == "Server" && DataConnection.ApplicationSettings.AutogeneratedPassphrase) { diff --git a/Duplicati/WebserverCore/DuplicatiWebserver.cs b/Duplicati/WebserverCore/DuplicatiWebserver.cs index 9f5f0b4fc..b92c5182a 100644 --- a/Duplicati/WebserverCore/DuplicatiWebserver.cs +++ b/Duplicati/WebserverCore/DuplicatiWebserver.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2025, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a @@ -73,6 +73,11 @@ public class DuplicatiWebserver /// public required int Port { get; init; } + /// + /// The interfaces being listened to + /// + public required string Interface { get; init; } + /// /// The port the server is listening on /// @@ -335,11 +340,19 @@ public class DuplicatiWebserver // Preload static system info, for better first-load experience var _ = Task.Run(() => app.Services.GetRequiredService().GetSystemInfo(null)); + // Get a string description of the listen interface used + var listenInterface = settings.Interface == System.Net.IPAddress.Any + ? "*" + : settings.Interface == System.Net.IPAddress.Loopback + ? "localhost" + : settings.Interface.ToString(); + return new DuplicatiWebserver() { Configuration = builder.Configuration, App = app, Port = settings.Port, + Interface = listenInterface, CorsEnabled = useCors }; From 356d117b86e74939b4d990973e346be720e67748 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sun, 16 Feb 2025 16:08:28 +0100 Subject: [PATCH 168/362] Ensure directory paths end with a separator --- Duplicati/Library/Snapshots/LinuxSnapshot.cs | 4 ++-- Duplicati/Library/Snapshots/NoSnapshotLinux.cs | 2 +- Duplicati/Library/Snapshots/NoSnapshotWindows.cs | 2 +- Duplicati/Library/Snapshots/SnapshotBase.cs | 5 +++-- Duplicati/Library/Snapshots/WindowsSnapshot.cs | 2 +- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/Duplicati/Library/Snapshots/LinuxSnapshot.cs b/Duplicati/Library/Snapshots/LinuxSnapshot.cs index 9d5a51832..0797f5108 100644 --- a/Duplicati/Library/Snapshots/LinuxSnapshot.cs +++ b/Duplicati/Library/Snapshots/LinuxSnapshot.cs @@ -119,7 +119,7 @@ namespace Duplicati.Library.Snapshots /// A flag indicating if the path is a folder /// The filesystem entry public override ISourceFileEntry GetFilesystemEntry(string path, bool isFolder) - => new SnapshotSourceFileEntry(this, path, isFolder, false); + => new SnapshotSourceFileEntry(this, isFolder ? Util.AppendDirSeparator(path) : path, isFolder, false); /// /// Enumerates the root source files and folders @@ -130,7 +130,7 @@ namespace Duplicati.Library.Snapshots foreach (var snap in m_snapShots) { if (DirectoryExists(snap.LocalPath) || snap.LocalPath.EndsWith(System.IO.Path.DirectorySeparatorChar)) - yield return new SnapshotSourceFileEntry(this, snap.LocalPath, true, true); + yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(snap.LocalPath), true, true); else yield return new SnapshotSourceFileEntry(this, snap.LocalPath, false, true); } diff --git a/Duplicati/Library/Snapshots/NoSnapshotLinux.cs b/Duplicati/Library/Snapshots/NoSnapshotLinux.cs index 3bad638eb..347fc539d 100644 --- a/Duplicati/Library/Snapshots/NoSnapshotLinux.cs +++ b/Duplicati/Library/Snapshots/NoSnapshotLinux.cs @@ -98,7 +98,7 @@ namespace Duplicati.Library.Snapshots foreach (var folder in m_folders) { if (DirectoryExists(folder) || folder.EndsWith(Path.DirectorySeparatorChar)) - yield return new SnapshotSourceFileEntry(this, folder, true, true); + yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(folder), true, true); else yield return new SnapshotSourceFileEntry(this, folder, false, true); } diff --git a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs index 706554cfd..67c21a282 100644 --- a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs +++ b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs @@ -89,7 +89,7 @@ namespace Duplicati.Library.Snapshots foreach (var folder in m_folders.Select(SystemIOWindows.AddExtendedDevicePathPrefix)) { if (DirectoryExists(folder) || folder.EndsWith(System.IO.Path.DirectorySeparatorChar)) - yield return new SnapshotSourceFileEntry(this, folder, true, true); + yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(folder), true, true); else yield return new SnapshotSourceFileEntry(this, folder, false, true); } diff --git a/Duplicati/Library/Snapshots/SnapshotBase.cs b/Duplicati/Library/Snapshots/SnapshotBase.cs index 1c6151137..0c1516790 100644 --- a/Duplicati/Library/Snapshots/SnapshotBase.cs +++ b/Duplicati/Library/Snapshots/SnapshotBase.cs @@ -25,6 +25,7 @@ using System.IO; using Duplicati.Library.Interface; using System.Threading.Tasks; using System.Threading; +using Duplicati.Library.Common.IO; namespace Duplicati.Library.Snapshots { @@ -72,7 +73,7 @@ namespace Duplicati.Library.Snapshots /// A flag indicating if the path is a folder /// The filesystem entry public virtual ISourceFileEntry GetFilesystemEntry(string path, bool isFolder) - => new SnapshotSourceFileEntry(this, path, isFolder, false); + => new SnapshotSourceFileEntry(this, isFolder ? Util.AppendDirSeparator(path) : path, isFolder, false); /// /// Enumerates files and folders in the given folder @@ -87,7 +88,7 @@ namespace Duplicati.Library.Snapshots yield return new SnapshotSourceFileEntry(this, f, false, false); foreach (var d in ListFolders(source.Path)) - yield return new SnapshotSourceFileEntry(this, d, true, false); + yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(d), true, false); } } diff --git a/Duplicati/Library/Snapshots/WindowsSnapshot.cs b/Duplicati/Library/Snapshots/WindowsSnapshot.cs index ae36027ec..cc05ca35e 100644 --- a/Duplicati/Library/Snapshots/WindowsSnapshot.cs +++ b/Duplicati/Library/Snapshots/WindowsSnapshot.cs @@ -120,7 +120,7 @@ namespace Duplicati.Library.Snapshots foreach (var folder in _vssBackupComponents.SnapshotLocalPaths) { if (folder.EndsWith(Path.DirectorySeparatorChar) || DirectoryExists(folder)) - yield return new SnapshotSourceFileEntry(this, folder, true, true); + yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(folder), true, true); else yield return new SnapshotSourceFileEntry(this, folder, false, true); } From d8f6b3370aafc9e2e1c6b07769e2573e65a691e3 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sun, 16 Feb 2025 16:09:20 +0100 Subject: [PATCH 169/362] Removed unused code --- Duplicati/UnitTest/TestUtils.cs | 62 ++++++++++----------------------- 1 file changed, 19 insertions(+), 43 deletions(-) diff --git a/Duplicati/UnitTest/TestUtils.cs b/Duplicati/UnitTest/TestUtils.cs index d8b4a37e8..596a193f1 100644 --- a/Duplicati/UnitTest/TestUtils.cs +++ b/Duplicati/UnitTest/TestUtils.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -24,25 +24,17 @@ using System.IO; using System.Collections.Generic; using Duplicati.Library.Utility; using System.Linq; -using Duplicati.Library.Logging; using System.Reflection; using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Common.IO; using NUnit.Framework; -using System.Runtime.InteropServices; -using Duplicati.Library.Common; using Duplicati.Library.Interface; namespace Duplicati.UnitTest { public static class TestUtils { - /// - /// The log tag - /// - private static readonly string LOGTAG = Library.Logging.Log.LogTagFromType(typeof(TestUtils)); - public static Dictionary DefaultOptions { get @@ -161,22 +153,6 @@ namespace Duplicati.UnitTest Console.WriteLine("Encountered additional {0} timestamp errors!", timestampfailures); } - /// - /// Returns the index of a given string, using the file system case sensitivity - /// - /// The index of the entry or -1 if no entry was found - /// The list to search - /// The string to find - private static int IndexOf(List lst, string m) - { - StringComparison sc = Duplicati.Library.Utility.Utility.ClientFilenameStringComparison; - for (int i = 0; i < lst.Count; i++) - if (lst[i].Equals(m, sc)) - return i; - - return -1; - } - /// /// Asserts that the two directory trees are equivalent; i.e., /// that they they contain the same directories and files, recursively. From 8cead82274d4002b950ba91e9d7c8fbd879270b0 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sun, 16 Feb 2025 16:09:48 +0100 Subject: [PATCH 170/362] Moved a sanity check up --- Duplicati/UnitTest/SVNCheckoutsTest.cs | 52 +++++++++++++------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/Duplicati/UnitTest/SVNCheckoutsTest.cs b/Duplicati/UnitTest/SVNCheckoutsTest.cs index 22a74dccf..a7808cf7e 100644 --- a/Duplicati/UnitTest/SVNCheckoutsTest.cs +++ b/Duplicati/UnitTest/SVNCheckoutsTest.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. @@ -406,6 +406,13 @@ namespace Duplicati.UnitTest restoredname = System.IO.Path.Combine(System.IO.Path.Combine(tempfolder, System.IO.Path.GetFileName(rootfolder.Split(System.IO.Path.PathSeparator)[int.Parse(s.Substring(0, six))])), s.Substring(six + 1)); } + if (!System.IO.File.Exists(sourcename)) + { + Log.WriteErrorMessage(LOGTAG, "PartialRestoreMissingFile", null, "Partial restore, missing SOURCE file: {0}", sourcename); + BasicSetupHelper.ProgressWriteLine("Partial restore missing file: " + sourcename); + throw new Exception("Unittest is broken"); + } + if (!System.IO.File.Exists(restoredname)) { Log.WriteErrorMessage(LOGTAG, "PartialRestoreMissingFile", null, "Partial restore missing file: {0}", restoredname); @@ -413,13 +420,6 @@ namespace Duplicati.UnitTest } else { - if (!System.IO.File.Exists(sourcename)) - { - Log.WriteErrorMessage(LOGTAG, "PartialRestoreMissingFile", null, "Partial restore missing file: {0}", sourcename); - BasicSetupHelper.ProgressWriteLine("Partial restore missing file: " + sourcename); - throw new Exception("Unittest is broken"); - } - TestUtils.AssertFilesAreEqual(sourcename, restoredname, verifymetadata, $"Partial restore file differs: {s}"); } } From 4a9ad45250fa4bf5376cf2ec4e55a348ed07c269 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sun, 16 Feb 2025 16:33:15 +0100 Subject: [PATCH 171/362] Repaired import/export test --- Duplicati/UnitTest/ImportExportTests.cs | 90 ++++++++++++++----------- 1 file changed, 51 insertions(+), 39 deletions(-) diff --git a/Duplicati/UnitTest/ImportExportTests.cs b/Duplicati/UnitTest/ImportExportTests.cs index 9663d5ec1..26062445c 100644 --- a/Duplicati/UnitTest/ImportExportTests.cs +++ b/Duplicati/UnitTest/ImportExportTests.cs @@ -1,28 +1,29 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; using System.IO; using Duplicati.Library.RestAPI; +using Duplicati.Library.SQLiteHelper; using Duplicati.Server; using Duplicati.Server.Database; using Duplicati.Server.Serializable; @@ -126,46 +127,57 @@ namespace Duplicati.UnitTest [Category("ImportExport")] public void RoundTrip() { - Dictionary metadata = new Dictionary { { "SourceFilesCount", "1" } }; - Dictionary advancedOptions = new Dictionary { { "server-datafolder", this.serverDatafolder } }; + var metadata = new Dictionary { { "SourceFilesCount", "1" } }; + var advancedOptions = new Dictionary { { "server-datafolder", this.serverDatafolder } }; + + // Mock the setup enough to get the import/export working var serviceCollection = new ServiceCollection(); serviceCollection.AddSingleton(); serviceCollection.AddSingleton(new EventPollNotify()); FIXMEGlobal.Provider = new DefaultServiceProviderFactory().CreateServiceProvider(serviceCollection); + FIXMEGlobal.DataFolder = this.serverDatafolder; - using (Program.DataConnection = Program.GetDatabaseConnection(advancedOptions, true)) + var dbpath = Path.Combine(this.serverDatafolder, Library.AutoUpdater.DataFolderManager.SERVER_DATABASE_FILENAME); + if (File.Exists(dbpath)) + File.Delete(dbpath); + + var con = SQLiteLoader.LoadConnection(); + SQLiteLoader.OpenDatabase(con, dbpath, null); + DatabaseUpgrader.UpgradeDatabase(con, dbpath, typeof(Duplicati.Library.RestAPI.Database.DatabaseConnectionSchemaMarker)); + + using (var connection = new Connection(con, true, null)) { // Unencrypted file, don't import metadata. string unencryptedWithoutMetadata = Path.Combine(this.serverDatafolder, Path.GetRandomFileName()); - File.WriteAllBytes(unencryptedWithoutMetadata, BackupImportExportHandler.ExportToJSON(Program.DataConnection, this.CreateBackup("unencrypted without metadata", "user", "password", metadata), null)); - BackupImportExportHandler.ImportBackup(Program.DataConnection, unencryptedWithoutMetadata, false, () => null); - Assert.AreEqual(1, Program.DataConnection.Backups.Length); - Assert.AreEqual(0, Program.DataConnection.Backups[0].Metadata.Count); + File.WriteAllBytes(unencryptedWithoutMetadata, BackupImportExportHandler.ExportToJSON(connection, this.CreateBackup("unencrypted without metadata", "user", "password", metadata), null)); + BackupImportExportHandler.ImportBackup(connection, unencryptedWithoutMetadata, false, () => null); + Assert.AreEqual(1, connection.Backups.Length); + Assert.AreEqual(0, connection.Backups[0].Metadata.Count); // Unencrypted file, import metadata. string unencryptedWithMetadata = Path.Combine(this.serverDatafolder, Path.GetRandomFileName()); - File.WriteAllBytes(unencryptedWithMetadata, BackupImportExportHandler.ExportToJSON(Program.DataConnection, this.CreateBackup("unencrypted with metadata", "user", "password", metadata), null)); - BackupImportExportHandler.ImportBackup(Program.DataConnection, unencryptedWithMetadata, true, () => null); - Assert.AreEqual(2, Program.DataConnection.Backups.Length); - Assert.AreEqual(metadata.Count, Program.DataConnection.Backups[1].Metadata.Count); + File.WriteAllBytes(unencryptedWithMetadata, BackupImportExportHandler.ExportToJSON(connection, this.CreateBackup("unencrypted with metadata", "user", "password", metadata), null)); + BackupImportExportHandler.ImportBackup(connection, unencryptedWithMetadata, true, () => null); + Assert.AreEqual(2, connection.Backups.Length); + Assert.AreEqual(metadata.Count, connection.Backups[1].Metadata.Count); // Encrypted file, don't import metadata. string encryptedWithoutMetadata = Path.Combine(this.serverDatafolder, Path.GetRandomFileName()); string passphrase = "abcde"; - File.WriteAllBytes(encryptedWithoutMetadata, BackupImportExportHandler.ExportToJSON(Program.DataConnection, this.CreateBackup("encrypted without metadata", "user", "password", metadata), passphrase)); - BackupImportExportHandler.ImportBackup(Program.DataConnection, encryptedWithoutMetadata, false, () => passphrase); - Assert.AreEqual(3, Program.DataConnection.Backups.Length); - Assert.AreEqual(0, Program.DataConnection.Backups[2].Metadata.Count); + File.WriteAllBytes(encryptedWithoutMetadata, BackupImportExportHandler.ExportToJSON(connection, this.CreateBackup("encrypted without metadata", "user", "password", metadata), passphrase)); + BackupImportExportHandler.ImportBackup(connection, encryptedWithoutMetadata, false, () => passphrase); + Assert.AreEqual(3, connection.Backups.Length); + Assert.AreEqual(0, connection.Backups[2].Metadata.Count); // Encrypted file, import metadata. string encryptedWithMetadata = Path.Combine(this.serverDatafolder, Path.GetRandomFileName()); - File.WriteAllBytes(encryptedWithMetadata, BackupImportExportHandler.ExportToJSON(Program.DataConnection, this.CreateBackup("encrypted with metadata", "user", "password", metadata), passphrase)); - BackupImportExportHandler.ImportBackup(Program.DataConnection, encryptedWithMetadata, true, () => passphrase); - Assert.AreEqual(4, Program.DataConnection.Backups.Length); - Assert.AreEqual(metadata.Count, Program.DataConnection.Backups[3].Metadata.Count); + File.WriteAllBytes(encryptedWithMetadata, BackupImportExportHandler.ExportToJSON(connection, this.CreateBackup("encrypted with metadata", "user", "password", metadata), passphrase)); + BackupImportExportHandler.ImportBackup(connection, encryptedWithMetadata, true, () => passphrase); + Assert.AreEqual(4, connection.Backups.Length); + Assert.AreEqual(metadata.Count, connection.Backups[3].Metadata.Count); // Encrypted file, incorrect passphrase. - Assert.Throws(Is.InstanceOf(), () => BackupImportExportHandler.ImportBackup(Program.DataConnection, encryptedWithMetadata, true, () => passphrase + " ")); + Assert.Throws(Is.InstanceOf(), () => BackupImportExportHandler.ImportBackup(connection, encryptedWithMetadata, true, () => passphrase + " ")); } } } From a621bb3d1419e63609b5d4afe2822582db838f25 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 17 Feb 2025 08:19:19 +0100 Subject: [PATCH 172/362] Fixed issue after merge --- Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs b/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs index 6ae8a85c3..02a983aac 100644 --- a/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs +++ b/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs @@ -120,7 +120,7 @@ namespace Duplicati.Library.Main.Operation.Backup else if (e.TimestampChanged) { await stats.AddTimestampChangedFile(); - Logging.Log.WriteVerboseMessage(FILELOGTAG, "FileTimestampChanged", "File has only timestamp changes {0}", e.Path); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "FileTimestampChanged", "File has only timestamp changes {0}", e.Entry.Path); await database.AddUnmodifiedAsync(e.OldId, e.LastWrite); } else /*if (e.OldId >= 0)*/ From 989ba602117857abfbbeea254a8241228c30ff61 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 17 Feb 2025 08:52:01 +0100 Subject: [PATCH 173/362] Updated license copyright years --- Duplicati/Library/DynamicLoader/SourceProviderLoader.cs | 2 +- .../Library/SourceProvider/Builtin/BackendSourceProvider.cs | 2 +- Duplicati/Library/SourceProvider/Builtin/Combiner.cs | 2 +- Duplicati/Library/SourceProvider/Builtin/File.cs | 2 +- Duplicati/Library/SourceProvider/Builtin/Strings.cs | 2 +- Duplicati/Library/SourceProviders/SourceProviderModules.cs | 2 +- ReleaseBuilder/Resources/Windows/Agent/License.rtf | 2 +- ReleaseBuilder/Resources/Windows/TrayIcon/License.rtf | 2 +- thirdparty/SharpAESCrypt/License.txt | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs b/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs index 5b38ac617..23920c594 100644 --- a/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs +++ b/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2024, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a diff --git a/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs index 54ad99034..9b144a7ab 100644 --- a/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs +++ b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2024, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a diff --git a/Duplicati/Library/SourceProvider/Builtin/Combiner.cs b/Duplicati/Library/SourceProvider/Builtin/Combiner.cs index 5ef6496ba..1dcb01fa8 100644 --- a/Duplicati/Library/SourceProvider/Builtin/Combiner.cs +++ b/Duplicati/Library/SourceProvider/Builtin/Combiner.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2024, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a diff --git a/Duplicati/Library/SourceProvider/Builtin/File.cs b/Duplicati/Library/SourceProvider/Builtin/File.cs index 82e43997a..985fc072c 100644 --- a/Duplicati/Library/SourceProvider/Builtin/File.cs +++ b/Duplicati/Library/SourceProvider/Builtin/File.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2024, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a diff --git a/Duplicati/Library/SourceProvider/Builtin/Strings.cs b/Duplicati/Library/SourceProvider/Builtin/Strings.cs index 5707fccd5..64cc9e118 100644 --- a/Duplicati/Library/SourceProvider/Builtin/Strings.cs +++ b/Duplicati/Library/SourceProvider/Builtin/Strings.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2024, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a diff --git a/Duplicati/Library/SourceProviders/SourceProviderModules.cs b/Duplicati/Library/SourceProviders/SourceProviderModules.cs index 1bf09d104..7bc0b612c 100644 --- a/Duplicati/Library/SourceProviders/SourceProviderModules.cs +++ b/Duplicati/Library/SourceProviders/SourceProviderModules.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2024, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a diff --git a/ReleaseBuilder/Resources/Windows/Agent/License.rtf b/ReleaseBuilder/Resources/Windows/Agent/License.rtf index 9b2b7cff7..7198f91ff 100644 --- a/ReleaseBuilder/Resources/Windows/Agent/License.rtf +++ b/ReleaseBuilder/Resources/Windows/Agent/License.rtf @@ -6,7 +6,7 @@ \deftab720 \pard\pardeftab720\ri0\sl380\sa240\partightenfactor0 -\f0\fs25 \cf0 Copyright (c) 2024 Team Duplicati, https://duplicati.com, hello@duplicati.com +\f0\fs25 \cf0 Copyright (c) 2025 Team Duplicati, https://duplicati.com, hello@duplicati.com \f1\fs24 \ \pard\pardeftab720\ri0\sl380\sb240\sa240\partightenfactor0 diff --git a/ReleaseBuilder/Resources/Windows/TrayIcon/License.rtf b/ReleaseBuilder/Resources/Windows/TrayIcon/License.rtf index 9b2b7cff7..7198f91ff 100644 --- a/ReleaseBuilder/Resources/Windows/TrayIcon/License.rtf +++ b/ReleaseBuilder/Resources/Windows/TrayIcon/License.rtf @@ -6,7 +6,7 @@ \deftab720 \pard\pardeftab720\ri0\sl380\sa240\partightenfactor0 -\f0\fs25 \cf0 Copyright (c) 2024 Team Duplicati, https://duplicati.com, hello@duplicati.com +\f0\fs25 \cf0 Copyright (c) 2025 Team Duplicati, https://duplicati.com, hello@duplicati.com \f1\fs24 \ \pard\pardeftab720\ri0\sl380\sb240\sa240\partightenfactor0 diff --git a/thirdparty/SharpAESCrypt/License.txt b/thirdparty/SharpAESCrypt/License.txt index 09905eb1a..4ac9a5e9c 100755 --- a/thirdparty/SharpAESCrypt/License.txt +++ b/thirdparty/SharpAESCrypt/License.txt @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2024 Duplicati +Copyright (c) 2025 Duplicati Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal From bb5fd80a29c7c8ee926523e806f54143f5843844 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 17 Feb 2025 14:01:08 +0100 Subject: [PATCH 174/362] Implemented support for some backends as sources: SSH, CIFS, and S3 --- Duplicati/Library/Backend/CIFS/CIFS.cs | 73 ++++-- .../Duplicati.Library.Backend.CIFS.csproj | 1 + .../Backend/CIFS/SMBShareConnection.cs | 8 +- Duplicati/Library/Backend/File/FileBackend.cs | 72 +----- .../Duplicati.Library.Backend.Idrivee2.csproj | 1 + .../Backend/Idrivee2/Idrivee2Backend.cs | 55 ++++- .../S3/Duplicati.Library.Backend.S3.csproj | 2 +- Duplicati/Library/Backend/S3/IS3Client.cs | 2 +- Duplicati/Library/Backend/S3/S3AwsClient.cs | 44 ++-- Duplicati/Library/Backend/S3/S3Backend.cs | 94 ++++++-- Duplicati/Library/Backend/S3/S3MinioClient.cs | 30 ++- .../Duplicati.Library.Backend.SSHv2.csproj | 1 + .../Library/Backend/SSHv2/SSHv2Backend.cs | 45 +++- Duplicati/Library/Backend/Storj/StorjFile.cs | 1 + .../Backend/pCloud/Model/pCloudFileEntry.cs | 13 +- Duplicati/Library/Common/IO/FileEntry.cs | 50 ++-- Duplicati/Library/Interface/IFileEntry.cs | 44 ++-- .../Interface/IFolderEnabledBackend.cs | 12 +- .../Library/Interface/ISourceProvider.cs | 7 +- .../Backup/FileEnumerationProcess.cs | 31 +-- .../Library/Main/Operation/BackupHandler.cs | 15 +- .../Main/Operation/FilelistProcessor.cs | 1 - .../Main/Operation/TestFilterHandler.cs | 19 +- .../Library/Snapshots/ISnapshotService.cs | 2 +- Duplicati/Library/Snapshots/LinuxSnapshot.cs | 9 - Duplicati/Library/Snapshots/SnapshotBase.cs | 9 +- .../Builtin/BackendSourceFileEntry.cs | 216 ++++++++++++++++++ .../Builtin/BackendSourceProvider.cs | 172 +------------- .../SourceProvider/Builtin/Combiner.cs | 22 +- .../Builtin/{File.cs => LocalFileSource.cs} | 7 +- 30 files changed, 667 insertions(+), 391 deletions(-) create mode 100644 Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs rename Duplicati/Library/SourceProvider/Builtin/{File.cs => LocalFileSource.cs} (86%) diff --git a/Duplicati/Library/Backend/CIFS/CIFS.cs b/Duplicati/Library/Backend/CIFS/CIFS.cs index 815773fad..eafcdfc45 100644 --- a/Duplicati/Library/Backend/CIFS/CIFS.cs +++ b/Duplicati/Library/Backend/CIFS/CIFS.cs @@ -30,13 +30,15 @@ using Duplicati.Library.Backend.CIFS; using Duplicati.Library.Backend.CIFS.Model; using Duplicati.Library.Utility; using SMBLibrary; +using Duplicati.Library.SourceProvider; +using System.Runtime.CompilerServices; namespace Duplicati.Library.Backend; /// /// Native CIFS/SMB Backend implementation /// -public class CIFSBackend : IStreamingBackend +public class CIFSBackend : IStreamingBackend, IFolderEnabledBackend { /// /// Implementation of interface property for the backend key @@ -58,6 +60,16 @@ public class CIFSBackend : IStreamingBackend /// private string _DnsName; + /// + /// Source provider to be used if the backend is a source + /// + private readonly BackendSourceProvider _sourceProvider; + + /// + /// Key to be used in the path for the source provider + /// + private readonly string _sourcePathKey; + /// /// Path separators (both Windows \ and unix /) to be used in path manipulation /// @@ -72,12 +84,12 @@ public class CIFSBackend : IStreamingBackend /// Shared connection between all methods to avoid re-authentication /// private SMBShareConnection _sharedConnection; - + /// /// Read buffer size for SMB operations (will be capped automatically by SMB negotiated values) /// private const string READ_BUFFER_SIZE_OPTION = "read-buffer-size"; - + /// /// Write buffer size for SMB operations (will be capped automatically by SMB negotiated values) /// @@ -139,33 +151,33 @@ public class CIFSBackend : IStreamingBackend var uri = new Utility.Uri(url); uri.RequireHost(); _DnsName = uri.Host; - + var input = uri.Path.TrimEnd('/'); var slashIndex = input.IndexOf('/'); // Find first slash to separate server and share if present. - + options.TryGetValue(AUTH_USERNAME_OPTION, out string authUsername); options.TryGetValue(AUTH_PASSWORD_OPTION, out string authPassword); options.TryGetValue(AUTH_DOMAIN_OPTION, out string authDomain); options.TryGetValue(TRANSPORT_OPTION, out string transport); - + int? readBufferSize = null, writeBufferSize = null; - + options.TryGetValue(READ_BUFFER_SIZE_OPTION, out string readBufferSizeConfig); if (!string.IsNullOrWhiteSpace(readBufferSizeConfig)) readBufferSize = Int32.TryParse(readBufferSizeConfig, out int value) ? value : null; - + options.TryGetValue(WRITE_BUFFER_SIZE_OPTION, out string writeBufferSizeConfig); if (!string.IsNullOrWhiteSpace(writeBufferSizeConfig)) writeBufferSize = Int32.TryParse(readBufferSizeConfig, out int value) ? value : null; - + // Normalize to 10KB minimum buffers size readBufferSize = readBufferSize < 1024 * 10 ? null : readBufferSize; writeBufferSize = writeBufferSize < 1024 * 10 ? null : writeBufferSize; SMBTransportType transportType = _transportMap.TryGetValue( - string.IsNullOrEmpty(transport) ? DEFAULT_TRANSPORT : transport.ToLower(), + string.IsNullOrEmpty(transport) ? DEFAULT_TRANSPORT : transport.ToLower(), out SMBTransportType type) ? type - : throw new UserInformationException($"Transport must be one of: {string.Join(", ", _transportMap.Keys)}","CIFSConfig"); - + : throw new UserInformationException($"Transport must be one of: {string.Join(", ", _transportMap.Keys)}", "CIFSConfig"); + _connectionParameters = new SMBConnectionParameters( uri.Host, transportType, @@ -177,6 +189,9 @@ public class CIFSBackend : IStreamingBackend readBufferSize, writeBufferSize ); + + _sourcePathKey = $"{ProtocolKey}://{_connectionParameters.ServerName}/~{_connectionParameters.AuthUser}/{_connectionParameters.Path}/"; + _sourceProvider = new BackendSourceProvider(this); } /// @@ -198,7 +213,7 @@ public class CIFSBackend : IStreamingBackend /// List of IFileEntry with directory listing result private async Task> ListAsync(CancellationToken cancellationToken) { - return await GetConnection().ListAsync(cancellationToken).ConfigureAwait(false); + return await GetConnection().ListAsync(_connectionParameters.Path, cancellationToken).ConfigureAwait(false); } /// @@ -282,7 +297,7 @@ public class CIFSBackend : IStreamingBackend /// /// CancellationToken, in this call not used. /// - public Task GetDNSNamesAsync(CancellationToken cancellationToken) => + public Task GetDNSNamesAsync(CancellationToken cancellationToken) => Task.FromResult(new[] { _DnsName ?? string.Empty }); /// @@ -293,7 +308,7 @@ public class CIFSBackend : IStreamingBackend public async Task TestAsync(CancellationToken cancellationToken) { // This will throw an exception if the folder is missing - await GetConnection().ListAsync(cancellationToken).ConfigureAwait(false); + await GetConnection().ListAsync(_connectionParameters.Path, cancellationToken).ConfigureAwait(false); } /// @@ -306,10 +321,10 @@ public class CIFSBackend : IStreamingBackend { var pathParts = _connectionParameters.Path? .Split(PATH_SEPARATORS, StringSplitOptions.RemoveEmptyEntries); - + if (pathParts == null || pathParts.Length == 0) return; - + await GetConnection().CreateFolderAsync(_connectionParameters.Path, cancellationToken) .ConfigureAwait(false); } @@ -320,6 +335,8 @@ public class CIFSBackend : IStreamingBackend /// An SMB connection that can be used for file operations private SMBShareConnection GetConnection() => _sharedConnection ??= new SMBShareConnection(_connectionParameters); + + /// /// Implementation of Dispose pattern enforced by interface /// @@ -335,4 +352,26 @@ public class CIFSBackend : IStreamingBackend System.Diagnostics.Debug.WriteLine($"Error disposing CIFS connection: {ex.Message}"); } } + + /// + public string PathKey => _sourcePathKey; + + /// + public async IAsyncEnumerable ListAsync(string path, [EnumeratorCancellation] CancellationToken cancellationToken) + { + var sourcePath = _connectionParameters.Path; + if (!sourcePath.EndsWith('\\')) + sourcePath += '\\'; + + var list = (await GetConnection().ListAsync(sourcePath + path, cancellationToken).ConfigureAwait(false)) + .Select(f => BackendSourceFileEntry.FromFileEntry(_sourceProvider, path, f)) + .Cast(); + + foreach (var v in list) + yield return v; + } + + /// + public Task GetEntryAsync(string path, CancellationToken cancellationToken) + => Task.FromResult(null); } \ No newline at end of file diff --git a/Duplicati/Library/Backend/CIFS/Duplicati.Library.Backend.CIFS.csproj b/Duplicati/Library/Backend/CIFS/Duplicati.Library.Backend.CIFS.csproj index b638e8235..11375a4ab 100644 --- a/Duplicati/Library/Backend/CIFS/Duplicati.Library.Backend.CIFS.csproj +++ b/Duplicati/Library/Backend/CIFS/Duplicati.Library.Backend.CIFS.csproj @@ -10,6 +10,7 @@ + diff --git a/Duplicati/Library/Backend/CIFS/SMBShareConnection.cs b/Duplicati/Library/Backend/CIFS/SMBShareConnection.cs index 44b936464..710e36ea2 100644 --- a/Duplicati/Library/Backend/CIFS/SMBShareConnection.cs +++ b/Duplicati/Library/Backend/CIFS/SMBShareConnection.cs @@ -194,9 +194,10 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable /// /// Lists the folder contents of the share and path specified in the connection parameters. /// + /// Path to list /// Cancellation Token /// Exception to be displayed to user - public async Task> ListAsync(CancellationToken cancellationToken) + public async Task> ListAsync(string path, CancellationToken cancellationToken) { await _semaphore.WaitAsync(cancellationToken); @@ -210,7 +211,7 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable var status = _smbFileStore.CreateFile( out directoryHandle, out fileStatus, - NormalizeSlashes(_connectionParameters.Path), + NormalizeSlashes(path), AccessMask.GENERIC_READ, FileAttributes.Directory, ShareAccess.Read | ShareAccess.Write, @@ -244,7 +245,8 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable info.LastAccessTime, info.LastWriteTime) { - IsFolder = info.FileAttributes == FileAttributes.Directory + IsFolder = info.FileAttributes == FileAttributes.Directory, + Created = info.CreationTime }) .ToList() ]; diff --git a/Duplicati/Library/Backend/File/FileBackend.cs b/Duplicati/Library/Backend/File/FileBackend.cs index de049627b..3a8b76ca8 100644 --- a/Duplicati/Library/Backend/File/FileBackend.cs +++ b/Duplicati/Library/Backend/File/FileBackend.cs @@ -468,6 +468,9 @@ namespace Duplicati.Library.Backend } } + /// + public string PathKey => m_path; + /// public async IAsyncEnumerable ListAsync(string path, [EnumeratorCancellation] CancellationToken cancellationToken) { @@ -481,10 +484,10 @@ namespace Duplicati.Library.Backend var service = SnapshotUtility.CreateNoSnapshot([remotePath], false, false); await foreach (var folderEntry in systemIO.EnumerateDirectories(remotePath).ToAsyncEnumerable().WithCancellation(cancellationToken).ConfigureAwait(false)) - yield return new SourceFileEntry(service, folderEntry, true); + yield return new SnapshotSourceFileEntry(service, folderEntry, true, false); await foreach (var fileEntry in systemIO.EnumerateFiles(remotePath).ToAsyncEnumerable().WithCancellation(cancellationToken).ConfigureAwait(false)) - yield return new SourceFileEntry(service, fileEntry, false); + yield return new SnapshotSourceFileEntry(service, fileEntry, false, false); } /// @@ -492,70 +495,7 @@ namespace Duplicati.Library.Backend { var service = SnapshotUtility.CreateNoSnapshot([path], false, false); var isFolder = service.DirectoryExists(path); - return Task.FromResult(new SourceFileEntry(service, path, isFolder)); - } - - private class SourceFileEntry(ISnapshotService service, string path, bool isFolder) : ISourceFileEntry - { - /// - public bool IsFolder => isFolder; - - /// - public bool IsMetaEntry => false; - /// - public bool IsRootEntry => false; - - /// - public DateTime CreatedUtc => service.GetCreationTimeUtc(path); - - /// - public DateTime LastModificationUtc => service.GetLastWriteTimeUtc(path); - - /// - public string Path => path; - - /// - public long Size => isFolder ? -1 : service.GetFileSize(path); - - /// - public bool IsSymlink => service.IsSymlink(path, Attributes); - - /// - public string SymlinkTarget => service.GetSymlinkTarget(path); - - /// - public FileAttributes Attributes => service.GetAttributes(path); - - /// - public Dictionary MinorMetadata => service.GetMetadata(path, IsSymlink); - - /// - public bool IsBlockDevice => service.IsBlockDevice(path); - - /// - public bool IsCharacterDevice => false; - - /// - public bool IsAlternateStream => false; - - /// - public string HardlinkTargetId => service.HardlinkTargetID(path); - - /// - public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) - => service.EnumerateFilesystemEntries(this).ToAsyncEnumerable(); - - /// - public Task FileExists(string filename, CancellationToken cancellationToken) - => Task.FromResult(service.FileExists(systemIO.PathCombine(Path, filename))); - - /// - public Task OpenMetadataRead(CancellationToken cancellationToken) - => Task.FromResult(new MemoryStream(System.Text.Json.JsonSerializer.SerializeToUtf8Bytes(MinorMetadata))); - - /// - public Task OpenRead(CancellationToken cancellationToken) - => service.OpenReadAsync(path, cancellationToken); + return Task.FromResult(new SnapshotSourceFileEntry(service, path, isFolder, false)); } } } diff --git a/Duplicati/Library/Backend/Idrivee2/Duplicati.Library.Backend.Idrivee2.csproj b/Duplicati/Library/Backend/Idrivee2/Duplicati.Library.Backend.Idrivee2.csproj index c8664ccd2..756380cd2 100644 --- a/Duplicati/Library/Backend/Idrivee2/Duplicati.Library.Backend.Idrivee2.csproj +++ b/Duplicati/Library/Backend/Idrivee2/Duplicati.Library.Backend.Idrivee2.csproj @@ -8,6 +8,7 @@ + diff --git a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs index 59dca09bd..f495ddf52 100644 --- a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs +++ b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs @@ -21,17 +21,18 @@ using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; -using Duplicati.Library.Utility; +using Duplicati.Library.SourceProvider; using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; namespace Duplicati.Library.Backend { - public class Idrivee2Backend : IBackend, IStreamingBackend + public class Idrivee2Backend : IBackend, IStreamingBackend, IFolderEnabledBackend { private static readonly string LOGTAG = Logging.Log.LogTagFromType(); @@ -43,6 +44,15 @@ namespace Duplicati.Library.Backend private readonly string m_prefix; private readonly string m_bucket; + /// + /// The source provider for this backend + /// + private readonly BackendSourceProvider m_sourceProvider; + /// + /// The key the backend is identified by if it is a source + /// + private readonly string m_sourcePathKey; + private IS3Client m_s3Client; public Idrivee2Backend() @@ -80,7 +90,8 @@ namespace Duplicati.Library.Backend m_s3Client = new S3AwsClient(accessKeyId, accessKeySecret, null, host, null, true, false, options); - + m_sourcePathKey = $"{ProtocolKey}://{host}/{m_bucket}/{m_prefix}/"; + m_sourceProvider = new BackendSourceProvider(this); } public string GetRegionEndpoint(string url) @@ -127,8 +138,11 @@ namespace Duplicati.Library.Backend public IEnumerable List() + => ListAsync(CancellationToken.None).ToBlockingEnumerable(); + + public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { - foreach (IFileEntry file in Connection.ListBucket(m_bucket, m_prefix)) + await foreach (IFileEntry file in Connection.ListBucketAsync(m_bucket, m_prefix, cancelToken).ConfigureAwait(false)) { ((FileEntry)file).Name = file.Name.Substring(m_prefix.Length); if (file.Name.StartsWith("/", StringComparison.Ordinal) && !m_prefix.StartsWith("/", StringComparison.Ordinal)) @@ -230,5 +244,38 @@ namespace Duplicati.Library.Backend //AWS SDK encodes the filenames correctly return m_prefix + name; } + + /// + public string PathKey => m_sourcePathKey; + + /// + public async IAsyncEnumerable ListAsync(string path, [EnumeratorCancellation] CancellationToken cancellationToken) + { + var prefixPath = m_prefix; + if (!prefixPath.EndsWith("/", StringComparison.Ordinal)) + prefixPath += "/"; + + var filterPath = prefixPath + path; + if (!filterPath.EndsWith("/", StringComparison.Ordinal)) + filterPath += "/"; + + await foreach (var f in m_s3Client.ListBucketAsync(m_bucket, filterPath, cancellationToken).ConfigureAwait(false)) + { + if (!f.Name.StartsWith(prefixPath, StringComparison.Ordinal)) + continue; + + ((FileEntry)f).Name = f.Name.Substring(prefixPath.Length); + + // Skip sub-folder items + if (f.Name.Contains("/")) + continue; + + yield return BackendSourceFileEntry.FromFileEntry(m_sourceProvider, path, f); + } + } + + /// + public Task GetEntryAsync(string path, CancellationToken cancellationToken) + => Task.FromResult(null); } } diff --git a/Duplicati/Library/Backend/S3/Duplicati.Library.Backend.S3.csproj b/Duplicati/Library/Backend/S3/Duplicati.Library.Backend.S3.csproj index 90016385f..148804d3b 100644 --- a/Duplicati/Library/Backend/S3/Duplicati.Library.Backend.S3.csproj +++ b/Duplicati/Library/Backend/S3/Duplicati.Library.Backend.S3.csproj @@ -12,10 +12,10 @@ - + diff --git a/Duplicati/Library/Backend/S3/IS3Client.cs b/Duplicati/Library/Backend/S3/IS3Client.cs index fee68361b..6d32833ee 100644 --- a/Duplicati/Library/Backend/S3/IS3Client.cs +++ b/Duplicati/Library/Backend/S3/IS3Client.cs @@ -28,7 +28,7 @@ namespace Duplicati.Library.Backend { public interface IS3Client : IDisposable { - IEnumerable ListBucket(string bucketName, string prefix); + IAsyncEnumerable ListBucketAsync(string bucketName, string prefix, CancellationToken cancellationToken); Task AddBucketAsync(string bucketName, CancellationToken cancelToken); diff --git a/Duplicati/Library/Backend/S3/S3AwsClient.cs b/Duplicati/Library/Backend/S3/S3AwsClient.cs index e3a480159..94e578182 100644 --- a/Duplicati/Library/Backend/S3/S3AwsClient.cs +++ b/Duplicati/Library/Backend/S3/S3AwsClient.cs @@ -1,33 +1,33 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Amazon.S3; using Amazon.S3.Model; -using Duplicati.Library.Backend.Strings; using Duplicati.Library.Interface; using Duplicati.Library.Utility; using System; using System.Collections.Generic; using System.Linq; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; @@ -192,7 +192,7 @@ namespace Duplicati.Library.Backend return m_client.DeleteObjectAsync(objectDeleteRequest, cancellationToken); } - public virtual IEnumerable ListBucket(string bucketName, string prefix) + public async IAsyncEnumerable ListBucketAsync(string bucketName, string prefix, [EnumeratorCancellation] CancellationToken cancellationToken) { bool isTruncated = true; string filename = null; @@ -217,7 +217,7 @@ namespace Duplicati.Library.Backend ListObjectsResponse listResponse; try { - listResponse = m_client.ListObjectsAsync(listRequest).Await(); + listResponse = await m_client.ListObjectsAsync(listRequest, cancellationToken).ConfigureAwait(false); } catch (AmazonS3Exception e) { diff --git a/Duplicati/Library/Backend/S3/S3Backend.cs b/Duplicati/Library/Backend/S3/S3Backend.cs index 8df4c8355..e6d4d99bb 100644 --- a/Duplicati/Library/Backend/S3/S3Backend.cs +++ b/Duplicati/Library/Backend/S3/S3Backend.cs @@ -1,31 +1,32 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; -using Duplicati.Library.Utility; +using Duplicati.Library.SourceProvider; using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Runtime.CompilerServices; using System.Text; using System.Text.RegularExpressions; using System.Threading; @@ -33,7 +34,7 @@ using System.Threading.Tasks; namespace Duplicati.Library.Backend { - public class S3 : IBackend, IStreamingBackend, IRenameEnabledBackend + public class S3 : IBackend, IStreamingBackend, IRenameEnabledBackend, IFolderEnabledBackend { private static readonly string LOGTAG = Logging.Log.LogTagFromType(); @@ -174,6 +175,15 @@ namespace Duplicati.Library.Backend private readonly string m_bucket; private readonly string m_prefix; + /// + /// Source provider to be used if the backend is a source + /// + private readonly BackendSourceProvider m_sourceProvider; + /// + /// The key path for the backend as a source + /// + private readonly string m_sourcePathKey; + private const string DEFAULT_S3_HOST = "s3.amazonaws.com"; private IS3Client s3Client; @@ -245,6 +255,9 @@ namespace Duplicati.Library.Backend { throw new UserInformationException(Strings.S3Backend.UnknownS3ClientError(s3ClientOptionValue), "UnknownS3Client"); } + + m_sourcePathKey = $"{ProtocolKey}://{hostname}/{m_bucket}/{m_prefix}/"; + m_sourceProvider = new BackendSourceProvider(this); } public static bool IsValidHostname(string bucketname) @@ -269,10 +282,12 @@ namespace Duplicati.Library.Backend get { return true; } } - public IEnumerable List() + => ListAsync(CancellationToken.None).ToBlockingEnumerable(); + + public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { - foreach (IFileEntry file in Connection.ListBucket(m_bucket, m_prefix)) + await foreach (IFileEntry file in Connection.ListBucketAsync(m_bucket, m_prefix, cancelToken)) { ((FileEntry)file).Name = file.Name.Substring(m_prefix.Length); @@ -396,5 +411,38 @@ namespace Duplicati.Library.Backend //AWS SDK encodes the filenames correctly return m_prefix + name; } + + /// + public string PathKey => m_sourcePathKey; + + /// + public async IAsyncEnumerable ListAsync(string path, [EnumeratorCancellation] CancellationToken cancellationToken) + { + var prefixPath = m_prefix; + if (!prefixPath.EndsWith("/", StringComparison.Ordinal)) + prefixPath += "/"; + + var filterPath = prefixPath + path; + if (!filterPath.EndsWith("/", StringComparison.Ordinal)) + filterPath += "/"; + + await foreach (var f in s3Client.ListBucketAsync(m_bucket, filterPath, cancellationToken).ConfigureAwait(false)) + { + if (!f.Name.StartsWith(prefixPath, StringComparison.Ordinal)) + continue; + + ((FileEntry)f).Name = f.Name.Substring(prefixPath.Length); + + // Skip sub-folder items + if (f.Name.Contains("/")) + continue; + + yield return BackendSourceFileEntry.FromFileEntry(m_sourceProvider, path, f); + } + } + + /// + public Task GetEntryAsync(string path, CancellationToken cancellationToken) + => Task.FromResult(null); } } diff --git a/Duplicati/Library/Backend/S3/S3MinioClient.cs b/Duplicati/Library/Backend/S3/S3MinioClient.cs index 09d2e5d52..cee966473 100644 --- a/Duplicati/Library/Backend/S3/S3MinioClient.cs +++ b/Duplicati/Library/Backend/S3/S3MinioClient.cs @@ -22,7 +22,9 @@ using System; using System.Collections.Generic; using System.IO; using System.Reactive.Linq; +using System.Runtime.CompilerServices; using System.Threading; +using System.Threading.Channels; using System.Threading.Tasks; using Duplicati.Library.Interface; using Duplicati.Library.Utility; @@ -58,13 +60,33 @@ namespace Duplicati.Library.Backend m_dnsHost = servername; } - public IEnumerable ListBucket(string bucketName, string prefix) + private static async IAsyncEnumerable ToAsyncEnumerable( + IObservable observable, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var channel = Channel.CreateUnbounded(); // Buffered channel for async iteration + + using var subscription = observable.Subscribe( + item => channel.Writer.TryWrite(item), + ex => channel.Writer.TryComplete(ex), + () => channel.Writer.TryComplete() + ); + + while (await channel.Reader.WaitToReadAsync(cancellationToken)) + { + while (channel.Reader.TryRead(out var item)) + { + yield return item; + } + } + } + + public async IAsyncEnumerable ListBucketAsync(string bucketName, string prefix, [EnumeratorCancellation] CancellationToken cancellationToken) { ThrowExceptionIfBucketDoesNotExist(bucketName); - var observable = m_client.ListObjectsAsync(bucketName, prefix, true); - - foreach (var obj in observable.ToEnumerable()) + var observable = m_client.ListObjectsAsync(bucketName, prefix, true, cancellationToken); + await foreach (var obj in ToAsyncEnumerable(observable, cancellationToken).ConfigureAwait(false)) { yield return new Common.IO.FileEntry( obj.Key, diff --git a/Duplicati/Library/Backend/SSHv2/Duplicati.Library.Backend.SSHv2.csproj b/Duplicati/Library/Backend/SSHv2/Duplicati.Library.Backend.SSHv2.csproj index 9182306df..7c7bbd948 100644 --- a/Duplicati/Library/Backend/SSHv2/Duplicati.Library.Backend.SSHv2.csproj +++ b/Duplicati/Library/Backend/SSHv2/Duplicati.Library.Backend.SSHv2.csproj @@ -12,6 +12,7 @@ + diff --git a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs index 05a1bfb7d..d8057f651 100644 --- a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs +++ b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs @@ -22,6 +22,7 @@ using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; +using Duplicati.Library.SourceProvider; using Renci.SshNet; using Renci.SshNet.Common; using System; @@ -33,7 +34,7 @@ using System.Threading.Tasks; namespace Duplicati.Library.Backend { - public class SSHv2 : IStreamingBackend, IRenameEnabledBackend + public class SSHv2 : IStreamingBackend, IRenameEnabledBackend, IFolderEnabledBackend { private const string SSH_KEYFILE_OPTION = "ssh-keyfile"; private const string SSH_KEYFILE_INLINE = "ssh-key"; @@ -56,6 +57,15 @@ namespace Duplicati.Library.Backend private readonly int m_port = 22; + /// + /// The source provider for this backend + /// + private readonly BackendSourceProvider m_sourceProvider; + /// + /// The path key for this backend if used as a source + /// + private readonly string m_sourcePathKey; + private SftpClient m_con; public SSHv2() @@ -107,6 +117,9 @@ namespace Duplicati.Library.Backend if (!string.IsNullOrWhiteSpace(timeoutstr)) m_keepaliveinterval = Library.Utility.Timeparser.ParseTimeSpan(timeoutstr); + + m_sourcePathKey = $"{ProtocolKey}://{m_server}:{m_port}/~{m_username}/{m_path}/"; + m_sourceProvider = new BackendSourceProvider(this); } #region IBackend Members @@ -429,5 +442,35 @@ namespace Duplicati.Library.Backend } public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new[] { m_server }); + + /// + public string PathKey => m_sourcePathKey; + + /// + public IAsyncEnumerable ListAsync(string path, CancellationToken cancellationToken) + { + var prefixPath = m_path; + if (!prefixPath.EndsWith("/")) + prefixPath += "/"; + + var filterPath = prefixPath + path; + if (!filterPath.EndsWith("/")) + filterPath += "/"; + + return Client.ListDirectoryAsync(filterPath, cancellationToken) + .Select(x => new BackendSourceFileEntry( + m_sourceProvider, + BackendSourceFileEntry.ConcatPaths(path, x.Name), + x.Attributes.IsDirectory, + false, + new DateTime(0), + x.Attributes.LastWriteTimeUtc, + x.Attributes.Size + )); + } + + /// + public Task GetEntryAsync(string path, CancellationToken cancellationToken) + => Task.FromResult(null); } } \ No newline at end of file diff --git a/Duplicati/Library/Backend/Storj/StorjFile.cs b/Duplicati/Library/Backend/Storj/StorjFile.cs index f7a0d6e51..6a4b18076 100644 --- a/Duplicati/Library/Backend/Storj/StorjFile.cs +++ b/Duplicati/Library/Backend/Storj/StorjFile.cs @@ -33,6 +33,7 @@ namespace Duplicati.Library.Backend.Storj public DateTime LastAccess { get; set; } public DateTime LastModification { get; set; } + public DateTime Created { get; set; } public string Name { get; set; } diff --git a/Duplicati/Library/Backend/pCloud/Model/pCloudFileEntry.cs b/Duplicati/Library/Backend/pCloud/Model/pCloudFileEntry.cs index bbe68ef6d..179c277c6 100644 --- a/Duplicati/Library/Backend/pCloud/Model/pCloudFileEntry.cs +++ b/Duplicati/Library/Backend/pCloud/Model/pCloudFileEntry.cs @@ -33,22 +33,27 @@ internal class pCloudFileEntry : IFileEntry /// Indicates it is a folder on the remote filesystem /// public bool IsFolder { get; set; } - + /// /// Last access date of the file /// public DateTime LastAccess { get; set; } - + /// /// Last modification of the file /// public DateTime LastModification { get; set; } - + + /// + /// Last modification of the file + /// + public DateTime Created { get; set; } + /// /// Filename or folder name /// public string Name { get; set; } - + /// /// Size of the object in bytes /// diff --git a/Duplicati/Library/Common/IO/FileEntry.cs b/Duplicati/Library/Common/IO/FileEntry.cs index fc1b18b49..b603edd0d 100644 --- a/Duplicati/Library/Common/IO/FileEntry.cs +++ b/Duplicati/Library/Common/IO/FileEntry.cs @@ -1,23 +1,23 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. using System; using Duplicati.Library.Interface; @@ -32,6 +32,7 @@ namespace Duplicati.Library.Common.IO private string m_name; private DateTime m_lastAccess; private DateTime m_lastModification; + private DateTime m_created; private long m_size; private bool m_isFolder; @@ -62,6 +63,15 @@ namespace Duplicati.Library.Common.IO set { m_lastModification = value; } } + /// + /// Gets or sets the time the file or folder was created + /// + public DateTime Created + { + get { return m_created; } + set { m_created = value; } + } + /// /// Gets or sets the size of the file or folder /// diff --git a/Duplicati/Library/Interface/IFileEntry.cs b/Duplicati/Library/Interface/IFileEntry.cs index 3a7f9e104..2e5fb1f01 100644 --- a/Duplicati/Library/Interface/IFileEntry.cs +++ b/Duplicati/Library/Interface/IFileEntry.cs @@ -1,23 +1,23 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. using System; @@ -41,6 +41,10 @@ namespace Duplicati.Library.Interface /// DateTime LastModification { get; } /// + /// The time the file or folder was created + /// + DateTime Created { get; } + /// /// The name of the file or folder /// string Name { get; } diff --git a/Duplicati/Library/Interface/IFolderEnabledBackend.cs b/Duplicati/Library/Interface/IFolderEnabledBackend.cs index 2ccd3f44c..4f83178ad 100644 --- a/Duplicati/Library/Interface/IFolderEnabledBackend.cs +++ b/Duplicati/Library/Interface/IFolderEnabledBackend.cs @@ -34,12 +34,22 @@ namespace Duplicati.Library.Interface; /// public interface IFolderEnabledBackend : IBackend { + /// + /// Returns the key for this backend instance's path + /// + string PathKey { get; } + /// /// Enumerates a list of files and folders found on the remote location /// /// The path to list, empty or null for the root folder /// The cancellation token - /// The list of files + /// The list of files, where paths are scoped to the backend. + /// + /// The paths returned should be relative to the backend root, and should not include the root path. + /// For example, if the backend is defined as proto://host/rootpath, then listing files should + /// return paths like "/subfolder/file.txt" and not "/rootpath/subfolder/file.txt". + /// IAsyncEnumerable ListAsync(string? path, CancellationToken cancellationToken); /// diff --git a/Duplicati/Library/Interface/ISourceProvider.cs b/Duplicati/Library/Interface/ISourceProvider.cs index 4b5725739..11c61d4a4 100644 --- a/Duplicati/Library/Interface/ISourceProvider.cs +++ b/Duplicati/Library/Interface/ISourceProvider.cs @@ -33,6 +33,11 @@ namespace Duplicati.Library.Interface; /// public interface ISourceProvider : IDisposable { + /// + /// Returns the key for this provider instance's path + /// + string PathKey { get; } + /// /// Gets the root entries /// @@ -47,5 +52,5 @@ public interface ISourceProvider : IDisposable /// True if the path is a folder /// The cancellation token /// The file entry - Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken); + Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken); } diff --git a/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs b/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs index 7283da48e..a1ef2ec65 100644 --- a/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs @@ -19,6 +19,8 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +#nullable enable + using System; using CoCoL; using System.Threading.Tasks; @@ -49,17 +51,17 @@ namespace Duplicati.Library.Main.Operation.Backup public static Task Run( Channels channels, ISourceProvider sourceProvider, - UsnJournalService journalService, + UsnJournalService? journalService, FileAttributes fileAttributeFilter, Library.Utility.IFilter emitfilter, Options.SymlinkStrategy symlinkPolicy, Options.HardlinkStrategy hardlinkPolicy, bool excludeemptyfolders, - string[] ignorenames, + string[]? ignorenames, HashSet blacklistPaths, - IEnumerable changedfilelist, + IEnumerable? changedfilelist, ITaskReader taskreader, - Action onStopRequested, + Action? onStopRequested, CancellationToken token) { return AutomationExtensions.RunTask( @@ -98,7 +100,11 @@ namespace Duplicati.Library.Main.Operation.Backup async IAsyncEnumerable ExpandSources(IEnumerable list) { foreach (var s in list) - yield return await sourceProvider.GetEntry(s, false, token).ConfigureAwait(false); + { + var r = await sourceProvider.GetEntry(s, false, token).ConfigureAwait(false); + if (r != null) + yield return r; + } } IAsyncEnumerable worklist; @@ -113,7 +119,7 @@ namespace Duplicati.Library.Main.Operation.Backup throw new NotSupportedException("USN is only supported on Windows"); var fileProviders = (sourceProvider is Combiner c ? c.Providers.AsEnumerable() : [sourceProvider]) - .OfType() + .OfType() .ToList(); if (fileProviders.Count <= 0) @@ -179,11 +185,11 @@ namespace Duplicati.Library.Main.Operation.Backup /// /// The item being tracked /// - public ISourceFileEntry Item; + public required ISourceFileEntry Item; /// /// A flag indicating if any items are found in this folder /// - public bool AnyEntries; + public required bool AnyEntries; } @@ -219,7 +225,7 @@ namespace Duplicati.Library.Main.Operation.Backup if (pathstack.Count == 0 || s.Path.StartsWith(pathstack.Peek().Item.Path, Library.Utility.Utility.ClientFilenameStringComparison)) { - pathstack.Push(new DirectoryStackEntry() { Item = s }); + pathstack.Push(new DirectoryStackEntry() { Item = s, AnyEntries = false }); continue; } } @@ -381,7 +387,7 @@ namespace Duplicati.Library.Main.Operation.Backup /// The ignore names. /// The mixin queue. /// True if the path should be returned, false otherwise. - private static async ValueTask SourceFileEntryFilter(ISourceFileEntry entry, HashSet blacklistPaths, Options.HardlinkStrategy hardlinkPolicy, Options.SymlinkStrategy symlinkPolicy, Dictionary hardlinkmap, FileAttributes fileAttributeFilter, Duplicati.Library.Utility.IFilter enumeratefilter, string[] ignorenames, Queue mixinqueue, CancellationToken cancellationToken) + private static async ValueTask SourceFileEntryFilter(ISourceFileEntry entry, HashSet blacklistPaths, Options.HardlinkStrategy hardlinkPolicy, Options.SymlinkStrategy symlinkPolicy, Dictionary hardlinkmap, FileAttributes fileAttributeFilter, Duplicati.Library.Utility.IFilter enumeratefilter, string[]? ignorenames, Queue mixinqueue, CancellationToken cancellationToken) { // Do the course pre-filtering first if (!PreFilterSourceEntry(entry, blacklistPaths)) @@ -406,8 +412,7 @@ namespace Duplicati.Library.Main.Operation.Backup } else if (hardlinkPolicy == Options.HardlinkStrategy.First) { - string prevPath; - if (hardlinkmap.TryGetValue(id, out prevPath)) + if (hardlinkmap.TryGetValue(id, out var prevPath)) { Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingDuplicateHardlink", "Excluding hardlink ({1}) for: {0}, previous hardlink: {2}", entry.Path, id, prevPath); return false; @@ -481,7 +486,7 @@ namespace Duplicati.Library.Main.Operation.Backup } // If the file is a symlink, apply special handling - string symlinkTarget = null; + string? symlinkTarget = null; try { symlinkTarget = entry.SymlinkTarget; diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 4c3171f6c..84fc89b69 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -132,9 +132,9 @@ namespace Duplicati.Library.Main.Operation foreach (var entry in sourceTypes) { if ("file".Equals(entry.Key, StringComparison.OrdinalIgnoreCase)) - results.Add(new SourceProvider.File(GetFileSnapshotService(entry, options))); + results.Add(new LocalFileSource(GetFileSnapshotService(entry, options))); else if ("vss".Equals(entry.Key, StringComparison.OrdinalIgnoreCase) || "lvm".Equals(entry.Key, StringComparison.OrdinalIgnoreCase)) - results.Add(new SourceProvider.File(SnapshotUtility.CreateSnapshot(entry, options.RawOptions, options.SymlinkPolicy == Options.SymlinkStrategy.Follow))); + results.Add(new LocalFileSource(SnapshotUtility.CreateSnapshot(entry, options.RawOptions, options.SymlinkPolicy == Options.SymlinkStrategy.Follow))); else { foreach (var url in entry) @@ -147,14 +147,15 @@ namespace Duplicati.Library.Main.Operation continue; } - // See if we can implement the same with a backend + // See if there is a backend that can also be a source var backend = BackendLoader.GetBackend(url, options.RawOptions); - if (backend != null) + if (backend is IFolderEnabledBackend folderBackend) { - results.Add(new SourceProvider.BackendSourceProvider(backend, url)); + results.Add(new BackendSourceProvider(folderBackend)); continue; } + backend?.Dispose(); throw new UserInformationException(string.Format("The source \"{0}\" is not supported", url), "SourceNotSupported"); } } @@ -179,14 +180,14 @@ namespace Duplicati.Library.Main.Operation var providers = (provider is Combiner c ? c.Providers : [provider]) .Select((x, i) => new { Provider = x, Index = i }) - .Where(x => x.Provider is SourceProvider.File) + .Where(x => x.Provider is LocalFileSource) .ToList(); if (providers.Count == 0) return null; if (providers.Count > 1) throw new UserInformationException("Multiple USN journal services are not supported", "MultipleUSNJournalServices"); - var fileProvider = providers.First().Provider as SourceProvider.File; + var fileProvider = providers.First().Provider as LocalFileSource; var journalData = m_database.GetChangeJournalData(lastfilesetid); var service = new UsnJournalService(fileProvider.SnapshotService, filter, m_options.FileAttributeFilter, m_options.SkipFilesLargerThan, diff --git a/Duplicati/Library/Main/Operation/FilelistProcessor.cs b/Duplicati/Library/Main/Operation/FilelistProcessor.cs index b3c6405c4..f1d10fdfb 100644 --- a/Duplicati/Library/Main/Operation/FilelistProcessor.cs +++ b/Duplicati/Library/Main/Operation/FilelistProcessor.cs @@ -26,7 +26,6 @@ using System.Data; using System.Linq; using Duplicati.Library.Interface; using System.Threading; -using Duplicati.Library.Utility; using System.Threading.Tasks; namespace Duplicati.Library.Main.Operation diff --git a/Duplicati/Library/Main/Operation/TestFilterHandler.cs b/Duplicati/Library/Main/Operation/TestFilterHandler.cs index 4a06bbc89..b2468738f 100644 --- a/Duplicati/Library/Main/Operation/TestFilterHandler.cs +++ b/Duplicati/Library/Main/Operation/TestFilterHandler.cs @@ -19,6 +19,8 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +#nullable enable + using System; using System.IO; using CoCoL; @@ -71,7 +73,7 @@ namespace Duplicati.Library.Main.Operation catch (Exception ex) { Logging.Log.WriteVerboseMessage(LOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", entry.Path, ex.Message); } // Analyze symlinks - string symlinkTarget = null; + string? symlinkTarget = null; try { symlinkTarget = entry.SymlinkTarget; } catch (Exception ex) { Logging.Log.WriteExplicitMessage(LOGTAG, "SymlinkTargetReadFailure", ex, "Failed to read symlink target for path: {0}", entry.Path); } @@ -84,14 +86,22 @@ namespace Duplicati.Library.Main.Operation // Go for the symlink target, as we know we follow symlinks if (!string.IsNullOrWhiteSpace(symlinkTarget)) { - entry = await provider.GetEntry(symlinkTarget, false, stopToken).ConfigureAwait(false); + var targetEntry = await provider.GetEntry(symlinkTarget, false, stopToken).ConfigureAwait(false); fa = FileAttributes.Normal; - try { fa = entry.Attributes; } + try { fa = targetEntry!.Attributes; } catch (Exception ex) { Logging.Log.WriteVerboseMessage(LOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", entry.Path, ex.Message); } + // If we guessed wrong and the symlink target is a folder, we need to fetch it with the correct flag if (fa.HasFlag(FileAttributes.Directory)) - entry = await provider.GetEntry(symlinkTarget, true, stopToken).ConfigureAwait(false); + targetEntry = await provider.GetEntry(symlinkTarget, true, stopToken).ConfigureAwait(false); + + // No such target + if (targetEntry == null) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "SymlinkTargetNotFound", "Symlink target not found: {0}", symlinkTarget); + continue; + } } // Proceed with non-folders @@ -110,7 +120,6 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteVerboseMessage(LOGTAG, "SizeReadFailed", "Failed to read length of file {0}: {1}", entry.Path, ex.Message); } - if (m_options.SkipFilesLargerThan == long.MaxValue || m_options.SkipFilesLargerThan == 0 || size < m_options.SkipFilesLargerThan) Logging.Log.WriteVerboseMessage(LOGTAG, "IncludeFile", "Including file: {0} ({1})", entry.Path, size < 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(size)); else diff --git a/Duplicati/Library/Snapshots/ISnapshotService.cs b/Duplicati/Library/Snapshots/ISnapshotService.cs index 4deea290f..fa95438b8 100644 --- a/Duplicati/Library/Snapshots/ISnapshotService.cs +++ b/Duplicati/Library/Snapshots/ISnapshotService.cs @@ -57,7 +57,7 @@ namespace Duplicati.Library.Snapshots /// The path to get /// True if the path is a folder /// The filesystem entry - ISourceFileEntry GetFilesystemEntry(string path, bool isFolder); + ISourceFileEntry? GetFilesystemEntry(string path, bool isFolder); /// /// Gets the last write time of a given file in UTC diff --git a/Duplicati/Library/Snapshots/LinuxSnapshot.cs b/Duplicati/Library/Snapshots/LinuxSnapshot.cs index 472324299..5df6084d6 100644 --- a/Duplicati/Library/Snapshots/LinuxSnapshot.cs +++ b/Duplicati/Library/Snapshots/LinuxSnapshot.cs @@ -112,15 +112,6 @@ namespace Duplicati.Library.Snapshots /// public override IEnumerable SourceFolders => m_snapShots.Select(e => e.LocalPath); - /// - /// Gets a filesystem entry for a given path - /// - /// The path to get the entry for - /// A flag indicating if the path is a folder - /// The filesystem entry - public override ISourceFileEntry GetFilesystemEntry(string path, bool isFolder) - => new SnapshotSourceFileEntry(this, isFolder ? Util.AppendDirSeparator(path) : path, isFolder, false); - /// /// Enumerates the root source files and folders /// diff --git a/Duplicati/Library/Snapshots/SnapshotBase.cs b/Duplicati/Library/Snapshots/SnapshotBase.cs index 0c1516790..7fcab2761 100644 --- a/Duplicati/Library/Snapshots/SnapshotBase.cs +++ b/Duplicati/Library/Snapshots/SnapshotBase.cs @@ -73,7 +73,14 @@ namespace Duplicati.Library.Snapshots /// A flag indicating if the path is a folder /// The filesystem entry public virtual ISourceFileEntry GetFilesystemEntry(string path, bool isFolder) - => new SnapshotSourceFileEntry(this, isFolder ? Util.AppendDirSeparator(path) : path, isFolder, false); + { + if (isFolder && !DirectoryExists(path)) + return null; + else if (!isFolder && !FileExists(path)) + return null; + + return new SnapshotSourceFileEntry(this, isFolder ? Util.AppendDirSeparator(path) : path, isFolder, false); + } /// /// Enumerates files and folders in the given folder diff --git a/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs b/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs new file mode 100644 index 000000000..740c51e00 --- /dev/null +++ b/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs @@ -0,0 +1,216 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using Duplicati.Library.Interface; +using Duplicati.Library.Utility; + +namespace Duplicati.Library.SourceProvider; + +/// +/// Creates a new backend source entry +/// +/// The parent backend +/// The path of the entry +/// True if the entry is a folder +/// True if the entry is a meta entry +/// The creation time of the entry +/// The last modification time of the entry +/// The size of the entry +public class BackendSourceFileEntry(BackendSourceProvider parent, string path, bool isFolder, bool isMetaEntry, DateTime createdUtc, DateTime lastModificationUtc, long size) + : ISourceFileEntry +{ + /// + public bool IsFolder => isFolder; + + /// + public bool IsMetaEntry => isMetaEntry; + + /// + public bool IsRootEntry => isMetaEntry; + + /// + public DateTime CreatedUtc => createdUtc; + + /// + public DateTime LastModificationUtc => lastModificationUtc; + + /// + public string Path => ConcatPaths(parent.PathKey, NormalizePath(path)); + + /// + /// The local path of the entry + /// + public string LocalPath => path; + + /// + public long Size => size; + /// + public bool IsSymlink => false; + + /// + public string? SymlinkTarget => null; + + /// + public FileAttributes Attributes => IsFolder + ? FileAttributes.Directory + : FileAttributes.Normal; + + /// + public Dictionary MinorMetadata => new Dictionary(); + + /// + public bool IsBlockDevice => false; + + /// + public bool IsCharacterDevice => false; + + /// + public bool IsAlternateStream => false; + + /// + public string? HardlinkTargetId => null; + + /// + public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) + { + if (!isFolder) + throw new InvalidOperationException("Enumerate can only be called on folders"); + + if (parent.WrappedBackend is IFolderEnabledBackend folderEnabledBackend) + { + return folderEnabledBackend.ListAsync(path, cancellationToken) + .Select(x => new BackendSourceFileEntry(parent, x.Path, x.IsFolder, false, x.CreatedUtc, x.LastModificationUtc, x.Size)); + } + else + { + // If we do not support folders, we can only list the root + if (this.IsMetaEntry) + return parent.WrappedBackend.List() + .Select(x => FromFileEntry(parent, "", x)) + .ToAsyncEnumerable(); + + return Array.Empty().ToAsyncEnumerable(); + } + } + + /// + public async Task FileExists(string path, CancellationToken cancellationToken) + { + if (!isFolder) + throw new InvalidOperationException("FileExists can only be called on folders"); + + if (parent.WrappedBackend is IFolderEnabledBackend folderEnabledBackend) + { + try + { + var entry = await folderEnabledBackend.GetEntryAsync(path, cancellationToken); + return entry != null && !entry.IsFolder; + } + catch (FileNotFoundException) + { + return false; + } + } + + return false; + } + + /// + public Task OpenMetadataRead(CancellationToken cancellationToken) + => Task.FromResult(null); + + /// + public async Task OpenRead(CancellationToken cancellationToken) + { + if (isFolder) + throw new InvalidOperationException("OpenRead can only be called on files"); + + if (parent.WrappedBackend is IStreamingBackend streamingBackend) + { + var mapStream = new MapStream(); + mapStream.CopyTask = streamingBackend.GetAsync(this.Path, mapStream, cancellationToken); + return mapStream; + } + else + { + TempFileStream? file = null; + try + { + file = TempFileStream.Create(); + await parent.WrappedBackend.GetAsync(this.Path, file.Path, cancellationToken); + file.Position = 0; + return file; + } + catch + { + file?.Dispose(); + throw; + } + } + } + + /// + /// Normalizes the path, turning backslashes into forward slashes + /// + /// The path to normalize + /// The normalized path + public static string NormalizePath(string path) + { + if (System.IO.Path.DirectorySeparatorChar == '/') + return path; + + if (string.IsNullOrEmpty(path)) + return path; + + return path.Replace('\\', '/'); + } + + /// + /// Concatenates two paths + /// + /// The first path + /// The second path + /// The concatenated path + public static string ConcatPaths(string path1, string path2) + { + if (string.IsNullOrEmpty(path1)) + return path2; + + if (string.IsNullOrEmpty(path2)) + return path1; + + if (path1.EndsWith('/') || path1.EndsWith('\\')) + return path1 + path2; + + return path1 + "/" + path2; + } + + /// + /// Creates a new backend source entry from a file entry + /// + /// The parent backend + /// The file entry + /// The prefix to add to the path + /// The new backend source entry + public static BackendSourceFileEntry FromFileEntry(BackendSourceProvider parent, string prefix, IFileEntry entry) + => new BackendSourceFileEntry(parent, ConcatPaths(prefix, entry.Name), entry.IsFolder, false, entry.Created, entry.LastModification, entry.Size); +} + diff --git a/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs index 9b144a7ab..815c7e48a 100644 --- a/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs +++ b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs @@ -20,7 +20,6 @@ // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Interface; -using Duplicati.Library.Utility; namespace Duplicati.Library.SourceProvider; @@ -28,179 +27,30 @@ namespace Duplicati.Library.SourceProvider; /// A source provider that wraps a backend /// /// The backend to wrap -public class BackendSourceProvider : ISourceProvider +public class BackendSourceProvider(IFolderEnabledBackend backend) : ISourceProvider { /// /// The wrapped backend /// - public IBackend WrappedBackend { get; init; } + public IFolderEnabledBackend WrappedBackend => backend; /// /// The path key unique for the destination /// - public readonly string PathKey; - - /// - /// Creates a new backend source provider - /// - /// The backend to wrap - /// The URL of the backend - public BackendSourceProvider(IBackend backend, string url) - { - WrappedBackend = backend; - var uri = new Utility.Uri(url); - // TODO: The username may be provided in other ways to the backend... - PathKey = $"{uri.Scheme}://{uri.Host}:{uri.Port}/~{uri.Username}/{uri.Path}/"; - } - - /// - /// Creates a new backend source entry - /// - /// The parent backend - /// The path of the entry - /// True if the entry is a folder - /// True if the entry is a meta entry - /// The creation time of the entry - /// The last modification time of the entry - /// The size of the entry - private class BackendSourceFileEntry(BackendSourceProvider parent, string path, bool isFolder, bool isMetaEntry, DateTime createdUtc, DateTime lastModificationUtc, long size) - : ISourceFileEntry - { - /// - public bool IsFolder => isFolder; - - /// - public bool IsMetaEntry => isMetaEntry; - - /// - public bool IsRootEntry => isMetaEntry; - - /// - public DateTime CreatedUtc => createdUtc; - - /// - public DateTime LastModificationUtc => lastModificationUtc; - - /// - public string Path => $"{parent.PathKey}{path}"; - - /// - public long Size => size; - /// - public bool IsSymlink => false; - - /// - public string? SymlinkTarget => null; - - /// - public FileAttributes Attributes => IsFolder - ? FileAttributes.Directory - : FileAttributes.Normal; - - /// - public Dictionary MinorMetadata => new Dictionary(); - - /// - public bool IsBlockDevice => false; - - /// - public bool IsCharacterDevice => false; - - /// - public bool IsAlternateStream => false; - - /// - public string? HardlinkTargetId => null; - - /// - public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) - { - if (!isFolder) - throw new InvalidOperationException("Enumerate can only be called on folders"); - - if (parent.WrappedBackend is IFolderEnabledBackend folderEnabledBackend) - { - return folderEnabledBackend.ListAsync(path, cancellationToken) - .Select(x => new BackendSourceFileEntry(parent, x.Path, x.IsFolder, false, new DateTime(0), x.LastModificationUtc, x.Size)); - } - else - { - - // If we do not support folders, we can only list the root - if (this.IsMetaEntry) - return parent.WrappedBackend.List() - .Select(x => new BackendSourceFileEntry(parent, x.Name, x.IsFolder, false, new DateTime(0), x.LastModification, x.Size)) - .ToAsyncEnumerable(); - - return Array.Empty().ToAsyncEnumerable(); - } - } - - /// - public async Task FileExists(string path, CancellationToken cancellationToken) - { - if (!isFolder) - throw new InvalidOperationException("FileExists can only be called on folders"); - - if (parent.WrappedBackend is IFolderEnabledBackend folderEnabledBackend) - { - try - { - var entry = await folderEnabledBackend.GetEntryAsync(path, cancellationToken); - return entry != null && !entry.IsFolder; - } - catch (FileNotFoundException) - { - return false; - } - } - - return false; - } - - /// - public Task OpenMetadataRead(CancellationToken cancellationToken) - => Task.FromResult(null); - - /// - public async Task OpenRead(CancellationToken cancellationToken) - { - if (isFolder) - throw new InvalidOperationException("OpenRead can only be called on files"); - - if (parent.WrappedBackend is IStreamingBackend streamingBackend) - { - var mapStream = new MapStream(); - mapStream.CopyTask = streamingBackend.GetAsync(this.Path, mapStream, cancellationToken); - return mapStream; - } - else - { - TempFileStream? file = null; - try - { - file = TempFileStream.Create(); - await parent.WrappedBackend.GetAsync(this.Path, file.Path, cancellationToken); - file.Position = 0; - return file; - } - catch - { - file?.Dispose(); - throw; - } - } - } - } + public string PathKey => backend.PathKey; /// public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) - => new[] { new BackendSourceFileEntry(this, "", true, true, new DateTime(0), new DateTime(0), 0) }.ToAsyncEnumerable(); + => new[] { new BackendSourceFileEntry(this, PathKey, true, true, new DateTime(0), new DateTime(0), 0) }.ToAsyncEnumerable(); /// - public Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) - // TODO: If we cache the enumerate result, we could check if the path exists in the cache and return it - => Task.FromResult(new BackendSourceFileEntry(this, path, isFolder, false, new DateTime(0), new DateTime(0), -1)); + public async Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) + { + if (this.WrappedBackend is IFolderEnabledBackend folderEnabledBackend) + return await folderEnabledBackend.GetEntryAsync(path, cancellationToken); + + return null; + } /// public void Dispose() diff --git a/Duplicati/Library/SourceProvider/Builtin/Combiner.cs b/Duplicati/Library/SourceProvider/Builtin/Combiner.cs index 1dcb01fa8..7f027847f 100644 --- a/Duplicati/Library/SourceProvider/Builtin/Combiner.cs +++ b/Duplicati/Library/SourceProvider/Builtin/Combiner.cs @@ -29,6 +29,9 @@ namespace Duplicati.Library.SourceProvider; /// public class Combiner(IEnumerable providers) : ISourceProvider { + /// + public string PathKey => string.Empty; + /// /// The providers to combine /// @@ -56,10 +59,23 @@ public class Combiner(IEnumerable providers) : ISourceProvider /// True if the path is a folder /// The cancellation token /// The filesystem entry - public Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) + public async Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) { - // TODO: Map roots to the paths so we get the right provider - throw new NotImplementedException(); + foreach (var provider in providers) + if (!string.IsNullOrWhiteSpace(PathKey) && provider.PathKey.StartsWith(path)) + { + var subpath = provider.PathKey.Substring(path.Length); + return await provider.GetEntry(subpath, isFolder, cancellationToken).ConfigureAwait(false); + } + + foreach (var provider in providers.Where(x => string.IsNullOrWhiteSpace(x.PathKey))) + { + var res = await provider.GetEntry(path, isFolder, cancellationToken).ConfigureAwait(false); + if (res != null) + return res; + } + + return null; } /// diff --git a/Duplicati/Library/SourceProvider/Builtin/File.cs b/Duplicati/Library/SourceProvider/Builtin/LocalFileSource.cs similarity index 86% rename from Duplicati/Library/SourceProvider/Builtin/File.cs rename to Duplicati/Library/SourceProvider/Builtin/LocalFileSource.cs index 985fc072c..9c75ca3bb 100644 --- a/Duplicati/Library/SourceProvider/Builtin/File.cs +++ b/Duplicati/Library/SourceProvider/Builtin/LocalFileSource.cs @@ -28,8 +28,11 @@ namespace Duplicati.Library.SourceProvider; /// A source provider that wraps a file system through a snapshot service /// /// The snapshot service to use -public class File(ISnapshotService snapshotService) : ISourceProvider +public class LocalFileSource(ISnapshotService snapshotService) : ISourceProvider { + /// + public string PathKey => string.Empty; + /// public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) => snapshotService.EnumerateFilesystemEntries().ToAsyncEnumerable(); @@ -40,7 +43,7 @@ public class File(ISnapshotService snapshotService) : ISourceProvider public ISnapshotService SnapshotService => snapshotService; /// - public Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) + public Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) => Task.FromResult(snapshotService.GetFilesystemEntry(path, isFolder)); /// From 0e8df7ccd88ee6ad405afa5635f968a730fcf55b Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 17 Feb 2025 14:05:21 +0100 Subject: [PATCH 175/362] Removed unsupported code --- Duplicati/Library/Backend/File/FileBackend.cs | 35 +----------------- .../Builtin/BackendSourceFileEntry.cs | 36 +++++-------------- .../Builtin/BackendSourceProvider.cs | 9 ++--- 3 files changed, 12 insertions(+), 68 deletions(-) diff --git a/Duplicati/Library/Backend/File/FileBackend.cs b/Duplicati/Library/Backend/File/FileBackend.cs index 3a8b76ca8..f13f6889a 100644 --- a/Duplicati/Library/Backend/File/FileBackend.cs +++ b/Duplicati/Library/Backend/File/FileBackend.cs @@ -21,12 +21,9 @@ using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; -using Duplicati.Library.Snapshots; using System; using System.Collections.Generic; using System.IO; -using System.Linq; -using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Versioning; using System.Threading; @@ -36,7 +33,7 @@ namespace Duplicati.Library.Backend { // ReSharper disable once UnusedMember.Global // This class is instantiated dynamically in the BackendLoader. - public class File : IBackend, IStreamingBackend, IQuotaEnabledBackend, IRenameEnabledBackend, IFolderEnabledBackend + public class File : IBackend, IStreamingBackend, IQuotaEnabledBackend, IRenameEnabledBackend { private const string OPTION_DESTINATION_MARKER = "alternate-destination-marker"; private const string OPTION_ALTERNATE_PATHS = "alternate-target-paths"; @@ -467,35 +464,5 @@ namespace Duplicati.Library.Backend throw; } } - - /// - public string PathKey => m_path; - - /// - public async IAsyncEnumerable ListAsync(string path, [EnumeratorCancellation] CancellationToken cancellationToken) - { - PreAuthenticate(); - - var remotePath = string.IsNullOrWhiteSpace(path) ? m_path : GetRemoteName(path); - - if (!systemIO.DirectoryExists(remotePath)) - throw new FolderMissingException(Strings.FileBackend.FolderMissingError(m_path)); - - var service = SnapshotUtility.CreateNoSnapshot([remotePath], false, false); - - await foreach (var folderEntry in systemIO.EnumerateDirectories(remotePath).ToAsyncEnumerable().WithCancellation(cancellationToken).ConfigureAwait(false)) - yield return new SnapshotSourceFileEntry(service, folderEntry, true, false); - - await foreach (var fileEntry in systemIO.EnumerateFiles(remotePath).ToAsyncEnumerable().WithCancellation(cancellationToken).ConfigureAwait(false)) - yield return new SnapshotSourceFileEntry(service, fileEntry, false, false); - } - - /// - public Task GetEntryAsync(string path, CancellationToken cancellationToken) - { - var service = SnapshotUtility.CreateNoSnapshot([path], false, false); - var isFolder = service.DirectoryExists(path); - return Task.FromResult(new SnapshotSourceFileEntry(service, path, isFolder, false)); - } } } diff --git a/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs b/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs index 740c51e00..7b6fa496c 100644 --- a/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs +++ b/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs @@ -94,21 +94,8 @@ public class BackendSourceFileEntry(BackendSourceProvider parent, string path, b if (!isFolder) throw new InvalidOperationException("Enumerate can only be called on folders"); - if (parent.WrappedBackend is IFolderEnabledBackend folderEnabledBackend) - { - return folderEnabledBackend.ListAsync(path, cancellationToken) - .Select(x => new BackendSourceFileEntry(parent, x.Path, x.IsFolder, false, x.CreatedUtc, x.LastModificationUtc, x.Size)); - } - else - { - // If we do not support folders, we can only list the root - if (this.IsMetaEntry) - return parent.WrappedBackend.List() - .Select(x => FromFileEntry(parent, "", x)) - .ToAsyncEnumerable(); - - return Array.Empty().ToAsyncEnumerable(); - } + return parent.WrappedBackend.ListAsync(path, cancellationToken) + .Select(x => new BackendSourceFileEntry(parent, x.Path, x.IsFolder, false, x.CreatedUtc, x.LastModificationUtc, x.Size)); } /// @@ -117,20 +104,15 @@ public class BackendSourceFileEntry(BackendSourceProvider parent, string path, b if (!isFolder) throw new InvalidOperationException("FileExists can only be called on folders"); - if (parent.WrappedBackend is IFolderEnabledBackend folderEnabledBackend) + try { - try - { - var entry = await folderEnabledBackend.GetEntryAsync(path, cancellationToken); - return entry != null && !entry.IsFolder; - } - catch (FileNotFoundException) - { - return false; - } + var entry = await parent.WrappedBackend.GetEntryAsync(path, cancellationToken); + return entry != null && !entry.IsFolder; + } + catch (FileNotFoundException) + { + return false; } - - return false; } /// diff --git a/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs index 815c7e48a..6411cc0a7 100644 --- a/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs +++ b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs @@ -44,13 +44,8 @@ public class BackendSourceProvider(IFolderEnabledBackend backend) : ISourceProvi => new[] { new BackendSourceFileEntry(this, PathKey, true, true, new DateTime(0), new DateTime(0), 0) }.ToAsyncEnumerable(); /// - public async Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) - { - if (this.WrappedBackend is IFolderEnabledBackend folderEnabledBackend) - return await folderEnabledBackend.GetEntryAsync(path, cancellationToken); - - return null; - } + public Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) + => backend.GetEntryAsync(path, cancellationToken); /// public void Dispose() From 0f9669b27378452c9ad0872d6f1e736a76f1c77c Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 17 Feb 2025 16:45:51 +0100 Subject: [PATCH 176/362] Final conversion to async backends This PR is the final change, where all backends are now purely implemented with async interfaces. The code changes here replace the LIST call with an async version. This affect all backends (and then some) but the changes are mostly mechanical rewrites of the code. In many places, the backeds were already prepared for async output, in other some glue was needed. To reduce the scope of this change, some backends simply report a synchronous result as an async list. --- .../CommandLine/BackendTester/Program.cs | 23 ++-- Duplicati/CommandLine/BackendTool/Program.cs | 3 +- .../CommandLine/RecoveryTool/Download.cs | 41 +++---- .../CommandLine/RecoveryTool/Recompress.cs | 4 +- .../Backend/AliyunOSS/AliyunOSSBackend.cs | 74 ++++++------ .../Backend/AzureBlob/AzureBlobBackend.cs | 10 +- .../Backend/AzureBlob/AzureBlobWrapper.cs | 88 +++++++------- Duplicati/Library/Backend/Backblaze/B2.cs | 50 ++++---- ...Duplicati.Library.Backend.Backblaze.csproj | 1 + Duplicati/Library/Backend/Box/BoxBackend.cs | 64 +++++------ .../Box/Duplicati.Library.Backend.Box.csproj | 1 + Duplicati/Library/Backend/CIFS/CIFS.cs | 41 +++---- .../Library/Backend/CloudFiles/CloudFiles.cs | 59 +++++----- Duplicati/Library/Backend/Dropbox/Dropbox.cs | 27 +++-- .../Library/Backend/Dropbox/DropboxHelper.cs | 8 +- .../FTP/Duplicati.Library.Backend.FTP.csproj | 1 + Duplicati/Library/Backend/FTP/FTPBackend.cs | 107 +++++++++--------- .../Duplicati.Library.Backend.File.csproj | 1 + Duplicati/Library/Backend/File/FileBackend.cs | 11 +- ...cati.Library.Backend.GoogleServices.csproj | 1 + .../GoogleServices/GoogleCloudStorage.cs | 53 +++++---- .../Backend/GoogleServices/GoogleDrive.cs | 27 +++-- .../Backend/Idrivee2/Idrivee2Backend.cs | 12 +- .../Library/Backend/Jottacloud/Jottacloud.cs | 12 +- Duplicati/Library/Backend/Mega/MegaBackend.cs | 59 +++++----- .../Backend/OneDrive/MicrosoftGraphBackend.cs | 49 ++++---- .../Backend/OpenStack/OpenStackStorage.cs | 54 ++++----- Duplicati/Library/Backend/Rclone/Rclone.cs | 52 ++++----- Duplicati/Library/Backend/S3/IS3Client.cs | 2 +- Duplicati/Library/Backend/S3/S3AwsClient.cs | 44 +++---- Duplicati/Library/Backend/S3/S3Backend.cs | 50 ++++---- Duplicati/Library/Backend/S3/S3MinioClient.cs | 30 ++++- .../Library/Backend/SSHv2/SSHv2Backend.cs | 106 +++++++++-------- .../Backend/SharePoint/SharePointBackend.cs | 20 +++- Duplicati/Library/Backend/Sia/Sia.cs | 48 ++++---- .../Library/Backend/Storj/StorjBackend.cs | 17 +-- .../Library/Backend/TahoeLAFS/TahoeBackend.cs | 54 ++++----- .../Library/Backend/TencentCOS/COSBackend.cs | 23 ++-- Duplicati/Library/Backend/WEBDAV/WEBDAV.cs | 65 +++++------ .../Library/Backend/pCloud/pCloudBackend.cs | 13 +-- .../Library/Interface/BackendExtensions.cs | 15 +-- Duplicati/Library/Interface/IBackend.cs | 3 +- .../Backend/BackendManager.DeleteOperation.cs | 2 +- .../Backend/BackendManager.ListOperation.cs | 7 +- .../Backend/BackendManager.PutOperation.cs | 4 +- .../UnitTest/DeterministicErrorBackend.cs | 4 +- Duplicati/UnitTest/Issue5845.cs | 4 +- Duplicati/UnitTest/RandomErrorBackend.cs | 42 +++---- Duplicati/UnitTest/SVNCheckoutsTest.cs | 43 +++---- Duplicati/UnitTest/SizeOmittingBackend.cs | 42 +++---- Tools/RemoteSynchronization/Program.cs | 8 +- 51 files changed, 792 insertions(+), 787 deletions(-) diff --git a/Duplicati/CommandLine/BackendTester/Program.cs b/Duplicati/CommandLine/BackendTester/Program.cs index 52a60857b..138226927 100644 --- a/Duplicati/CommandLine/BackendTester/Program.cs +++ b/Duplicati/CommandLine/BackendTester/Program.cs @@ -29,11 +29,12 @@ using System.Globalization; using System.Threading; using Duplicati.Library.Utility; using Duplicati.StreamUtil; +using System.Threading.Tasks; namespace Duplicati.CommandLine.BackendTester { public class Program - { + { class TempFile { public readonly string remotefilename; @@ -173,7 +174,7 @@ namespace Duplicati.CommandLine.BackendTester try { backend.TestAsync(CancellationToken.None).Await(); - curlist = backend.List(); + curlist = backend.ListAsync(CancellationToken.None).ToBlockingEnumerable().ToList(); } catch (FolderMissingException) { @@ -182,7 +183,7 @@ namespace Duplicati.CommandLine.BackendTester try { backend.CreateFolderAsync(CancellationToken.None).Await(); - curlist = backend.List(); + curlist = backend.ListAsync(CancellationToken.None).ToBlockingEnumerable().ToList(); } catch (Exception ex) { @@ -250,20 +251,20 @@ namespace Duplicati.CommandLine.BackendTester throttleDownload = Duplicati.Library.Utility.Sizeparser.ParseSize(throttleDownloadString, "kb"); } - int readWriteTimeout = backend is ITimeoutExemptBackend || - (Environment.GetEnvironmentVariable("READ_WRITE_TIMEOUT_SECONDS") is { } timeout - && int.TryParse(timeout, out var seconds) + int readWriteTimeout = backend is ITimeoutExemptBackend || + (Environment.GetEnvironmentVariable("READ_WRITE_TIMEOUT_SECONDS") is { } timeout + && int.TryParse(timeout, out var seconds) && seconds == -1) ? Timeout.Infinite - : Environment.GetEnvironmentVariable("READ_WRITE_TIMEOUT_SECONDS") is { } timeoutRetry + : Environment.GetEnvironmentVariable("READ_WRITE_TIMEOUT_SECONDS") is { } timeoutRetry && int.TryParse(timeoutRetry, out var secondsRetry) ? (int)TimeSpan.FromSeconds(secondsRetry).TotalMilliseconds : (int)TimeSpan.FromSeconds(60).TotalMilliseconds; - Console.WriteLine(LogTimeStamp + "Read Write Timeout set to {0}{1}", + Console.WriteLine(LogTimeStamp + "Read Write Timeout set to {0}{1}", readWriteTimeout == Timeout.Infinite ? "infinite" : readWriteTimeout + " ms", backend is ITimeoutExemptBackend ? " (because implements ITimeoutExemptBackend)" : ""); - + // Allow overriding the timeout for the backend here, even if timeouts are disabled if (options.TryGetValue("read-write-timeout", out var readWriteTimeoutString)) readWriteTimeout = (int)Timeparser.ParseTimeSpan(readWriteTimeoutString).TotalMilliseconds; @@ -353,7 +354,7 @@ namespace Duplicati.CommandLine.BackendTester Console.WriteLine(LogTimeStamp + "Verifying file list ..."); - curlist = backend.List(); + curlist = backend.ListAsync(CancellationToken.None).ToBlockingEnumerable().ToList(); foreach (var fe in curlist) if (!fe.IsFolder) { @@ -460,7 +461,7 @@ namespace Duplicati.CommandLine.BackendTester Thread.Sleep(waitAfterDelete); } - curlist = backend.List(); + curlist = backend.ListAsync(CancellationToken.None).ToBlockingEnumerable().ToList(); foreach (var fe in curlist) if (!fe.IsFolder) { diff --git a/Duplicati/CommandLine/BackendTool/Program.cs b/Duplicati/CommandLine/BackendTool/Program.cs index 46782329d..071f27037 100644 --- a/Duplicati/CommandLine/BackendTool/Program.cs +++ b/Duplicati/CommandLine/BackendTool/Program.cs @@ -26,6 +26,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading; +using System.Threading.Tasks; namespace Duplicati.CommandLine.BackendTool { @@ -124,7 +125,7 @@ namespace Duplicati.CommandLine.BackendTool throw new UserInformationException(string.Format("too many arguments: {0}", string.Join(",", args)), "BackendToolTooManyArguments"); Console.WriteLine("{0}\t{1}\t{2}\t{3}", "Name", "Dir/File", "LastChange", "Size"); - foreach (var e in backend.List()) + foreach (var e in backend.ListAsync(CancellationToken.None).ToBlockingEnumerable()) Console.WriteLine("{0}\t{1}\t{2}\t{3}", e.Name, e.IsFolder ? "Dir" : "File", e.LastModification, e.Size < 0 ? "" : Library.Utility.Utility.FormatSizeString(e.Size)); return 0; diff --git a/Duplicati/CommandLine/RecoveryTool/Download.cs b/Duplicati/CommandLine/RecoveryTool/Download.cs index f5cf2d949..21f023ff4 100644 --- a/Duplicati/CommandLine/RecoveryTool/Download.cs +++ b/Duplicati/CommandLine/RecoveryTool/Download.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -25,6 +25,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading; using Duplicati.Library.Utility; +using System.Threading.Tasks; namespace Duplicati.CommandLine.RecoveryTool { @@ -56,7 +57,7 @@ namespace Duplicati.CommandLine.RecoveryTool Console.WriteLine("Listing files on backend: {0} ...", backend.ProtocolKey); - var lst = backend.List().ToList(); + var lst = backend.ListAsync(CancellationToken.None).ToBlockingEnumerable().ToList(); Console.WriteLine("Found {0} files", lst.Count); diff --git a/Duplicati/CommandLine/RecoveryTool/Recompress.cs b/Duplicati/CommandLine/RecoveryTool/Recompress.cs index ce83d6b75..42a92542e 100644 --- a/Duplicati/CommandLine/RecoveryTool/Recompress.cs +++ b/Duplicati/CommandLine/RecoveryTool/Recompress.cs @@ -29,7 +29,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading; -using System.Security.Cryptography; +using System.Threading.Tasks; namespace Duplicati.CommandLine.RecoveryTool { @@ -71,7 +71,7 @@ namespace Duplicati.CommandLine.RecoveryTool Console.WriteLine("Listing files on backend: {0} ...", backend.ProtocolKey); - var rawlist = backend.List().ToList(); + var rawlist = backend.ListAsync(CancellationToken.None).ToBlockingEnumerable().ToList(); Console.WriteLine("Found {0} files at remote storage", rawlist.Count); diff --git a/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs b/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs index 158e54cdd..53dcb37b7 100644 --- a/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs +++ b/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs @@ -22,11 +22,11 @@ using Aliyun.OSS; using Aliyun.OSS.Common; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; -using Duplicati.Library.Utility; using System; using System.Collections.Generic; using System.IO; using System.Net; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; @@ -99,7 +99,8 @@ namespace Duplicati.Library.Backend.AliyunOSS }); } - public IEnumerable List() + + public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { var bucketName = _ossOptions.BucketName; @@ -107,51 +108,48 @@ namespace Duplicati.Library.Backend.AliyunOSS var client = GetClient(false); - var list = new List(); - try + var nextMarker = string.Empty; + bool isTruncated; + do { - var nextMarker = string.Empty; - var isTruncated = false; - do + var listObjectsRequest = new ListObjectsRequest(bucketName) { - var listObjectsRequest = new ListObjectsRequest(bucketName) - { - MaxKeys = 1000, - Marker = nextMarker, - Prefix = prefix, - }; - var result = client.ListObjects(listObjectsRequest); + MaxKeys = 1000, + Marker = nextMarker, + Prefix = prefix, + }; + + ObjectListing result; + try + { + result = await Task.Run(() => client.ListObjects(listObjectsRequest), cancelToken).ConfigureAwait(ConfigureAwaitOptions.ForceYielding); if (result.HttpStatusCode != HttpStatusCode.OK) { throw new Exception(result.HttpStatusCode.ToString()); } - foreach (var summary in result.ObjectSummaries) - { - list.Add(summary); - } + } + catch (OssException ex) + { + Logging.Log.WriteErrorMessage(LOGTAG, "List", ex, "List object failed. {0}, {1}", ex.Message, ex.ErrorCode); + throw; + } + catch (Exception ex) + { + Logging.Log.WriteErrorMessage(LOGTAG, "List", ex, "List object failed. {0}", ex.Message); + throw; + } - nextMarker = result.NextMarker; - isTruncated = result.IsTruncated; - } while (isTruncated); - } - catch (OssException ex) - { - Logging.Log.WriteErrorMessage(LOGTAG, "List", ex, "List object failed. {0}, {1}", ex.Message, ex.ErrorCode); - throw; - } - catch (Exception ex) - { - Logging.Log.WriteErrorMessage(LOGTAG, "List", ex, "List object failed. {0}", ex.Message); - throw; - } + foreach (var summary in result.ObjectSummaries) + { + var fileName = Path.GetFileName(summary.Key); + var time = summary.LastModified; // DateTimeOffset.Parse(summary.LastModified).ToLocalTime().DateTime; + yield return new FileEntry(fileName, summary.Size, time, time); + } - foreach (var item in list) - { - var fileName = Path.GetFileName(item.Key); - var time = item.LastModified; // DateTimeOffset.Parse(item.LastModified).ToLocalTime().DateTime; - yield return new FileEntry(fileName, item.Size, time, time); - } + nextMarker = result.NextMarker; + isTruncated = result.IsTruncated; + } while (isTruncated); } public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) diff --git a/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs b/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs index 1eb5f5bcb..2c6b6fcd8 100644 --- a/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs +++ b/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs @@ -92,10 +92,8 @@ namespace Duplicati.Library.Backend.AzureBlob get { return "azure"; } } - public IEnumerable List() - { - return _azureBlob.ListContainerEntries(); - } + public IAsyncEnumerable ListAsync(CancellationToken cancelToken) + => _azureBlob.ListContainerEntriesAsync(cancelToken); public async Task PutAsync(string remotename, string localname, CancellationToken cancelToken) { @@ -172,9 +170,7 @@ namespace Duplicati.Library.Backend.AzureBlob public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(_azureBlob.DnsNames); public Task TestAsync(CancellationToken cancellationToken) - { - return WrapWithExceptionHandler(Task.Run(() => this.TestList())); - } + => this.TestListAsync(cancellationToken); public Task CreateFolderAsync(CancellationToken cancellationToken) { diff --git a/Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs b/Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs index b18290ae6..462ec6abd 100644 --- a/Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs +++ b/Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs @@ -21,7 +21,7 @@ using System.Collections.Generic; using System.IO; -using System.Linq; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Common.IO; @@ -118,53 +118,13 @@ namespace Duplicati.Library.Backend.AzureBlob return _container.GetBlockBlobReference(keyName).DeleteIfExistsAsync(default, default, default, _operationContext, cancelToken); } - private async Task> ListContainerEntriesAsync() + private async IAsyncEnumerable ListContainerBlobEntriesAsync([EnumeratorCancellation] CancellationToken cancelToken) { - var segment = await _container.ListBlobsSegmentedAsync(null, false, ListDetails, null, null, null, _operationContext); - var list = new List(); - - list.AddRange(segment.Results); - - while (segment.ContinuationToken != null) - { - segment = await _container.ListBlobsSegmentedAsync(null, false, ListDetails, null, segment.ContinuationToken, null, _operationContext); - list.AddRange(segment.Results); - } - - return list; - } - - public virtual List ListContainerEntries() - { - var listBlobItems = ListContainerEntriesAsync().GetAwaiter().GetResult(); + BlobResultSegment segment; try { - return listBlobItems.Select(x => - { - var absolutePath = x.StorageUri.PrimaryUri.AbsolutePath; - var containerSegment = string.Concat("/", _containerName, "/"); - var blobName = absolutePath.Substring(absolutePath.IndexOf( - containerSegment, System.StringComparison.Ordinal) + containerSegment.Length); - try - { - if (x is CloudBlockBlob cb) - { - var lastModified = new System.DateTime(); - if (cb.Properties.LastModified != null) - lastModified = new System.DateTime(cb.Properties.LastModified.Value.Ticks, System.DateTimeKind.Utc); - return new FileEntry(Uri.UrlDecode(blobName.Replace("+", "%2B")), cb.Properties.Length, lastModified, lastModified); - } - } - catch - { - // If the metadata fails to parse, return the basic entry - } - - return new FileEntry(Uri.UrlDecode(blobName.Replace("+", "%2B"))); - }) - .Cast() - .ToList(); + segment = await _container.ListBlobsSegmentedAsync(null, false, ListDetails, null, null, null, _operationContext, cancelToken).ConfigureAwait(false); } catch (StorageException ex) { @@ -174,6 +134,46 @@ namespace Duplicati.Library.Backend.AzureBlob } throw; } + + foreach (var item in segment.Results) + yield return item; + + while (segment.ContinuationToken != null) + { + segment = await _container.ListBlobsSegmentedAsync(null, false, ListDetails, null, segment.ContinuationToken, null, _operationContext); + + foreach (var item in segment.Results) + yield return item; + } + } + + public virtual async IAsyncEnumerable ListContainerEntriesAsync([EnumeratorCancellation] CancellationToken cancelToken) + { + await foreach (var x in ListContainerBlobEntriesAsync(cancelToken).ConfigureAwait(false)) + { + var absolutePath = x.StorageUri.PrimaryUri.AbsolutePath; + var containerSegment = string.Concat("/", _containerName, "/"); + var blobName = absolutePath.Substring(absolutePath.IndexOf( + containerSegment, System.StringComparison.Ordinal) + containerSegment.Length); + + var res = new FileEntry(Uri.UrlDecode(blobName.Replace("+", "%2B"))); + try + { + if (x is CloudBlockBlob cb) + { + var lastModified = new System.DateTime(); + if (cb.Properties.LastModified != null) + lastModified = new System.DateTime(cb.Properties.LastModified.Value.Ticks, System.DateTimeKind.Utc); + res = new FileEntry(res.Name, cb.Properties.Length, lastModified, lastModified); + } + } + catch + { + // If the metadata fails to parse, return the basic entry + } + + yield return res; + } } } } diff --git a/Duplicati/Library/Backend/Backblaze/B2.cs b/Duplicati/Library/Backend/Backblaze/B2.cs index 08c2437dc..8d72ed7c0 100644 --- a/Duplicati/Library/Backend/Backblaze/B2.cs +++ b/Duplicati/Library/Backend/Backblaze/B2.cs @@ -19,7 +19,6 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System.Diagnostics; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using Duplicati.Library.Utility; @@ -28,6 +27,7 @@ using System.Net.Http.Headers; using System.Reflection; using Duplicati.Library.Backend.Backblaze.Model; using FileEntry = Duplicati.Library.Common.IO.FileEntry; +using System.Runtime.CompilerServices; namespace Duplicati.Library.Backend.Backblaze; @@ -80,7 +80,7 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend /// Recommended chunk size as per Backblaze B2 documentation (100MB) /// private const int B2_RECOMMENDED_CHUNK_SIZE = 100 * 1024 * 1024; - + /// /// Default retry-after time in seconds for B2 API requests /// @@ -283,14 +283,15 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend /// Retrieves the fileID searching by filename. /// /// Filename + /// Cancellation Token /// /// Exception thrown when file is not found - private string GetFileId(string filename) + private async Task GetFileId(string filename, CancellationToken cancelToken) { if (_filecache != null && _filecache.TryGetValue(filename, out var value)) return value.OrderByDescending(x => x.UploadTimestamp).First().FileID; - List(); + await RebuildFileCache(cancelToken).ConfigureAwait(false); if (_filecache != null && _filecache.TryGetValue(filename, out var value1)) return value1.OrderByDescending(x => x.UploadTimestamp).First().FileID; @@ -358,7 +359,7 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend } if (_filecache == null) - await ListAsync(cancelToken).ConfigureAwait(false); + await RebuildFileCache(cancelToken).ConfigureAwait(false); var uploadUrlData = await GetUploadUrlDataAsync(cancelToken).ConfigureAwait(false); try @@ -427,11 +428,11 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend { if (_filecache == null || !_filecache.ContainsKey(remotename)) - await ListAsync(cancellationToken).ConfigureAwait(false); + await RebuildFileCache(cancellationToken).ConfigureAwait(false); using var request = _filecache != null && _filecache.ContainsKey(remotename) ? _b2AuthHelper.CreateRequest( - $"{DownloadUrl}/b2api/v1/b2_download_file_by_id?fileId={Utility.Uri.UrlEncode(GetFileId(remotename))}") + $"{DownloadUrl}/b2api/v1/b2_download_file_by_id?fileId={Utility.Uri.UrlEncode(await GetFileId(remotename, cancellationToken))}") : _b2AuthHelper.CreateRequest( $"{DownloadUrl}/{_urlencodedPrefix}{Utility.Uri.UrlPathEncode(remotename)}"); @@ -463,7 +464,7 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend /// Implementation of interface method for listing remote folder contents /// /// List of IFileEntry with directory listing result - private async Task> ListAsync(CancellationToken cancellationToken, bool testMode = false) + private async Task RebuildFileCache(CancellationToken cancellationToken) { _filecache = null; var cache = new Dictionary>(); @@ -524,28 +525,28 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend new RetryConditionHeaderValue(TimeSpan.FromSeconds(B2_RETRY_AFTER_SECONDS))); else listRetryHelper.SetRetryAfter(tex.RetryAfter); - + await listRetryHelper.WaitForRetryAfterAsync(cancellationToken); } - if (testMode) // For testing purposes, it suffices to get the first page of results. - break; } while (nextFileId != null); _filecache = cache; - - return - (from x in _filecache - let newest = x.Value.OrderByDescending(y => y.UploadTimestamp).First() - let ts = Utility.Utility.EPOCH.AddMilliseconds(newest.UploadTimestamp) - select (IFileEntry)new FileEntry(x.Key, newest.Size, ts, ts) - ).ToList(); } /// - /// Wrapper method of legacy non async call to list files in the remote folder + /// Implementation of interface method for listing remote folder contents /// - /// - public IEnumerable List() => ListAsync(CancellationToken.None).Await(); + /// List of IFileEntry with directory listing result + public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancellationToken) + { + await RebuildFileCache(cancellationToken).ConfigureAwait(false); + foreach (var x in _filecache) + { + var newest = x.Value.OrderByDescending(y => y.UploadTimestamp).First(); + var ts = Utility.Utility.EPOCH.AddMilliseconds(newest.UploadTimestamp); + yield return new FileEntry(x.Key, newest.Size, ts, ts); + } + } // public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) @@ -585,7 +586,7 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend using var combinedCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken); if (_filecache == null || !_filecache.ContainsKey(remotename)) - await ListAsync(cancellationToken).ConfigureAwait(false); + await RebuildFileCache(cancellationToken).ConfigureAwait(false); if (!_filecache.TryGetValue(remotename, out var value)) throw new FileMissingException(); @@ -615,10 +616,7 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend /// /// Cancellation Token public Task TestAsync(CancellationToken cancelToken) - { - _ = ListAsync(cancelToken, true).Await(); - return Task.CompletedTask; - } + => this.TestListAsync(cancelToken); /// /// Create remote folder diff --git a/Duplicati/Library/Backend/Backblaze/Duplicati.Library.Backend.Backblaze.csproj b/Duplicati/Library/Backend/Backblaze/Duplicati.Library.Backend.Backblaze.csproj index fbdff62cf..b97001bf1 100644 --- a/Duplicati/Library/Backend/Backblaze/Duplicati.Library.Backend.Backblaze.csproj +++ b/Duplicati/Library/Backend/Backblaze/Duplicati.Library.Backend.Backblaze.csproj @@ -8,6 +8,7 @@ + diff --git a/Duplicati/Library/Backend/Box/BoxBackend.cs b/Duplicati/Library/Backend/Box/BoxBackend.cs index 98c8c50cb..d256255d9 100644 --- a/Duplicati/Library/Backend/Box/BoxBackend.cs +++ b/Duplicati/Library/Backend/Box/BoxBackend.cs @@ -1,32 +1,32 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; -using Duplicati.Library.Utility; using Newtonsoft.Json; using System; using System.Collections.Generic; using System.Linq; using System.Net; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; @@ -133,7 +133,7 @@ namespace Duplicati.Library.Backend.Box foreach (var p in m_path.Split(new string[] { "/" }, StringSplitOptions.RemoveEmptyEntries)) { - var el = (MiniFolder)PagedFileListResponse(parentid, true).FirstOrDefault(x => x.Name == p); + var el = (MiniFolder)await PagedFileListResponse(parentid, true, cancelToken).FirstOrDefaultAsync(x => x.Name == p).ConfigureAwait(false); if (el == null) { if (!create) @@ -159,7 +159,7 @@ namespace Duplicati.Library.Backend.Box // Make sure we enumerate this, otherwise the m_filecache is empty. var currentFolder = await GetCurrentFolderWithCacheAsync(cancelToken).ConfigureAwait(false); - PagedFileListResponse(currentFolder, false).LastOrDefault(); + await PagedFileListResponse(currentFolder, false, cancelToken).LastOrDefaultAsync().ConfigureAwait(false); if (m_filecache.ContainsKey(name)) return m_filecache[name]; @@ -167,7 +167,7 @@ namespace Duplicati.Library.Backend.Box throw new FileMissingException(); } - private IEnumerable PagedFileListResponse(string parentid, bool onlyfolders) + private async IAsyncEnumerable PagedFileListResponse(string parentid, bool onlyfolders, [EnumeratorCancellation] CancellationToken cancelToken) { var offset = 0; var done = false; @@ -177,7 +177,7 @@ namespace Duplicati.Library.Backend.Box do { - var resp = m_oauth.GetJSONData(string.Format("{0}/folders/{1}/items?limit={2}&offset={3}&fields=name,size,modified_at", BOX_API_URL, parentid, PAGE_SIZE, offset)); + var resp = await m_oauth.GetJSONDataAsync($"{BOX_API_URL}/folders/{parentid}/items?limit={PAGE_SIZE}&offset={offset}&fields=name,size,modified_at", cancelToken).ConfigureAwait(false); if (resp.Entries == null || resp.Entries.Length == 0) break; @@ -221,7 +221,7 @@ namespace Duplicati.Library.Backend.Box }; if (m_filecache.Count == 0) - PagedFileListResponse(currentFolder, false); + await PagedFileListResponse(currentFolder, false, cancelToken).LastOrDefaultAsync().ConfigureAwait(false); var existing = m_filecache.ContainsKey(remotename); @@ -262,12 +262,11 @@ namespace Duplicati.Library.Backend.Box #region IBackend implementation - public System.Collections.Generic.IEnumerable List() + public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { - var currentFolder = GetCurrentFolderWithCacheAsync(CancellationToken.None).Await(); - return - from n in PagedFileListResponse(currentFolder, false) - select (IFileEntry)new FileEntry(n.Name, n.Size, n.ModifiedAt, n.ModifiedAt) { IsFolder = n.Type == "folder" }; + var currentFolder = await GetCurrentFolderWithCacheAsync(cancelToken).ConfigureAwait(false); + await foreach (var n in PagedFileListResponse(currentFolder, false, cancelToken).ConfigureAwait(false)) + yield return new FileEntry(n.Name, n.Size, n.ModifiedAt, n.ModifiedAt) { IsFolder = n.Type == "folder" }; } public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) @@ -304,10 +303,7 @@ namespace Duplicati.Library.Backend.Box } public Task TestAsync(CancellationToken cancelToken) - { - this.TestList(); - return Task.CompletedTask; - } + => this.TestListAsync(cancelToken); public Task CreateFolderAsync(CancellationToken cancellationToken) { diff --git a/Duplicati/Library/Backend/Box/Duplicati.Library.Backend.Box.csproj b/Duplicati/Library/Backend/Box/Duplicati.Library.Backend.Box.csproj index 261a7a42f..d9cbc0442 100644 --- a/Duplicati/Library/Backend/Box/Duplicati.Library.Backend.Box.csproj +++ b/Duplicati/Library/Backend/Box/Duplicati.Library.Backend.Box.csproj @@ -8,6 +8,7 @@ + diff --git a/Duplicati/Library/Backend/CIFS/CIFS.cs b/Duplicati/Library/Backend/CIFS/CIFS.cs index 815773fad..13fab1caf 100644 --- a/Duplicati/Library/Backend/CIFS/CIFS.cs +++ b/Duplicati/Library/Backend/CIFS/CIFS.cs @@ -28,8 +28,8 @@ using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Backend.CIFS; using Duplicati.Library.Backend.CIFS.Model; -using Duplicati.Library.Utility; using SMBLibrary; +using System.Runtime.CompilerServices; namespace Duplicati.Library.Backend; @@ -72,12 +72,12 @@ public class CIFSBackend : IStreamingBackend /// Shared connection between all methods to avoid re-authentication /// private SMBShareConnection _sharedConnection; - + /// /// Read buffer size for SMB operations (will be capped automatically by SMB negotiated values) /// private const string READ_BUFFER_SIZE_OPTION = "read-buffer-size"; - + /// /// Write buffer size for SMB operations (will be capped automatically by SMB negotiated values) /// @@ -139,33 +139,33 @@ public class CIFSBackend : IStreamingBackend var uri = new Utility.Uri(url); uri.RequireHost(); _DnsName = uri.Host; - + var input = uri.Path.TrimEnd('/'); var slashIndex = input.IndexOf('/'); // Find first slash to separate server and share if present. - + options.TryGetValue(AUTH_USERNAME_OPTION, out string authUsername); options.TryGetValue(AUTH_PASSWORD_OPTION, out string authPassword); options.TryGetValue(AUTH_DOMAIN_OPTION, out string authDomain); options.TryGetValue(TRANSPORT_OPTION, out string transport); - + int? readBufferSize = null, writeBufferSize = null; - + options.TryGetValue(READ_BUFFER_SIZE_OPTION, out string readBufferSizeConfig); if (!string.IsNullOrWhiteSpace(readBufferSizeConfig)) readBufferSize = Int32.TryParse(readBufferSizeConfig, out int value) ? value : null; - + options.TryGetValue(WRITE_BUFFER_SIZE_OPTION, out string writeBufferSizeConfig); if (!string.IsNullOrWhiteSpace(writeBufferSizeConfig)) writeBufferSize = Int32.TryParse(readBufferSizeConfig, out int value) ? value : null; - + // Normalize to 10KB minimum buffers size readBufferSize = readBufferSize < 1024 * 10 ? null : readBufferSize; writeBufferSize = writeBufferSize < 1024 * 10 ? null : writeBufferSize; SMBTransportType transportType = _transportMap.TryGetValue( - string.IsNullOrEmpty(transport) ? DEFAULT_TRANSPORT : transport.ToLower(), + string.IsNullOrEmpty(transport) ? DEFAULT_TRANSPORT : transport.ToLower(), out SMBTransportType type) ? type - : throw new UserInformationException($"Transport must be one of: {string.Join(", ", _transportMap.Keys)}","CIFSConfig"); - + : throw new UserInformationException($"Transport must be one of: {string.Join(", ", _transportMap.Keys)}", "CIFSConfig"); + _connectionParameters = new SMBConnectionParameters( uri.Host, transportType, @@ -196,17 +196,12 @@ public class CIFSBackend : IStreamingBackend /// Implementation of interface method for listing remote folder contents /// /// List of IFileEntry with directory listing result - private async Task> ListAsync(CancellationToken cancellationToken) + public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancellationToken) { - return await GetConnection().ListAsync(cancellationToken).ConfigureAwait(false); + foreach (var v in await GetConnection().ListAsync(cancellationToken).ConfigureAwait(false)) + yield return v; } - /// - /// Wrapper method of legacy non async call to list files in the remote folder - /// - /// - public IEnumerable List() => ListAsync(CancellationToken.None).Await(); - /// /// Upload files to remote location /// @@ -282,7 +277,7 @@ public class CIFSBackend : IStreamingBackend /// /// CancellationToken, in this call not used. /// - public Task GetDNSNamesAsync(CancellationToken cancellationToken) => + public Task GetDNSNamesAsync(CancellationToken cancellationToken) => Task.FromResult(new[] { _DnsName ?? string.Empty }); /// @@ -306,10 +301,10 @@ public class CIFSBackend : IStreamingBackend { var pathParts = _connectionParameters.Path? .Split(PATH_SEPARATORS, StringSplitOptions.RemoveEmptyEntries); - + if (pathParts == null || pathParts.Length == 0) return; - + await GetConnection().CreateFolderAsync(_connectionParameters.Path, cancellationToken) .ConfigureAwait(false); } diff --git a/Duplicati/Library/Backend/CloudFiles/CloudFiles.cs b/Duplicati/Library/Backend/CloudFiles/CloudFiles.cs index dbcd48dba..22329ead1 100644 --- a/Duplicati/Library/Backend/CloudFiles/CloudFiles.cs +++ b/Duplicati/Library/Backend/CloudFiles/CloudFiles.cs @@ -1,31 +1,31 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; -using Duplicati.Library.Utility; using System; using System.Collections.Generic; using System.Linq; using System.Net; +using System.Runtime.CompilerServices; using System.Security.Cryptography; using System.Threading; using System.Threading.Tasks; @@ -133,10 +133,10 @@ namespace Duplicati.Library.Backend get { return "cloudfiles"; } } - public IEnumerable List() + public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { - string extraUrl = "?format=xml&limit=" + ITEM_LIST_LIMIT.ToString(); - string markerUrl = ""; + var extraUrl = "?format=xml&limit=" + ITEM_LIST_LIMIT.ToString(); + var markerUrl = ""; bool repeat; @@ -169,14 +169,14 @@ namespace Duplicati.Library.Backend //The response should be 404 from the server, but it is not :( if (lst.Count == 0 && markerUrl == "") //Only on first iteration { - try { CreateFolderAsync(CancellationToken.None).Await(); } + try { await CreateFolderAsync(cancelToken).ConfigureAwait(false); } catch { } //Ignore } - string lastItemName = ""; + var lastItemName = ""; foreach (System.Xml.XmlNode n in lst) { - string name = n["name"].InnerText; + var name = n["name"].InnerText; long size; DateTime mod; @@ -264,12 +264,9 @@ namespace Duplicati.Library.Backend #region IBackend_v2 Members - public Task TestAsync(CancellationToken cancelToken) - { + public Task TestAsync(CancellationToken cancelToken) => //The "Folder not found" is not detectable :( - this.TestList(); - return Task.CompletedTask; - } + this.TestListAsync(cancelToken); public Task CreateFolderAsync(CancellationToken cancelToken) { diff --git a/Duplicati/Library/Backend/Dropbox/Dropbox.cs b/Duplicati/Library/Backend/Dropbox/Dropbox.cs index 1713a730e..e929d9da9 100644 --- a/Duplicati/Library/Backend/Dropbox/Dropbox.cs +++ b/Duplicati/Library/Backend/Dropbox/Dropbox.cs @@ -23,6 +23,7 @@ using Duplicati.Library.Interface; using System; using System.Collections.Generic; using System.IO; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; @@ -112,16 +113,31 @@ namespace Duplicati.Library.Backend } } - public IEnumerable List() + private async Task HandleListExceptions(Func> func) { - var lfr = HandleListExceptions(() => dbx.ListFiles(m_path)); + try + { + return await func().ConfigureAwait(false); + } + catch (DropboxException de) + { + if (de.errorJSON["error"][".tag"].ToString() == "path" && de.errorJSON["error"]["path"][".tag"].ToString() == "not_found") + throw new FolderMissingException(); + + throw; + } + } + + public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) + { + var lfr = await HandleListExceptions(() => dbx.ListFiles(m_path, cancelToken)).ConfigureAwait(false); foreach (var md in lfr.entries) yield return ParseEntry(md); while (lfr.has_more) { - lfr = HandleListExceptions(() => dbx.ListFilesContinue(lfr.cursor)); + lfr = await HandleListExceptions(() => dbx.ListFilesContinue(lfr.cursor, cancelToken)).ConfigureAwait(false); foreach (var md in lfr.entries) yield return ParseEntry(md); } @@ -168,10 +184,7 @@ namespace Duplicati.Library.Backend public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(WebApi.Dropbox.Hosts()); public Task TestAsync(CancellationToken cancelToken) - { - this.TestList(); - return Task.CompletedTask; - } + => this.TestListAsync(cancelToken); public async Task CreateFolderAsync(CancellationToken cancelToken) { diff --git a/Duplicati/Library/Backend/Dropbox/DropboxHelper.cs b/Duplicati/Library/Backend/Dropbox/DropboxHelper.cs index a8d6e3cf7..cb9d23c2a 100644 --- a/Duplicati/Library/Backend/Dropbox/DropboxHelper.cs +++ b/Duplicati/Library/Backend/Dropbox/DropboxHelper.cs @@ -43,7 +43,7 @@ namespace Duplicati.Library.Backend base.AccessTokenOnly = !accessToken.Contains(":"); } - public ListFolderResult ListFiles(string path) + public async Task ListFiles(string path, CancellationToken cancelToken) { var pa = new PathArg { @@ -52,7 +52,7 @@ namespace Duplicati.Library.Backend try { - return PostAndGetJSONData(WebApi.Dropbox.ListFilesUrl(), pa); + return await PostAndGetJSONDataAsync(WebApi.Dropbox.ListFilesUrl(), cancelToken, pa).ConfigureAwait(false); } catch (Exception ex) { @@ -61,13 +61,13 @@ namespace Duplicati.Library.Backend } } - public ListFolderResult ListFilesContinue(string cursor) + public async Task ListFilesContinue(string cursor, CancellationToken cancelToken) { var lfca = new ListFolderContinueArg() { cursor = cursor }; try { - return PostAndGetJSONData(WebApi.Dropbox.ListFilesContinueUrl(), lfca); + return await PostAndGetJSONDataAsync(WebApi.Dropbox.ListFilesContinueUrl(), cancelToken, lfca).ConfigureAwait(false); } catch (Exception ex) { diff --git a/Duplicati/Library/Backend/FTP/Duplicati.Library.Backend.FTP.csproj b/Duplicati/Library/Backend/FTP/Duplicati.Library.Backend.FTP.csproj index 1a417bc16..3f6aade5c 100644 --- a/Duplicati/Library/Backend/FTP/Duplicati.Library.Backend.FTP.csproj +++ b/Duplicati/Library/Backend/FTP/Duplicati.Library.Backend.FTP.csproj @@ -19,6 +19,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/Duplicati/Library/Backend/FTP/FTPBackend.cs b/Duplicati/Library/Backend/FTP/FTPBackend.cs index 939311ab5..1f5095120 100644 --- a/Duplicati/Library/Backend/FTP/FTPBackend.cs +++ b/Duplicati/Library/Backend/FTP/FTPBackend.cs @@ -28,6 +28,7 @@ using System.IO; using System.Linq; using System.Net; using System.Net.Security; +using System.Runtime.CompilerServices; using System.Security.Authentication; using System.Text; using System.Threading; @@ -366,63 +367,14 @@ namespace Duplicati.Library.Backend } /// - public IEnumerable List() + public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { + FtpListItem[] items; try { - var list = new List(); var client = CreateClient(CancellationToken.None).Await(); var remotePath = PreparePathForClient(null); - - foreach (var item in client.GetListing(remotePath, FtpListOption.Modify | FtpListOption.Size).Await()) - { - switch (item.Type) - { - case FtpObjectType.Directory: - if (item.Name == "." || item.Name == "..") - continue; - - list.Add(new FileEntry(item.Name, -1, new DateTime(), item.Modified) - { - IsFolder = true, - }); - break; - - case FtpObjectType.File: - list.Add(new FileEntry(item.Name, item.Size, new DateTime(), item.Modified)); - break; - - case FtpObjectType.Link: - { - if (item.Name == "." || item.Name == "..") - continue; - - if (item.LinkObject != null) - { - switch (item.LinkObject.Type) - { - case FtpObjectType.Directory: - if (item.Name == "." || item.Name == "..") - continue; - - list.Add(new FileEntry(item.Name, -1, new DateTime(), item.Modified) - { - IsFolder = true, - }); - break; - - case FtpObjectType.File: - list.Add(new FileEntry(item.Name, item.Size, new DateTime(), item.Modified)); - break; - } - } - break; - } - - } - } - - return list; + items = await client.GetListing(remotePath, FtpListOption.Modify | FtpListOption.Size).ConfigureAwait(false); } catch (Exception e) { @@ -431,6 +383,54 @@ namespace Duplicati.Library.Backend throw; } + + foreach (var item in items) + { + switch (item.Type) + { + case FtpObjectType.Directory: + if (item.Name == "." || item.Name == "..") + continue; + + yield return new FileEntry(item.Name, -1, new DateTime(), item.Modified) + { + IsFolder = true, + }; + break; + + case FtpObjectType.File: + yield return new FileEntry(item.Name, item.Size, new DateTime(), item.Modified); + break; + + case FtpObjectType.Link: + { + if (item.Name == "." || item.Name == "..") + continue; + + if (item.LinkObject != null) + { + switch (item.LinkObject.Type) + { + case FtpObjectType.Directory: + if (item.Name == "." || item.Name == "..") + continue; + + yield return new FileEntry(item.Name, -1, new DateTime(), item.Modified) + { + IsFolder = true, + }; + break; + + case FtpObjectType.File: + yield return new FileEntry(item.Name, item.Size, new DateTime(), item.Modified); + break; + } + } + break; + } + + } + } } /// @@ -552,8 +552,7 @@ namespace Duplicati.Library.Backend // Remove the file if it exists try { - var list = List(); - if (list.Any(entry => entry.Name == TEST_FILE_NAME)) + if (await ListAsync(cancellationToken).AnyAsync(entry => entry.Name == TEST_FILE_NAME).ConfigureAwait(false)) await DeleteAsync(TEST_FILE_NAME, cancellationToken).ConfigureAwait(false); } catch (Exception e) diff --git a/Duplicati/Library/Backend/File/Duplicati.Library.Backend.File.csproj b/Duplicati/Library/Backend/File/Duplicati.Library.Backend.File.csproj index f8a99e4c3..51a86094d 100644 --- a/Duplicati/Library/Backend/File/Duplicati.Library.Backend.File.csproj +++ b/Duplicati/Library/Backend/File/Duplicati.Library.Backend.File.csproj @@ -18,6 +18,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/Duplicati/Library/Backend/File/FileBackend.cs b/Duplicati/Library/Backend/File/FileBackend.cs index 89dbb44fc..aae86fe65 100644 --- a/Duplicati/Library/Backend/File/FileBackend.cs +++ b/Duplicati/Library/Backend/File/FileBackend.cs @@ -24,6 +24,7 @@ using Duplicati.Library.Interface; using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Runtime.InteropServices; using System.Runtime.Versioning; using System.Threading; @@ -172,14 +173,15 @@ namespace Duplicati.Library.Backend get { return "file"; } } - public IEnumerable List() + /// + public IAsyncEnumerable ListAsync(CancellationToken cancelToken) { PreAuthenticate(); if (!systemIO.DirectoryExists(m_path)) throw new FolderMissingException(Strings.FileBackend.FolderMissingError(m_path)); - return systemIO.EnumerateFileEntries(m_path); + return systemIO.EnumerateFileEntries(m_path).ToAsyncEnumerable(); } #if DEBUG_RETRY @@ -281,10 +283,7 @@ namespace Duplicati.Library.Backend } public Task TestAsync(CancellationToken cancelToken) - { - this.TestList(); - return Task.CompletedTask; - } + => this.TestListAsync(cancelToken); public Task CreateFolderAsync(CancellationToken cancelToken) { diff --git a/Duplicati/Library/Backend/GoogleServices/Duplicati.Library.Backend.GoogleServices.csproj b/Duplicati/Library/Backend/GoogleServices/Duplicati.Library.Backend.GoogleServices.csproj index b9c7aa57c..198df6dd6 100644 --- a/Duplicati/Library/Backend/GoogleServices/Duplicati.Library.Backend.GoogleServices.csproj +++ b/Duplicati/Library/Backend/GoogleServices/Duplicati.Library.Backend.GoogleServices.csproj @@ -7,6 +7,7 @@ + diff --git a/Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs b/Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs index fae9abd03..30e9db6de 100644 --- a/Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs +++ b/Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Backend.GoogleServices; @@ -27,6 +27,7 @@ using Newtonsoft.Json; using System; using System.Collections.Generic; using System.Net; +using System.Runtime.CompilerServices; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -100,11 +101,11 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage public string storageClass { get; set; } } - private T HandleListExceptions(Func func) + private async Task HandleListExceptions(Func> func) { try { - return func(); + return await func().ConfigureAwait(false); } catch (WebException wex) { @@ -116,12 +117,13 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage } #region IBackend implementation - public IEnumerable List() + /// + public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { var url = WebApi.GoogleCloudStorage.ListUrl(m_bucket, Utility.Uri.UrlEncode(m_prefix)); while (true) { - var resp = HandleListExceptions(() => m_oauth.ReadJSONResponse(url)); + var resp = await HandleListExceptions(() => m_oauth.ReadJSONResponseAsync(url, cancelToken)).ConfigureAwait(false); if (resp.items != null) foreach (var f in resp.items) @@ -165,10 +167,7 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage } public Task TestAsync(CancellationToken cancelToken) - { - this.TestList(); - return Task.CompletedTask; - } + => this.TestListAsync(cancelToken); public async Task CreateFolderAsync(CancellationToken cancelToken) { diff --git a/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs b/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs index eea2ae4cb..560974d61 100644 --- a/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs +++ b/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs @@ -29,6 +29,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; +using System.Runtime.CompilerServices; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -78,8 +79,7 @@ namespace Duplicati.Library.Backend.GoogleDrive foreach (var p in path.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries)) { - var res = ListFolder(curparent, true, p).ToArray(); - + var res = await ListFolder(curparent, true, p, cancelToken).ToArrayAsync(cancelToken).ConfigureAwait(false); if (res.Length == 0) { if (!autocreate) @@ -119,7 +119,7 @@ namespace Duplicati.Library.Backend.GoogleDrive return entries; var currentFolderId = await GetCurrentFolderIdAsync(cancelToken).ConfigureAwait(false); - entries = ListFolder(currentFolderId, false, remotename).ToArray(); + entries = await ListFolder(currentFolderId, false, remotename, cancelToken).ToArrayAsync(cancelToken).ConfigureAwait(false); if (entries == null || entries.Length == 0) { @@ -145,7 +145,7 @@ namespace Duplicati.Library.Backend.GoogleDrive { // Figure out if we update or create the file if (m_filecache.Count == 0) - foreach (var file in List()) { /* Enumerate the full listing */ } + await foreach (var file in ListAsync(cancelToken)) { /* Enumerate the full listing */ } GoogleDriveFolderItem[] files; m_filecache.TryGetValue(remotename, out files); @@ -188,7 +188,7 @@ namespace Duplicati.Library.Backend.GoogleDrive { // Prevent repeated download url lookups if (m_filecache.Count == 0) - foreach (var file in List()) { /* Enumerate the full listing */ } + await foreach (var file in ListAsync(cancelToken)) { /* Enumerate the full listing */ } var fileId = (await GetFileEntriesAsync(remotename, true, cancelToken).ConfigureAwait(false)).OrderByDescending(x => x.createdDate).First().id; @@ -203,7 +203,9 @@ namespace Duplicati.Library.Backend.GoogleDrive #region IBackend implementation - public IEnumerable List() + /// + public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) + { bool success = false; try @@ -211,8 +213,8 @@ namespace Duplicati.Library.Backend.GoogleDrive m_filecache.Clear(); // For now, this class assumes that List() fully populates the file cache - var currentFolderId = GetCurrentFolderIdAsync(CancellationToken.None).Await(); - foreach (var n in ListFolder(currentFolderId)) + var currentFolderId = await GetCurrentFolderIdAsync(cancelToken).ConfigureAwait(false); + await foreach (var n in ListFolder(currentFolderId, null, null, cancelToken)) { FileEntry fe = null; @@ -294,10 +296,7 @@ namespace Duplicati.Library.Backend.GoogleDrive } public Task TestAsync(CancellationToken cancelToken) - { - this.TestList(); - return Task.CompletedTask; - } + => this.TestListAsync(cancelToken); public async Task CreateFolderAsync(CancellationToken cancelToken) { @@ -432,7 +431,7 @@ namespace Duplicati.Library.Backend.GoogleDrive public string rootFolderId { get; set; } } - private IEnumerable ListFolder(string parentfolder, bool? onlyFolders = null, string name = null) + private async IAsyncEnumerable ListFolder(string parentfolder, bool? onlyFolders, string name, [EnumeratorCancellation] CancellationToken cancelToken) { var fileQuery = new string[] { string.IsNullOrEmpty(name) ? null : string.Format("title = '{0}'", EscapeTitleEntries(name)), @@ -446,7 +445,7 @@ namespace Duplicati.Library.Backend.GoogleDrive while (true) { - var res = m_oauth.GetJSONData(url); + var res = await m_oauth.GetJSONDataAsync(url, cancelToken).ConfigureAwait(false); foreach (var n in res.items) yield return n; diff --git a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs index 59dca09bd..3cd5cf7f1 100644 --- a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs +++ b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs @@ -21,11 +21,11 @@ using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; -using Duplicati.Library.Utility; using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; @@ -126,9 +126,10 @@ namespace Duplicati.Library.Backend public bool SupportsStreaming => true; - public IEnumerable List() + /// + public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { - foreach (IFileEntry file in Connection.ListBucket(m_bucket, m_prefix)) + await foreach (IFileEntry file in Connection.ListBucketAsync(m_bucket, m_prefix, cancelToken)) { ((FileEntry)file).Name = file.Name.Substring(m_prefix.Length); if (file.Name.StartsWith("/", StringComparison.Ordinal) && !m_prefix.StartsWith("/", StringComparison.Ordinal)) @@ -189,10 +190,7 @@ namespace Duplicati.Library.Backend } public Task TestAsync(CancellationToken cancelToken) - { - this.TestList(); - return Task.CompletedTask; - } + => this.TestListAsync(cancelToken); public Task CreateFolderAsync(CancellationToken cancelToken) { diff --git a/Duplicati/Library/Backend/Jottacloud/Jottacloud.cs b/Duplicati/Library/Backend/Jottacloud/Jottacloud.cs index f7bdc164d..9a4c25336 100644 --- a/Duplicati/Library/Backend/Jottacloud/Jottacloud.cs +++ b/Duplicati/Library/Backend/Jottacloud/Jottacloud.cs @@ -25,6 +25,7 @@ using Duplicati.Library.Localization.Short; using System; using System.Collections.Generic; using System.Net; +using System.Runtime.CompilerServices; using System.Security.Cryptography; using System.Threading; using System.Threading.Tasks; @@ -182,8 +183,12 @@ namespace Duplicati.Library.Backend get { return "jottacloud"; } } - public IEnumerable List() + /// + public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { + // Remove warning until this backend is rewritten with HttpClient + await Task.CompletedTask; + var doc = new System.Xml.XmlDocument(); try { @@ -330,10 +335,7 @@ namespace Duplicati.Library.Backend } public Task TestAsync(CancellationToken cancelToken) - { - this.TestList(); - return Task.CompletedTask; - } + => this.TestListAsync(cancelToken); public Task CreateFolderAsync(CancellationToken cancelToken) { diff --git a/Duplicati/Library/Backend/Mega/MegaBackend.cs b/Duplicati/Library/Backend/Mega/MegaBackend.cs index 0357976ed..f60b2af07 100644 --- a/Duplicati/Library/Backend/Mega/MegaBackend.cs +++ b/Duplicati/Library/Backend/Mega/MegaBackend.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using CG.Web.MegaApiClient; @@ -28,7 +28,7 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; using OtpNet; -using Duplicati.Library.Utility; +using System.Runtime.CompilerServices; namespace Duplicati.Library.Backend.Mega { @@ -190,15 +190,17 @@ namespace Duplicati.Library.Backend.Mega #region IBackend implementation - public IEnumerable List() + /// + public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { if (m_filecache == null) - ResetFileCacheAsync(null, CancellationToken.None).Await(); + await ResetFileCacheAsync(null, CancellationToken.None).ConfigureAwait(false); - return - from n in m_filecache.Values - let item = n.OrderByDescending(x => x.ModificationDate).First() - select new FileEntry(item.Name, item.Size, item.ModificationDate ?? new DateTime(0), item.ModificationDate ?? new DateTime(0)); + foreach (var n in m_filecache.Values) + { + var item = n.OrderByDescending(x => x.ModificationDate).First(); + yield return new FileEntry(item.Name, item.Size, item.ModificationDate ?? new DateTime(0), item.ModificationDate ?? new DateTime(0)); + } } public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) @@ -236,10 +238,7 @@ namespace Duplicati.Library.Backend.Mega } public Task TestAsync(CancellationToken cancelToken) - { - this.TestList(); - return Task.CompletedTask; - } + => this.TestListAsync(cancelToken); public Task CreateFolderAsync(CancellationToken cancelToken) { diff --git a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs index f33c8931e..96476b793 100644 --- a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs +++ b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Backend.MicrosoftGraph; using Duplicati.Library.Common.IO; @@ -31,6 +31,7 @@ using System.Linq; using System.Net; using System.Net.Http; using System.Net.Http.Headers; +using System.Runtime.CompilerServices; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -318,9 +319,11 @@ namespace Duplicati.Library.Backend } } - public IEnumerable List() + /// + public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) + { - foreach (DriveItem item in this.Enumerate(string.Format("{0}/root:{1}:/children", this.DrivePrefix, this.RootPath))) + await foreach (var item in this.Enumerate(string.Format("{0}/root:{1}:/children", this.DrivePrefix, this.RootPath), cancelToken)) { // Exclude non-files and deleted items (not sure if they show up in this listing, but make sure anyway) if (item.IsFile && !item.IsDeleted) @@ -667,7 +670,7 @@ namespace Duplicati.Library.Backend } } - private IEnumerable Enumerate(string url) + private async IAsyncEnumerable Enumerate(string url, [EnumeratorCancellation] CancellationToken cancelToken) { string nextUrl = url; while (!string.IsNullOrEmpty(nextUrl)) @@ -675,7 +678,7 @@ namespace Duplicati.Library.Backend GraphCollection results; try { - results = this.GetAsync>(nextUrl, CancellationToken.None).Await(); + results = await this.GetAsync>(nextUrl, cancelToken).ConfigureAwait(false); } catch (DriveItemNotFoundException ex) { diff --git a/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs b/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs index 2b626e0f6..579388eca 100644 --- a/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs +++ b/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -32,6 +32,7 @@ using System.Text; using Duplicati.Library.Common.IO; using System.Threading; using System.Threading.Tasks; +using System.Runtime.CompilerServices; namespace Duplicati.Library.Backend.OpenStack { @@ -528,11 +529,11 @@ namespace Duplicati.Library.Backend.OpenStack #endregion #region IBackend implementation - private T HandleListExceptions(Func func) + private async Task HandleListExceptions(Func> func) { try { - return func(); + return await func().ConfigureAwait(false); } catch (WebException wex) { @@ -543,7 +544,8 @@ namespace Duplicati.Library.Backend.OpenStack } } - public IEnumerable List() + /// + public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { var plainurl = JoinUrls(SimpleStorageEndPoint, m_container) + string.Format("?format=json&delimiter=/&limit={0}", PAGE_LIMIT); if (!string.IsNullOrEmpty(m_prefix)) @@ -556,7 +558,7 @@ namespace Duplicati.Library.Backend.OpenStack var req = m_helper.CreateRequest(url); req.Accept = "application/json"; - var items = HandleListExceptions(() => m_helper.ReadJSONResponse(req)); + var items = await HandleListExceptions(() => m_helper.ReadJSONResponseAsync(req, cancelToken)).ConfigureAwait(false); foreach (var n in items) { var name = n.name; @@ -596,10 +598,8 @@ namespace Duplicati.Library.Backend.OpenStack return m_helper.ReadJSONResponseAsync(url, cancelToken, null, "DELETE"); } public Task TestAsync(CancellationToken cancelToken) - { - this.TestList(); - return Task.CompletedTask; - } + => this.TestListAsync(cancelToken); + public async Task CreateFolderAsync(CancellationToken cancelToken) { var url = JoinUrls(SimpleStorageEndPoint, m_container); diff --git a/Duplicati/Library/Backend/Rclone/Rclone.cs b/Duplicati/Library/Backend/Rclone/Rclone.cs index 8e790d17e..7bdd2f8c2 100644 --- a/Duplicati/Library/Backend/Rclone/Rclone.cs +++ b/Duplicati/Library/Backend/Rclone/Rclone.cs @@ -1,33 +1,33 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; -using Duplicati.Library.Utility; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.Runtime.CompilerServices; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -192,14 +192,15 @@ namespace Duplicati.Library.Backend } - public IEnumerable List() + /// + public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { - String str_result; + string str_result; try { - str_result = RcloneCommandExecuter(rclone_executable, $"lsjson {remote_repo}:{remote_path}", CancellationToken.None).Await(); + str_result = await RcloneCommandExecuter(rclone_executable, $"lsjson {remote_repo}:{remote_path}", cancelToken).ConfigureAwait(false); // this will give an error if the executable does not exist. } @@ -297,10 +298,7 @@ namespace Duplicati.Library.Backend public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new[] { remote_repo }); public Task TestAsync(CancellationToken cancelToken) - { - this.TestList(); - return Task.CompletedTask; - } + => this.TestListAsync(cancelToken); public Task CreateFolderAsync(CancellationToken cancelToken) { diff --git a/Duplicati/Library/Backend/S3/IS3Client.cs b/Duplicati/Library/Backend/S3/IS3Client.cs index fee68361b..6d32833ee 100644 --- a/Duplicati/Library/Backend/S3/IS3Client.cs +++ b/Duplicati/Library/Backend/S3/IS3Client.cs @@ -28,7 +28,7 @@ namespace Duplicati.Library.Backend { public interface IS3Client : IDisposable { - IEnumerable ListBucket(string bucketName, string prefix); + IAsyncEnumerable ListBucketAsync(string bucketName, string prefix, CancellationToken cancellationToken); Task AddBucketAsync(string bucketName, CancellationToken cancelToken); diff --git a/Duplicati/Library/Backend/S3/S3AwsClient.cs b/Duplicati/Library/Backend/S3/S3AwsClient.cs index e3a480159..94e578182 100644 --- a/Duplicati/Library/Backend/S3/S3AwsClient.cs +++ b/Duplicati/Library/Backend/S3/S3AwsClient.cs @@ -1,33 +1,33 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Amazon.S3; using Amazon.S3.Model; -using Duplicati.Library.Backend.Strings; using Duplicati.Library.Interface; using Duplicati.Library.Utility; using System; using System.Collections.Generic; using System.Linq; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; @@ -192,7 +192,7 @@ namespace Duplicati.Library.Backend return m_client.DeleteObjectAsync(objectDeleteRequest, cancellationToken); } - public virtual IEnumerable ListBucket(string bucketName, string prefix) + public async IAsyncEnumerable ListBucketAsync(string bucketName, string prefix, [EnumeratorCancellation] CancellationToken cancellationToken) { bool isTruncated = true; string filename = null; @@ -217,7 +217,7 @@ namespace Duplicati.Library.Backend ListObjectsResponse listResponse; try { - listResponse = m_client.ListObjectsAsync(listRequest).Await(); + listResponse = await m_client.ListObjectsAsync(listRequest, cancellationToken).ConfigureAwait(false); } catch (AmazonS3Exception e) { diff --git a/Duplicati/Library/Backend/S3/S3Backend.cs b/Duplicati/Library/Backend/S3/S3Backend.cs index 8df4c8355..f15cf5dfe 100644 --- a/Duplicati/Library/Backend/S3/S3Backend.cs +++ b/Duplicati/Library/Backend/S3/S3Backend.cs @@ -1,31 +1,31 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; -using Duplicati.Library.Utility; using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Runtime.CompilerServices; using System.Text; using System.Text.RegularExpressions; using System.Threading; @@ -270,9 +270,10 @@ namespace Duplicati.Library.Backend } - public IEnumerable List() + /// + public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { - foreach (IFileEntry file in Connection.ListBucket(m_bucket, m_prefix)) + await foreach (IFileEntry file in Connection.ListBucketAsync(m_bucket, m_prefix, cancelToken)) { ((FileEntry)file).Name = file.Name.Substring(m_prefix.Length); @@ -352,10 +353,7 @@ namespace Duplicati.Library.Backend } public Task TestAsync(CancellationToken cancelToken) - { - this.TestList(); - return Task.CompletedTask; - } + => this.TestListAsync(cancelToken); public Task CreateFolderAsync(CancellationToken cancelToken) { diff --git a/Duplicati/Library/Backend/S3/S3MinioClient.cs b/Duplicati/Library/Backend/S3/S3MinioClient.cs index 09d2e5d52..cee966473 100644 --- a/Duplicati/Library/Backend/S3/S3MinioClient.cs +++ b/Duplicati/Library/Backend/S3/S3MinioClient.cs @@ -22,7 +22,9 @@ using System; using System.Collections.Generic; using System.IO; using System.Reactive.Linq; +using System.Runtime.CompilerServices; using System.Threading; +using System.Threading.Channels; using System.Threading.Tasks; using Duplicati.Library.Interface; using Duplicati.Library.Utility; @@ -58,13 +60,33 @@ namespace Duplicati.Library.Backend m_dnsHost = servername; } - public IEnumerable ListBucket(string bucketName, string prefix) + private static async IAsyncEnumerable ToAsyncEnumerable( + IObservable observable, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var channel = Channel.CreateUnbounded(); // Buffered channel for async iteration + + using var subscription = observable.Subscribe( + item => channel.Writer.TryWrite(item), + ex => channel.Writer.TryComplete(ex), + () => channel.Writer.TryComplete() + ); + + while (await channel.Reader.WaitToReadAsync(cancellationToken)) + { + while (channel.Reader.TryRead(out var item)) + { + yield return item; + } + } + } + + public async IAsyncEnumerable ListBucketAsync(string bucketName, string prefix, [EnumeratorCancellation] CancellationToken cancellationToken) { ThrowExceptionIfBucketDoesNotExist(bucketName); - var observable = m_client.ListObjectsAsync(bucketName, prefix, true); - - foreach (var obj in observable.ToEnumerable()) + var observable = m_client.ListObjectsAsync(bucketName, prefix, true, cancellationToken); + await foreach (var obj in ToAsyncEnumerable(observable, cancellationToken).ConfigureAwait(false)) { yield return new Common.IO.FileEntry( obj.Key, diff --git a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs index 073217e2b..000211b6f 100644 --- a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs +++ b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. @@ -28,6 +28,7 @@ using System; using System.Collections.Generic; using System.Globalization; using System.Linq; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; @@ -112,22 +113,19 @@ namespace Duplicati.Library.Backend #region IBackend Members public Task TestAsync(CancellationToken cancelToken) - { - this.TestList(); - return Task.CompletedTask; - } + => this.TestListAsync(cancelToken); - public Task CreateFolderAsync(CancellationToken cancelToken) + public async Task CreateFolderAsync(CancellationToken cancelToken) { - CreateConnection(); + await CreateConnection(cancelToken).ConfigureAwait(false); // Since the SftpClient.CreateDirectory method does not create all the parent directories // as needed, this has to be done manually. - string partialPath = String.Empty; + var partialPath = string.Empty; foreach (string part in m_path.Split('/').Where(x => !String.IsNullOrEmpty(x))) { partialPath += $"/{part}"; - if (this.m_con.Exists(partialPath)) + if (await this.m_con.ExistsAsync(partialPath, cancelToken)) { if (!this.m_con.GetAttributes(partialPath).IsDirectory) { @@ -136,11 +134,10 @@ namespace Duplicati.Library.Backend } else { - this.m_con.CreateDirectory(partialPath); + await this.m_con.CreateDirectoryAsync(partialPath, cancelToken).ConfigureAwait(false); } } - return Task.CompletedTask; } public string DisplayName => Strings.SSHv2Backend.DisplayName; @@ -165,8 +162,8 @@ namespace Duplicati.Library.Backend { try { - CreateConnection(); - ChangeDirectory(m_path); + await CreateConnection(cancelToken).ConfigureAwait(false); + await ChangeDirectory(m_path).ConfigureAwait(false); await m_con.DeleteFileAsync(remotename, cancelToken).ConfigureAwait(false); } catch (SftpPathNotFoundException ex) @@ -239,11 +236,11 @@ namespace Duplicati.Library.Backend #region IStreamingBackend Implementation - public Task PutAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) + public async Task PutAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) { - CreateConnection(); - ChangeDirectory(m_path); - return Task.Factory.FromAsync( + await CreateConnection(cancelToken).ConfigureAwait(false); + await ChangeDirectory(m_path).ConfigureAwait(false); + await Task.Factory.FromAsync( (cb, state) => m_con.BeginUploadFile(stream, remotename, cb, state, _ => cancelToken.ThrowIfCancellationRequested()), m_con.EndUploadFile, null); @@ -251,8 +248,8 @@ namespace Duplicati.Library.Backend public async Task GetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) { - CreateConnection(); - ChangeDirectory(m_path); + await CreateConnection(cancelToken).ConfigureAwait(false); + await ChangeDirectory(m_path).ConfigureAwait(false); try { @@ -271,25 +268,25 @@ namespace Duplicati.Library.Backend #region IRenameEnabledBackend Implementation - public Task RenameAsync(string source, string target, CancellationToken cancelToken) + public async Task RenameAsync(string source, string target, CancellationToken cancelToken) { - CreateConnection(); - ChangeDirectory(m_path); - return m_con.RenameFileAsync(source, target, cancelToken); + await CreateConnection(cancelToken).ConfigureAwait(false); + await ChangeDirectory(m_path).ConfigureAwait(false); + await m_con.RenameFileAsync(source, target, cancelToken).ConfigureAwait(false); } #endregion #region Implementation - private void CreateConnection() + private async Task CreateConnection(CancellationToken cancelToken) { if (m_con != null && m_con.IsConnected) return; if (m_con != null && !m_con.IsConnected) { - this.TryConnect(m_con); + await this.TryConnect(m_con, cancelToken).ConfigureAwait(false); return; } @@ -344,17 +341,15 @@ namespace Duplicati.Library.Backend if (m_keepaliveinterval.Ticks != 0) con.KeepAliveInterval = m_keepaliveinterval; - this.TryConnect(con); + await this.TryConnect(con, cancelToken).ConfigureAwait(false); m_con = con; } - private void TryConnect(SftpClient client) - { - client.Connect(); - } + private Task TryConnect(SftpClient client, CancellationToken cancelToken) + => client.ConnectAsync(cancelToken); - private void ChangeDirectory(string path) + private async Task ChangeDirectory(string path) { if (string.IsNullOrEmpty(path)) return; @@ -365,7 +360,7 @@ namespace Duplicati.Library.Backend try { - m_con.ChangeDirectory(path); + await m_con.ChangeDirectoryAsync(path); } catch (Exception ex) { @@ -374,14 +369,15 @@ namespace Duplicati.Library.Backend } } - public IEnumerable List() + /// + public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { - string path = "."; + var path = "."; - CreateConnection(); - ChangeDirectory(m_path); + await CreateConnection(cancelToken).ConfigureAwait(false); + await ChangeDirectory(m_path).ConfigureAwait(false); - foreach (var ls in m_con.ListDirectory(path)) + await foreach (var ls in m_con.ListDirectoryAsync(path, cancelToken)) { if (ls.Name.ToString() == "." || ls.Name.ToString() == "..") continue; yield return new FileEntry(ls.Name, ls.Length, diff --git a/Duplicati/Library/Backend/SharePoint/SharePointBackend.cs b/Duplicati/Library/Backend/SharePoint/SharePointBackend.cs index 188826783..07a00330c 100644 --- a/Duplicati/Library/Backend/SharePoint/SharePointBackend.cs +++ b/Duplicati/Library/Backend/SharePoint/SharePointBackend.cs @@ -28,6 +28,7 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Runtime.CompilerServices; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -473,8 +474,11 @@ namespace Duplicati.Library.Backend await testContextForWebAsync(ctx, true, cancelToken).ConfigureAwait(false); } - public IEnumerable List() { return doListAsync(false, CancellationToken.None).Await(); } - private async Task> doListAsync(bool useNewContext, CancellationToken cancelToken) + /// + public IAsyncEnumerable ListAsync(CancellationToken cancelToken) + => doListAsync(false, cancelToken); + + private async IAsyncEnumerable doListAsync(bool useNewContext, [EnumeratorCancellation] CancellationToken cancelToken) { var ctx = await getSpClientContextAsync(useNewContext, cancelToken).ConfigureAwait(false); SP.Folder remoteFolder = null; @@ -502,15 +506,19 @@ namespace Duplicati.Library.Backend if (retry) { // An exception was caught, and List() should be retried. - return await doListAsync(true, cancelToken).ConfigureAwait(false); + await foreach (var f in doListAsync(true, cancelToken)) + yield return f; } else { - return remoteFolder.Folders.Where(ff => ff.Exists) + + var list = remoteFolder.Folders.Where(ff => ff.Exists) .Select(f => new FileEntry(f.Name, -1, f.TimeLastModified, f.TimeLastModified) { IsFolder = true }) .Concat(remoteFolder.Files.Where(ff => ff.Exists) - .Select(f => new FileEntry(f.Name, f.Length, f.TimeLastModified, f.TimeLastModified) { IsFolder = false })) - .ToArray(); + .Select(f => new FileEntry(f.Name, f.Length, f.TimeLastModified, f.TimeLastModified) { IsFolder = false })); + + foreach (var f in list) + yield return f; } } diff --git a/Duplicati/Library/Backend/Sia/Sia.cs b/Duplicati/Library/Backend/Sia/Sia.cs index 3c13bc6ce..e99b91c1a 100644 --- a/Duplicati/Library/Backend/Sia/Sia.cs +++ b/Duplicati/Library/Backend/Sia/Sia.cs @@ -26,6 +26,7 @@ using System; using System.Collections.Generic; using System.Globalization; using System.Net; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; @@ -161,19 +162,22 @@ namespace Duplicati.Library.Backend.Sia public SiaDownloadFile[] Files { get; set; } } - private SiaFileList GetFiles() + private async Task GetFiles(CancellationToken cancelToken) { + // Remove warning until this is rewritten to use HttpClient + await Task.CompletedTask; + var fl = new SiaFileList(); - string endpoint = "/renter/files"; + var endpoint = "/renter/files"; try { - System.Net.HttpWebRequest req = CreateRequest(endpoint); + var req = CreateRequest(endpoint); req.Method = System.Net.WebRequestMethods.Http.Get; - Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req); + var areq = new Utility.AsyncHttpRequest(req); - using (System.Net.HttpWebResponse resp = (System.Net.HttpWebResponse)areq.GetResponse()) + using (var resp = (System.Net.HttpWebResponse)areq.GetResponse()) { int code = (int)resp.StatusCode; if (code < 200 || code >= 300) @@ -196,9 +200,9 @@ namespace Duplicati.Library.Backend.Sia return fl; } - private bool IsUploadComplete(string siafilename) + private async Task IsUploadComplete(string siafilename, CancellationToken cancelToken) { - SiaFileList fl = GetFiles(); + var fl = await GetFiles(cancelToken).ConfigureAwait(false); if (fl.Files == null) return false; @@ -286,10 +290,7 @@ namespace Duplicati.Library.Backend.Sia #region IBackend Members public Task TestAsync(CancellationToken cancelToken) - { - this.TestList(); - return Task.CompletedTask; - } + => this.TestListAsync(cancelToken); public Task CreateFolderAsync(CancellationToken cancelToken) { @@ -307,12 +308,13 @@ namespace Duplicati.Library.Backend.Sia get { return "sia"; } } - public IEnumerable List() + /// + public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { SiaFileList fl; try { - fl = GetFiles(); + fl = await GetFiles(cancelToken).ConfigureAwait(false); } catch (System.Net.WebException wex) { @@ -338,10 +340,10 @@ namespace Duplicati.Library.Backend.Sia } } - public Task PutAsync(string remotename, string filename, CancellationToken cancelToken) + public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) { - string endpoint = ""; - string siafile = m_targetpath + "/" + remotename; + var endpoint = ""; + var siafile = m_targetpath + "/" + remotename; try { @@ -351,29 +353,25 @@ namespace Duplicati.Library.Backend.Sia Utility.Uri.UrlEncode(filename).Replace("+", "%20") ); - HttpWebRequest req = CreateRequest(endpoint); + var req = CreateRequest(endpoint); req.Method = WebRequestMethods.Http.Post; - AsyncHttpRequest areq = new AsyncHttpRequest(req); + var areq = new AsyncHttpRequest(req); - using (HttpWebResponse resp = (HttpWebResponse)areq.GetResponse()) + using (var resp = (HttpWebResponse)areq.GetResponse()) { int code = (int)resp.StatusCode; if (code < 200 || code >= 300) throw new WebException(resp.StatusDescription, null, WebExceptionStatus.ProtocolError, resp); - while (!IsUploadComplete(siafile)) - { - Thread.Sleep(5000); - } + while (!await IsUploadComplete(siafile, cancelToken).ConfigureAwait(false)) + await Task.Delay(5000, cancelToken).ConfigureAwait(false); } } catch (WebException wex) { throw new Exception(getResponseBodyOnError(endpoint, wex)); } - - return Task.FromResult(true); } public async Task GetAsync(string remotename, string localname, CancellationToken cancelToken) diff --git a/Duplicati/Library/Backend/Storj/StorjBackend.cs b/Duplicati/Library/Backend/Storj/StorjBackend.cs index a626d3869..5de882aea 100644 --- a/Duplicati/Library/Backend/Storj/StorjBackend.cs +++ b/Duplicati/Library/Backend/Storj/StorjBackend.cs @@ -19,10 +19,10 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Interface; -using Duplicati.Library.Utility; using System; using System.Collections.Generic; using System.IO; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using uplink.NET.Interfaces; @@ -238,29 +238,22 @@ namespace Duplicati.Library.Backend.Storj await download.StartDownloadAsync().ConfigureAwait(false); } - public IEnumerable List() + /// + public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { - return ListAsync().Await(); - } - - private async Task> ListAsync() - { - List files = new List(); var bucket = await _bucketService.EnsureBucketAsync(_bucket).ConfigureAwait(false); var prefix = GetBasePath(); var objects = await _objectService.ListObjectsAsync(bucket, new ListObjectsOptions { Recursive = true, System = true, Custom = true, Prefix = prefix }).ConfigureAwait(false); foreach (var obj in objects.Items) { - StorjFile file = new StorjFile(obj); + var file = new StorjFile(obj); if (prefix != "") { file.Name = file.Name.Replace(prefix, ""); } - files.Add(file); + yield return file; } - - return files; } public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) diff --git a/Duplicati/Library/Backend/TahoeLAFS/TahoeBackend.cs b/Duplicati/Library/Backend/TahoeLAFS/TahoeBackend.cs index fc31f1d3b..22d693924 100644 --- a/Duplicati/Library/Backend/TahoeLAFS/TahoeBackend.cs +++ b/Duplicati/Library/Backend/TahoeLAFS/TahoeBackend.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Common.IO; @@ -26,6 +26,7 @@ using Newtonsoft.Json.Linq; using System; using System.Collections.Generic; using System.Net; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; @@ -124,10 +125,7 @@ namespace Duplicati.Library.Backend #region IBackend Members public Task TestAsync(CancellationToken cancelToken) - { - this.TestList(); - return Task.CompletedTask; - } + => this.TestListAsync(cancelToken); public Task CreateFolderAsync(CancellationToken cancelToken) { @@ -150,8 +148,12 @@ namespace Duplicati.Library.Backend get { return "tahoe"; } } - public IEnumerable List() + /// + public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { + // Remove warning until this is rewritten to use HttpClient + await Task.CompletedTask; + TahoeEl data; try @@ -160,9 +162,9 @@ namespace Duplicati.Library.Backend req.Method = System.Net.WebRequestMethods.Http.Get; var areq = new Utility.AsyncHttpRequest(req); - using (System.Net.HttpWebResponse resp = (System.Net.HttpWebResponse)areq.GetResponse()) + using (var resp = (System.Net.HttpWebResponse)areq.GetResponse()) { - int code = (int)resp.StatusCode; + var code = (int)resp.StatusCode; if (code < 200 || code >= 300) //For some reason Mono does not throw this automatically throw new System.Net.WebException(resp.StatusDescription, null, System.Net.WebExceptionStatus.ProtocolError, resp); diff --git a/Duplicati/Library/Backend/TencentCOS/COSBackend.cs b/Duplicati/Library/Backend/TencentCOS/COSBackend.cs index af2688017..66f2071e7 100644 --- a/Duplicati/Library/Backend/TencentCOS/COSBackend.cs +++ b/Duplicati/Library/Backend/TencentCOS/COSBackend.cs @@ -26,11 +26,11 @@ using COSXML.Model.Tag; using COSXML.Utils; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; -using Duplicati.Library.Utility; using Newtonsoft.Json; using System; using System.Collections.Generic; using System.IO; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; @@ -178,7 +178,8 @@ namespace Duplicati.Library.Backend.TencentCOS return new CosXmlServer(config, cosCredentialProvider); } - public IEnumerable List() + /// + public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { bool isTruncated = true; string filename = null; @@ -186,26 +187,24 @@ namespace Duplicati.Library.Backend.TencentCOS while (isTruncated) { cosXml = GetCosXml(); - string bucket = _cosOptions.Bucket; - string prefix = _cosOptions.Path; - - GetBucketRequest request = new GetBucketRequest(bucket); + var bucket = _cosOptions.Bucket; + var prefix = _cosOptions.Path; + var request = new GetBucketRequest(bucket); request.SetSign(TimeUtils.GetCurrentTime(TimeUnit.SECONDS), 600); if (!string.IsNullOrEmpty(filename)) - { request.SetMarker(filename); - } if (!string.IsNullOrEmpty(prefix)) - { request.SetPrefix(prefix); - } - GetBucketResult result = cosXml.GetBucket(request); + var tcs = new TaskCompletionSource(); + cancelToken.Register(() => tcs.TrySetCanceled()); + cosXml.GetBucket(request, (result) => tcs.SetResult(result as GetBucketResult), (clientEx, serverEx) => tcs.SetException((Exception)clientEx ?? serverEx)); + var result = await tcs.Task.ConfigureAwait(false); - ListBucket info = result.listBucket; + var info = result.listBucket; isTruncated = result.listBucket.isTruncated; filename = result.listBucket.nextMarker; diff --git a/Duplicati/Library/Backend/WEBDAV/WEBDAV.cs b/Duplicati/Library/Backend/WEBDAV/WEBDAV.cs index b9c70a055..70602a49f 100644 --- a/Duplicati/Library/Backend/WEBDAV/WEBDAV.cs +++ b/Duplicati/Library/Backend/WEBDAV/WEBDAV.cs @@ -27,6 +27,7 @@ using System.Collections.Generic; using System.IO; using System.Net; using System.Net.Http; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; @@ -75,7 +76,7 @@ namespace Duplicati.Library.Backend /// It is used to detect a problem with IIS where a file is listed, /// but IIS responds 404 because the file mapping is incorrect. /// - private List m_filenamelist = null; + private HashSet m_filenamelist = null; // According to the WEBDAV standard, the "allprop" request should return all properties, however this seems to fail on some servers (box.net). // I've found this description: http://www.webdav.org/specs/rfc2518.html#METHOD_PROPFIND @@ -173,11 +174,24 @@ namespace Duplicati.Library.Backend public string ProtocolKey => "webdav"; /// - public IEnumerable List() + public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { + using var timeoutToken = new CancellationTokenSource(); + timeoutToken.CancelAfter(TimeSpan.FromSeconds(LIST_OPERATION_TIMEOUT_SECONDS)); + + using var requestResources = CreateRequest(string.Empty, new HttpMethod("PROPFIND")); + requestResources.RequestMessage.Headers.Add("Depth", "1"); + requestResources.RequestMessage.Content = new StreamContent(new MemoryStream(PROPFIND_BODY)); + requestResources.RequestMessage.Content.Headers.ContentLength = PROPFIND_BODY.Length; + + var doc = new System.Xml.XmlDocument(); + try { - return ListWithoutExceptionCatch(); + using var response = await requestResources.HttpClient.SendAsync(requestResources.RequestMessage, HttpCompletionOption.ResponseContentRead, timeoutToken.Token).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); // This replaces the if needed when Mono was used. + using var stream = await response.Content.ReadAsStreamAsync(cancelToken).ConfigureAwait(false); + doc.Load(stream); } catch (HttpRequestException wex) { @@ -189,32 +203,11 @@ namespace Duplicati.Library.Backend throw; } - } - private IEnumerable ListWithoutExceptionCatch() - { - using var timeoutToken = new CancellationTokenSource(); - timeoutToken.CancelAfter(TimeSpan.FromSeconds(LIST_OPERATION_TIMEOUT_SECONDS)); - - using var requestResources = CreateRequest(string.Empty, new HttpMethod("PROPFIND")); - requestResources.RequestMessage.Headers.Add("Depth", "1"); - requestResources.RequestMessage.Content = new StreamContent(new MemoryStream(PROPFIND_BODY)); - requestResources.RequestMessage.Content.Headers.ContentLength = PROPFIND_BODY.Length; - - using var response = requestResources.HttpClient.SendAsync(requestResources.RequestMessage, HttpCompletionOption.ResponseContentRead, timeoutToken.Token).ConfigureAwait(false).GetAwaiter().GetResult(); - - response.EnsureSuccessStatusCode(); // This replaces the if needed when Mono was used. - - var doc = new System.Xml.XmlDocument(); - - doc.Load(response.Content.ReadAsStream()); - - System.Xml.XmlNamespaceManager nm = new System.Xml.XmlNamespaceManager(doc.NameTable); + var nm = new System.Xml.XmlNamespaceManager(doc.NameTable); nm.AddNamespace("D", "DAV:"); - List files = new List(); - m_filenamelist = new List(); - + var filenamelist = new HashSet(); foreach (System.Xml.XmlNode n in doc.SelectNodes("D:multistatus/D:response/D:href", nm)) { //IIS uses %20 for spaces and %2B for + @@ -274,13 +267,16 @@ namespace Duplicati.Library.Backend isCollection = stat.SelectSingleNode("D:resourcetype/D:collection", nm) != null; } - FileEntry fe = new FileEntry(name, size, lastAccess, lastModified); - fe.IsFolder = isCollection; - files.Add(fe); - m_filenamelist.Add(name); + var fe = new FileEntry(name, size, lastAccess, lastModified) + { + IsFolder = isCollection + }; + + filenamelist.Add(name); + yield return fe; } - return files; + m_filenamelist = filenamelist; } /// @@ -347,10 +343,7 @@ namespace Duplicati.Library.Backend /// public Task TestAsync(CancellationToken cancelToken) - { - List(); - return Task.CompletedTask; - } + => this.TestListAsync(cancelToken); /// public async Task CreateFolderAsync(CancellationToken cancelToken) @@ -409,7 +402,7 @@ namespace Duplicati.Library.Backend var request = new HttpRequestMessage(HttpMethod.Get, $"{m_url}{Utility.Uri.UrlEncode(remotename).Replace("+", "%20")}"); request.Headers.Add(HttpRequestHeader.UserAgent.ToString(), "Duplicati WEBDAV Client v" + System.Reflection.Assembly.GetExecutingAssembly().GetName().Version); - + request.Headers.ConnectionClose = !m_useIntegratedAuthentication; // ConnectionClose is incompatible with integrated authentication if (method != null) diff --git a/Duplicati/Library/Backend/pCloud/pCloudBackend.cs b/Duplicati/Library/Backend/pCloud/pCloudBackend.cs index f60ace0c9..a7a0ee75f 100644 --- a/Duplicati/Library/Backend/pCloud/pCloudBackend.cs +++ b/Duplicati/Library/Backend/pCloud/pCloudBackend.cs @@ -33,6 +33,7 @@ using System.Threading.Tasks; using Duplicati.Library.Backend.pCloud; using Duplicati.Library.Utility; using Uri = System.Uri; +using System.Runtime.CompilerServices; namespace Duplicati.Library.Backend; @@ -222,19 +223,13 @@ public class pCloudBackend : IStreamingBackend /// Implementation of interface method for listing remote folder contents /// /// List of IFileEntry with directory listing result - public async Task> ListAsync(CancellationToken cancellationToken) + public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancellationToken) { _CachedFolderID ??= await GetFolderId(cancellationToken).ConfigureAwait(false); - - return await List(_CachedFolderID.Value, cancellationToken).ConfigureAwait(false); + foreach (var v in await List(_CachedFolderID.Value, cancellationToken).ConfigureAwait(false)) + yield return v; } - /// - /// Wrapper method of legacy non async call to list files in the remote folder - /// - /// - public IEnumerable List() => ListAsync(CancellationToken.None).Await(); - /// /// Upload files to remote location /// diff --git a/Duplicati/Library/Interface/BackendExtensions.cs b/Duplicati/Library/Interface/BackendExtensions.cs index 8844bd127..14dfeb6e2 100644 --- a/Duplicati/Library/Interface/BackendExtensions.cs +++ b/Duplicati/Library/Interface/BackendExtensions.cs @@ -18,10 +18,8 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; + +using System.Threading; using System.Threading.Tasks; namespace Duplicati.Library.Interface @@ -36,12 +34,15 @@ namespace Duplicati.Library.Interface /// As long as the iteration can either complete or find at least one file without throwing, the test is successful /// /// Backend to test - public static void TestList(this IBackend backend) + /// Cancellation token + public static async Task TestListAsync(this IBackend backend, CancellationToken cancelToken) { + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancelToken); // If we can iterate successfully, even if it's empty, then the backend test is successful - foreach (IFileEntry file in backend.List()) + await foreach (IFileEntry file in backend.ListAsync(cts.Token)) { - break; + cts.Cancel(); + return; } } } diff --git a/Duplicati/Library/Interface/IBackend.cs b/Duplicati/Library/Interface/IBackend.cs index 26f41c6df..eee508a3a 100644 --- a/Duplicati/Library/Interface/IBackend.cs +++ b/Duplicati/Library/Interface/IBackend.cs @@ -51,8 +51,9 @@ namespace Duplicati.Library.Interface /// /// Enumerates a list of files found on the remote location /// + /// Token to cancel the operation. /// The list of files - IEnumerable List(); + IAsyncEnumerable ListAsync(CancellationToken cancellationToken); /// /// Puts the content of the file to the url passed diff --git a/Duplicati/Library/Main/Backend/BackendManager.DeleteOperation.cs b/Duplicati/Library/Main/Backend/BackendManager.DeleteOperation.cs index a846328b3..8b313aafa 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.DeleteOperation.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.DeleteOperation.cs @@ -72,7 +72,7 @@ partial class BackendManager try { - recovered = !backend.List().Select(x => x.Name).Contains(RemoteFilename); + recovered = !await backend.ListAsync(cancelToken).Select(x => x.Name).ContainsAsync(RemoteFilename).ConfigureAwait(false); } catch { diff --git a/Duplicati/Library/Main/Backend/BackendManager.ListOperation.cs b/Duplicati/Library/Main/Backend/BackendManager.ListOperation.cs index c3389ff31..de9647e86 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.ListOperation.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.ListOperation.cs @@ -33,11 +33,12 @@ partial class BackendManager /// The backend to list /// The cancellation token /// An awaitable task - public override Task> ExecuteAsync(IBackend backend, CancellationToken cancelToken) + public override async Task> ExecuteAsync(IBackend backend, CancellationToken cancelToken) { Context.Statwriter.SendEvent(BackendActionType.List, BackendEventType.Started, null, -1); - var r = backend.List().ToList(); + // TODO: Consider returning IAsyncEnumerable instead of List + var r = await backend.ListAsync(cancelToken).ToListAsync(cancelToken).ConfigureAwait(false); // TODO: Investigate better way to solve this so we do not use memory for large lists var sb = new StringBuilder(); @@ -57,7 +58,7 @@ partial class BackendManager Context.Statwriter.SendEvent(BackendActionType.List, BackendEventType.Completed, null, r.Count); - return Task.FromResult(r); + return r; } } } \ No newline at end of file diff --git a/Duplicati/Library/Main/Backend/BackendManager.PutOperation.cs b/Duplicati/Library/Main/Backend/BackendManager.PutOperation.cs index cea59320d..a63d5f07a 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.PutOperation.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.PutOperation.cs @@ -270,7 +270,7 @@ partial class BackendManager { using var fs = System.IO.File.OpenRead(LocalFilename); using var ts = new ThrottledStream(fs, Context.Options.MaxUploadPrSecond, 0); - using var pgs = new ProgressReportingStream(ts,pg => Context.HandleProgress(ts, pg, RemoteFilename)); + using var pgs = new ProgressReportingStream(ts, pg => Context.HandleProgress(ts, pg, RemoteFilename)); await streamingBackend.PutAsync(RemoteFilename, pgs, cancelToken).ConfigureAwait(false); } else @@ -286,7 +286,7 @@ partial class BackendManager if (Context.Options.ListVerifyUploads) { - var f = backend.List().FirstOrDefault(n => n.Name.Equals(RemoteFilename, StringComparison.OrdinalIgnoreCase)); + var f = await backend.ListAsync(cancelToken).FirstOrDefaultAsync(n => n.Name.Equals(RemoteFilename, StringComparison.OrdinalIgnoreCase)).ConfigureAwait(false); if (f == null) throw new Exception(string.Format($"List verify failed, file was not found after upload: {RemoteFilename}")); else if (f.Size != Size && f.Size >= 0) diff --git a/Duplicati/UnitTest/DeterministicErrorBackend.cs b/Duplicati/UnitTest/DeterministicErrorBackend.cs index 34a62ad7e..8ba705e9d 100644 --- a/Duplicati/UnitTest/DeterministicErrorBackend.cs +++ b/Duplicati/UnitTest/DeterministicErrorBackend.cs @@ -79,9 +79,9 @@ namespace Duplicati.UnitTest #endregion #region IBackend implementation - public IEnumerable List() + public IAsyncEnumerable ListAsync(CancellationToken cancelToken) { - return m_backend.List(); + return m_backend.ListAsync(cancelToken); } public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) { diff --git a/Duplicati/UnitTest/Issue5845.cs b/Duplicati/UnitTest/Issue5845.cs index 661228aaa..9eef94cd9 100644 --- a/Duplicati/UnitTest/Issue5845.cs +++ b/Duplicati/UnitTest/Issue5845.cs @@ -68,8 +68,8 @@ namespace Duplicati.UnitTest => backend.GetAsync(remotename, filename, cancellationToken); public Task GetDNSNamesAsync(CancellationToken cancelToken) => GetDNSNamesAsync(cancelToken); - public IEnumerable List() - => Sorter(backend.List()); + public IAsyncEnumerable ListAsync(CancellationToken cancellationToken) + => Sorter(backend.ListAsync(cancellationToken).ToBlockingEnumerable()).ToAsyncEnumerable(); public Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken) => backend.PutAsync(remotename, stream, cancelToken); diff --git a/Duplicati/UnitTest/RandomErrorBackend.cs b/Duplicati/UnitTest/RandomErrorBackend.cs index d295ff3f1..e520c937b 100644 --- a/Duplicati/UnitTest/RandomErrorBackend.cs +++ b/Duplicati/UnitTest/RandomErrorBackend.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Interface; @@ -72,9 +72,9 @@ namespace Duplicati.UnitTest #endregion #region IBackend implementation - public IEnumerable List() + public IAsyncEnumerable ListAsync(CancellationToken cancellationToken) { - return m_backend.List(); + return m_backend.ListAsync(cancellationToken); } public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) { diff --git a/Duplicati/UnitTest/SVNCheckoutsTest.cs b/Duplicati/UnitTest/SVNCheckoutsTest.cs index 22a74dccf..807ea93f3 100644 --- a/Duplicati/UnitTest/SVNCheckoutsTest.cs +++ b/Duplicati/UnitTest/SVNCheckoutsTest.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. @@ -27,6 +27,7 @@ using Duplicati.Library.Logging; using Duplicati.Library.Utility; using System.Linq; using Duplicati.Library.Common.IO; +using System.Threading.Tasks; namespace Duplicati.UnitTest { @@ -173,7 +174,7 @@ namespace Duplicati.UnitTest try { using (var bk = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(target, options)) - foreach (var f in bk.List()) + foreach (var f in bk.ListAsync(System.Threading.CancellationToken.None).ToBlockingEnumerable()) if (!f.IsFolder) Utility.Await(bk.DeleteAsync(f.Name, System.Threading.CancellationToken.None)); } @@ -215,7 +216,7 @@ namespace Duplicati.UnitTest Duplicati.Library.Main.Options opts = new Duplicati.Library.Main.Options(options); using (Duplicati.Library.Interface.IBackend bk = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(target, options)) - foreach (Duplicati.Library.Interface.IFileEntry fe in bk.List()) + foreach (Duplicati.Library.Interface.IFileEntry fe in bk.ListAsync(System.Threading.CancellationToken.None).ToBlockingEnumerable()) if (fe.Size > opts.VolumeSize) { string msg = string.Format("The file {0} is {1} bytes larger than allowed", fe.Name, fe.Size - opts.VolumeSize); diff --git a/Duplicati/UnitTest/SizeOmittingBackend.cs b/Duplicati/UnitTest/SizeOmittingBackend.cs index 74efe540b..a1a464a91 100644 --- a/Duplicati/UnitTest/SizeOmittingBackend.cs +++ b/Duplicati/UnitTest/SizeOmittingBackend.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Common.IO; @@ -59,10 +59,10 @@ namespace Duplicati.UnitTest #endregion #region IBackend implementation - public IEnumerable List() + public IAsyncEnumerable ListAsync(CancellationToken cancellationToken) { return - from n in m_backend.List() + from n in m_backend.ListAsync(cancellationToken) where !n.IsFolder select new FileEntry(n.Name); } diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 3ac9c0eef..cda66367d 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -204,7 +204,7 @@ destination will be verified before being overwritten (if they seemingly match). var b2s = b2 as IStreamingBackend ?? throw throw_message("destination"); // Prepare the operations - var (to_copy, to_delete, to_verify) = PrepareFileLists(b1s, b2s, config); + var (to_copy, to_delete, to_verify) = await PrepareFileLists(b1s, b2s, config, CancellationToken.None); // Verify the files if requested. If the files are not verified, they will be deleted and copied again. long verified = 0, failed_verify = 0; @@ -521,15 +521,15 @@ destination will be verified before being overwritten (if they seemingly match). /// The destination backend. /// The parsed configuration for the tool. /// A tuple of Lists each holding the files to copy, delete and verify. - private static (IEnumerable, IEnumerable, IEnumerable) PrepareFileLists(IStreamingBackend b_src, IStreamingBackend b_dst, Config config) + private static async Task<(IEnumerable, IEnumerable, IEnumerable)> PrepareFileLists(IStreamingBackend b_src, IStreamingBackend b_dst, Config config, CancellationToken cancelToken) { IEnumerable files_src, files_dst; using (new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Prepare | List source")) - files_src = b_src.List(); + files_src = await b_src.ListAsync(cancelToken).ToListAsync(cancelToken).ConfigureAwait(false); using (new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Prepare | List destination")) - files_dst = b_dst.List(); + files_dst = await b_dst.ListAsync(cancelToken).ToListAsync(cancelToken).ConfigureAwait(false); // Shortcut for force if (config.Force) From a99aa87dbba84062a0269eb78c31d7ece6a2d198 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 17 Feb 2025 19:51:26 +0100 Subject: [PATCH 177/362] Fixed issue with Windows paths being stored in extended format --- Duplicati/Library/Snapshots/NoSnapshotWindows.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs index 3108484c4..b4a3b16e0 100644 --- a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs +++ b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs @@ -86,7 +86,7 @@ namespace Duplicati.Library.Snapshots /// The source files and folders public override IEnumerable EnumerateFilesystemEntries() { - foreach (var folder in m_folders.Select(SystemIOWindows.AddExtendedDevicePathPrefix)) + foreach (var folder in m_folders.Select(SystemIOWindows.RemoveExtendedDevicePathPrefix)) { if (DirectoryExists(folder) || folder.EndsWith(System.IO.Path.DirectorySeparatorChar)) yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(folder), true, true); From 64dc7d094f755964293832cb45b416120ffdc3ee Mon Sep 17 00:00:00 2001 From: Alex-Heard Date: Mon, 17 Feb 2025 22:11:41 -0500 Subject: [PATCH 178/362] Prevents double clicks from opening two windows by ignoring click events that happen within 1 second of last click --- .../Duplicati.GUI.TrayIcon/AvaloniaRunner.cs | 13 ++++++++- .../Duplicati.GUI.TrayIcon/ClickDebouncer.cs | 27 +++++++++++++++++++ .../Duplicati.GUI.TrayIcon.sln | 24 +++++++++++++++++ 3 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 Duplicati/GUI/Duplicati.GUI.TrayIcon/ClickDebouncer.cs create mode 100644 Duplicati/GUI/Duplicati.GUI.TrayIcon/Duplicati.GUI.TrayIcon.sln diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/AvaloniaRunner.cs b/Duplicati/GUI/Duplicati.GUI.TrayIcon/AvaloniaRunner.cs index ae31a1828..6473f334d 100644 --- a/Duplicati/GUI/Duplicati.GUI.TrayIcon/AvaloniaRunner.cs +++ b/Duplicati/GUI/Duplicati.GUI.TrayIcon/AvaloniaRunner.cs @@ -395,7 +395,18 @@ namespace Duplicati.GUI.TrayIcon menu.Add(item.GetNativeItem()); } trayIcon.Menu = menu; - trayIcon.Clicked += (_, _) => this.menuItems.FirstOrDefault(x => x.IsDefault)?.Callback(); + trayIcon.Clicked -= HandleTrayIconClick; + trayIcon.Clicked += HandleTrayIconClick; + } + } + + private readonly ClickDebouncer _clickDebouncer = new ClickDebouncer(); + + private void HandleTrayIconClick(object? sender, EventArgs e) + { + if (_clickDebouncer.ShouldProcessClick()) + { + this.menuItems?.FirstOrDefault(x => x.IsDefault)?.Callback(); } } diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/ClickDebouncer.cs b/Duplicati/GUI/Duplicati.GUI.TrayIcon/ClickDebouncer.cs new file mode 100644 index 000000000..e3f012c86 --- /dev/null +++ b/Duplicati/GUI/Duplicati.GUI.TrayIcon/ClickDebouncer.cs @@ -0,0 +1,27 @@ +using System; + +namespace Duplicati.GUI.TrayIcon +{ + internal class ClickDebouncer + { + private DateTime _lastClickTime = DateTime.MinValue; + private readonly TimeSpan _debounceInterval; + + public ClickDebouncer(TimeSpan? debounceInterval = null) + { + _debounceInterval = debounceInterval ?? TimeSpan.FromSeconds(1); + } + + public bool ShouldProcessClick() + { + var now = DateTime.UtcNow; + if (now - _lastClickTime < _debounceInterval) + { + return false; + } + + _lastClickTime = now; + return true; + } + } +} \ No newline at end of file diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Duplicati.GUI.TrayIcon.sln b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Duplicati.GUI.TrayIcon.sln new file mode 100644 index 000000000..9e8f17693 --- /dev/null +++ b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Duplicati.GUI.TrayIcon.sln @@ -0,0 +1,24 @@ +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.5.2.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.GUI.TrayIcon", "Duplicati.GUI.TrayIcon.csproj", "{E35D4E94-4FFB-4503-2107-3B0B0BCECB00}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {E35D4E94-4FFB-4503-2107-3B0B0BCECB00}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E35D4E94-4FFB-4503-2107-3B0B0BCECB00}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E35D4E94-4FFB-4503-2107-3B0B0BCECB00}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E35D4E94-4FFB-4503-2107-3B0B0BCECB00}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {5FCDB925-2807-4C6C-A0C5-FBF8B201B580} + EndGlobalSection +EndGlobal From 9f3f15615d44f471ef950bf3184b0b2f12a0d5f2 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 18 Feb 2025 09:10:13 +0100 Subject: [PATCH 179/362] Exposed spawning multiple backends from the BackendManager --- Duplicati/Library/Main/Backend/BackendManager.cs | 12 ++++++++++++ Duplicati/Library/Main/IBackendManager.cs | 3 +++ 2 files changed, 15 insertions(+) diff --git a/Duplicati/Library/Main/Backend/BackendManager.cs b/Duplicati/Library/Main/Backend/BackendManager.cs index 2e8cedf1d..5813cdf79 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.cs @@ -6,6 +6,7 @@ using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using CoCoL; +using Duplicati.Library.DynamicLoader; using Duplicati.Library.Interface; using Duplicati.Library.Main.Database; using Duplicati.Library.Main.Operation.Common; @@ -30,6 +31,9 @@ internal partial class BackendManager : IBackendManager /// private readonly List backendPool = []; + private string m_backendurl; + private Options m_options; + /// /// The channel for issuing and handling requests /// @@ -71,6 +75,9 @@ internal partial class BackendManager : IBackendManager if (string.IsNullOrWhiteSpace(backendUrl)) throw new ArgumentNullException(nameof(backendUrl)); + m_backendurl = backendUrl; + m_options = options; + // To avoid excessive parameter passing, the context is captured here context = new ExecuteContext( new ProgressHandler(options, backendWriter, taskReader).HandleProgress, @@ -167,6 +174,11 @@ internal partial class BackendManager : IBackendManager return file; } + public IBackend GetBackend() + { + return BackendLoader.GetBackend(m_backendurl, m_options.RawOptions); + } + /// /// Gets quota information from the backend /// diff --git a/Duplicati/Library/Main/IBackendManager.cs b/Duplicati/Library/Main/IBackendManager.cs index 270f6c946..df92577a3 100644 --- a/Duplicati/Library/Main/IBackendManager.cs +++ b/Duplicati/Library/Main/IBackendManager.cs @@ -91,6 +91,9 @@ internal interface IBackendManager : IDisposable /// Task GetAsync(string remotename, string hash, long size, CancellationToken cancelToken); + + public IBackend GetBackend(); + /// /// Performs a download of the files specified, with pre-fetch to overlap the download and processing /// From 6600e418a7e97035cbefe2aa01067f4f0d9a8364 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 18 Feb 2025 09:14:17 +0100 Subject: [PATCH 180/362] Fixed a bug during profile logging --- Duplicati/Library/Main/Operation/Restore/BlockManager.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs index de1e8178f..25d673a07 100644 --- a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs @@ -306,7 +306,7 @@ namespace Duplicati.Library.Main.Operation.Restore if (m_options.InternalProfiling) { - Logging.Log.WriteProfilingMessage(LOGTAG, "InternalTimings", null, $"Sleepable dictionary - CheckCounts: {sw_checkcounts.ElapsedMilliseconds}ms, Get wait: {sw_get_wait.ElapsedMilliseconds}ms, Cache evict: {sw_cacheevict.ElapsedMilliseconds}ms"); + Logging.Log.WriteProfilingMessage(LOGTAG, "InternalTimings", $"Sleepable dictionary - CheckCounts: {sw_checkcounts.ElapsedMilliseconds}ms, Get wait: {sw_get_wait.ElapsedMilliseconds}ms, Cache evict: {sw_cacheevict.ElapsedMilliseconds}ms"); } } @@ -372,7 +372,7 @@ namespace Duplicati.Library.Main.Operation.Restore if (options.InternalProfiling) { - Logging.Log.WriteProfilingMessage(LOGTAG, "InternalTimings", null, $"Volume consumer - Read: {sw_read.ElapsedMilliseconds}ms, Set: {sw_set.ElapsedMilliseconds}ms"); + Logging.Log.WriteProfilingMessage(LOGTAG, "InternalTimings", $"Volume consumer - Read: {sw_read.ElapsedMilliseconds}ms, Set: {sw_set.ElapsedMilliseconds}ms"); } // Cancel any remaining readers - although there shouldn't be any. @@ -426,7 +426,7 @@ namespace Duplicati.Library.Main.Operation.Restore if (options.InternalProfiling) { - Logging.Log.WriteProfilingMessage(LOGTAG, "InternalTimings", null, $"Block handler - Req: {sw_req.ElapsedMilliseconds}ms, Resp: {sw_resp.ElapsedMilliseconds}ms, Cache: {sw_cache.ElapsedMilliseconds}ms, Get: {sw_get.ElapsedMilliseconds}ms"); + Logging.Log.WriteProfilingMessage(LOGTAG, "InternalTimings", $"Block handler - Req: {sw_req.ElapsedMilliseconds}ms, Resp: {sw_resp.ElapsedMilliseconds}ms, Cache: {sw_cache.ElapsedMilliseconds}ms, Get: {sw_get.ElapsedMilliseconds}ms"); } cache.Retire(); From 93756cd74b25d6b8104334e0813a4a6af93ba490 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 18 Feb 2025 09:17:08 +0100 Subject: [PATCH 181/362] Whitespace fixes --- .../Main/Operation/Restore/BlockManager.cs | 30 ++++++------- .../Main/Operation/Restore/Channels.cs | 30 ++++++------- .../Main/Operation/Restore/FileLister.cs | 34 +++++++------- .../Operation/Restore/VolumeDecompressor.cs | 44 ++++++++++--------- .../Main/Operation/Restore/VolumeDecryptor.cs | 36 +++++++-------- .../Operation/Restore/VolumeDownloader.cs | 30 ++++++------- .../Main/Operation/Restore/VolumeManager.cs | 30 ++++++------- .../Library/Main/Operation/RestoreHandler.cs | 30 ++++++------- 8 files changed, 133 insertions(+), 131 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs index 25d673a07..6ee1ea0a5 100644 --- a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; diff --git a/Duplicati/Library/Main/Operation/Restore/Channels.cs b/Duplicati/Library/Main/Operation/Restore/Channels.cs index de1f18aab..4d14ee055 100644 --- a/Duplicati/Library/Main/Operation/Restore/Channels.cs +++ b/Duplicati/Library/Main/Operation/Restore/Channels.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System.Threading.Tasks; diff --git a/Duplicati/Library/Main/Operation/Restore/FileLister.cs b/Duplicati/Library/Main/Operation/Restore/FileLister.cs index e12b2f0b2..64f3dec02 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileLister.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileLister.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -55,8 +55,8 @@ namespace Duplicati.Library.Main.Operation.Restore }, async self => { - Stopwatch sw_prework = options.InternalProfiling ? new () : null; - Stopwatch sw_write = options.InternalProfiling ? new () : null; + Stopwatch sw_prework = options.InternalProfiling ? new() : null; + Stopwatch sw_write = options.InternalProfiling ? new() : null; try { sw_prework?.Start(); diff --git a/Duplicati/Library/Main/Operation/Restore/VolumeDecompressor.cs b/Duplicati/Library/Main/Operation/Restore/VolumeDecompressor.cs index 37bc9ad3f..f2e409e70 100644 --- a/Duplicati/Library/Main/Operation/Restore/VolumeDecompressor.cs +++ b/Duplicati/Library/Main/Operation/Restore/VolumeDecompressor.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -52,12 +52,13 @@ namespace Duplicati.Library.Main.Operation.Restore }, async self => { - Stopwatch sw_read = options.InternalProfiling ? new () : null; - Stopwatch sw_write = options.InternalProfiling ? new () : null; - Stopwatch sw_decompress = options.InternalProfiling ? new () : null; - Stopwatch sw_verify = options.InternalProfiling ? new () : null; + Stopwatch sw_read = options.InternalProfiling ? new() : null; + Stopwatch sw_write = options.InternalProfiling ? new() : null; + Stopwatch sw_decompress = options.InternalProfiling ? new() : null; + Stopwatch sw_verify = options.InternalProfiling ? new() : null; - try { + try + { using var block_hasher = HashFactory.CreateHasher(options.BlockHashAlgorithm); while (true) @@ -77,7 +78,8 @@ namespace Duplicati.Library.Main.Operation.Restore sw_verify?.Start(); var hash = Convert.ToBase64String(block_hasher.ComputeHash(data, 0, (int)block_request.BlockSize)); - if (hash != block_request.BlockHash) { + if (hash != block_request.BlockHash) + { Logging.Log.WriteErrorMessage(LOGTAG, "InvalidBlock", null, $"Invalid block detected for block {block_request.BlockID} in volume {block_request.VolumeID}, expected hash: {block_request.BlockHash}, actual hash: {hash}"); } sw_verify?.Stop(); diff --git a/Duplicati/Library/Main/Operation/Restore/VolumeDecryptor.cs b/Duplicati/Library/Main/Operation/Restore/VolumeDecryptor.cs index 749a8e67b..ae154b3d8 100644 --- a/Duplicati/Library/Main/Operation/Restore/VolumeDecryptor.cs +++ b/Duplicati/Library/Main/Operation/Restore/VolumeDecryptor.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -52,9 +52,9 @@ namespace Duplicati.Library.Main.Operation.Restore }, async self => { - Stopwatch sw_read = options.InternalProfiling ? new () : null; - Stopwatch sw_write = options.InternalProfiling ? new () : null; - Stopwatch sw_decrypt = options.InternalProfiling ? new () : null; + Stopwatch sw_read = options.InternalProfiling ? new() : null; + Stopwatch sw_write = options.InternalProfiling ? new() : null; + Stopwatch sw_decrypt = options.InternalProfiling ? new() : null; try { while (true) diff --git a/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs b/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs index 7aafb3e38..40a28d772 100644 --- a/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs +++ b/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; diff --git a/Duplicati/Library/Main/Operation/Restore/VolumeManager.cs b/Duplicati/Library/Main/Operation/Restore/VolumeManager.cs index 4622ce03d..95d3c7b2d 100644 --- a/Duplicati/Library/Main/Operation/Restore/VolumeManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/VolumeManager.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using CoCoL; diff --git a/Duplicati/Library/Main/Operation/RestoreHandler.cs b/Duplicati/Library/Main/Operation/RestoreHandler.cs index 73620a62f..fb6e2ef64 100644 --- a/Duplicati/Library/Main/Operation/RestoreHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreHandler.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; From 3337e6a3cc2eb096ea4eb7083ac52f7f6a3fb9be Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 18 Feb 2025 09:18:09 +0100 Subject: [PATCH 182/362] Fixed review comment --- Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs b/Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs index 462ec6abd..7d5b00b90 100644 --- a/Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs +++ b/Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs @@ -123,7 +123,6 @@ namespace Duplicati.Library.Backend.AzureBlob BlobResultSegment segment; try { - segment = await _container.ListBlobsSegmentedAsync(null, false, ListDetails, null, null, null, _operationContext, cancelToken).ConfigureAwait(false); } catch (StorageException ex) @@ -140,7 +139,7 @@ namespace Duplicati.Library.Backend.AzureBlob while (segment.ContinuationToken != null) { - segment = await _container.ListBlobsSegmentedAsync(null, false, ListDetails, null, segment.ContinuationToken, null, _operationContext); + segment = await _container.ListBlobsSegmentedAsync(null, false, ListDetails, null, segment.ContinuationToken, null, _operationContext, cancelToken).ConfigureAwait(false); foreach (var item in segment.Results) yield return item; From 7031bc57b6c897a5acdbf9c0abc1af79995ed9f9 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 18 Feb 2025 09:18:36 +0100 Subject: [PATCH 183/362] Adding `ConfigureAwait(false)` to all async enumerations --- .../Backend/GoogleServices/GoogleDrive.cs | 44 +++++++++---------- .../Backend/Idrivee2/Idrivee2Backend.cs | 2 +- .../Backend/OneDrive/MicrosoftGraphBackend.cs | 2 +- Duplicati/Library/Backend/S3/S3Backend.cs | 2 +- .../Library/Backend/SSHv2/SSHv2Backend.cs | 2 +- .../Backend/SharePoint/SharePointBackend.cs | 2 +- .../Library/Interface/BackendExtensions.cs | 2 +- .../Library/Main/Operation/CompactHandler.cs | 2 +- .../Main/Operation/ListFilesHandler.cs | 4 +- .../Main/Operation/RecreateDatabaseHandler.cs | 6 +-- .../Library/Main/Operation/RepairHandler.cs | 2 +- .../Library/Main/Operation/RestoreHandler.cs | 2 +- .../Library/Main/Operation/TestHandler.cs | 2 +- 13 files changed, 37 insertions(+), 37 deletions(-) diff --git a/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs b/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs index 560974d61..96088c5b7 100644 --- a/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs +++ b/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Backend.GoogleServices; @@ -145,7 +145,7 @@ namespace Duplicati.Library.Backend.GoogleDrive { // Figure out if we update or create the file if (m_filecache.Count == 0) - await foreach (var file in ListAsync(cancelToken)) { /* Enumerate the full listing */ } + await foreach (var file in ListAsync(cancelToken).ConfigureAwait(false)) { /* Enumerate the full listing */ } GoogleDriveFolderItem[] files; m_filecache.TryGetValue(remotename, out files); @@ -188,7 +188,7 @@ namespace Duplicati.Library.Backend.GoogleDrive { // Prevent repeated download url lookups if (m_filecache.Count == 0) - await foreach (var file in ListAsync(cancelToken)) { /* Enumerate the full listing */ } + await foreach (var file in ListAsync(cancelToken).ConfigureAwait(false)) { /* Enumerate the full listing */ } var fileId = (await GetFileEntriesAsync(remotename, true, cancelToken).ConfigureAwait(false)).OrderByDescending(x => x.createdDate).First().id; @@ -214,7 +214,7 @@ namespace Duplicati.Library.Backend.GoogleDrive // For now, this class assumes that List() fully populates the file cache var currentFolderId = await GetCurrentFolderIdAsync(cancelToken).ConfigureAwait(false); - await foreach (var n in ListFolder(currentFolderId, null, null, cancelToken)) + await foreach (var n in ListFolder(currentFolderId, null, null, cancelToken).ConfigureAwait(false)) { FileEntry fe = null; diff --git a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs index 3cd5cf7f1..e28954e10 100644 --- a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs +++ b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs @@ -129,7 +129,7 @@ namespace Duplicati.Library.Backend /// public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { - await foreach (IFileEntry file in Connection.ListBucketAsync(m_bucket, m_prefix, cancelToken)) + await foreach (IFileEntry file in Connection.ListBucketAsync(m_bucket, m_prefix, cancelToken).ConfigureAwait(false)) { ((FileEntry)file).Name = file.Name.Substring(m_prefix.Length); if (file.Name.StartsWith("/", StringComparison.Ordinal) && !m_prefix.StartsWith("/", StringComparison.Ordinal)) diff --git a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs index 96476b793..5726522cd 100644 --- a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs +++ b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs @@ -323,7 +323,7 @@ namespace Duplicati.Library.Backend public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { - await foreach (var item in this.Enumerate(string.Format("{0}/root:{1}:/children", this.DrivePrefix, this.RootPath), cancelToken)) + await foreach (var item in this.Enumerate(string.Format("{0}/root:{1}:/children", this.DrivePrefix, this.RootPath), cancelToken).ConfigureAwait(false)) { // Exclude non-files and deleted items (not sure if they show up in this listing, but make sure anyway) if (item.IsFile && !item.IsDeleted) diff --git a/Duplicati/Library/Backend/S3/S3Backend.cs b/Duplicati/Library/Backend/S3/S3Backend.cs index f15cf5dfe..9be7af533 100644 --- a/Duplicati/Library/Backend/S3/S3Backend.cs +++ b/Duplicati/Library/Backend/S3/S3Backend.cs @@ -273,7 +273,7 @@ namespace Duplicati.Library.Backend /// public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { - await foreach (IFileEntry file in Connection.ListBucketAsync(m_bucket, m_prefix, cancelToken)) + await foreach (IFileEntry file in Connection.ListBucketAsync(m_bucket, m_prefix, cancelToken).ConfigureAwait(false)) { ((FileEntry)file).Name = file.Name.Substring(m_prefix.Length); diff --git a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs index 000211b6f..1c3281bc0 100644 --- a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs +++ b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs @@ -377,7 +377,7 @@ namespace Duplicati.Library.Backend await CreateConnection(cancelToken).ConfigureAwait(false); await ChangeDirectory(m_path).ConfigureAwait(false); - await foreach (var ls in m_con.ListDirectoryAsync(path, cancelToken)) + await foreach (var ls in m_con.ListDirectoryAsync(path, cancelToken).ConfigureAwait(false)) { if (ls.Name.ToString() == "." || ls.Name.ToString() == "..") continue; yield return new FileEntry(ls.Name, ls.Length, diff --git a/Duplicati/Library/Backend/SharePoint/SharePointBackend.cs b/Duplicati/Library/Backend/SharePoint/SharePointBackend.cs index 07a00330c..e3f553736 100644 --- a/Duplicati/Library/Backend/SharePoint/SharePointBackend.cs +++ b/Duplicati/Library/Backend/SharePoint/SharePointBackend.cs @@ -506,7 +506,7 @@ namespace Duplicati.Library.Backend if (retry) { // An exception was caught, and List() should be retried. - await foreach (var f in doListAsync(true, cancelToken)) + await foreach (var f in doListAsync(true, cancelToken).ConfigureAwait(false)) yield return f; } else diff --git a/Duplicati/Library/Interface/BackendExtensions.cs b/Duplicati/Library/Interface/BackendExtensions.cs index 14dfeb6e2..cf8c78f0b 100644 --- a/Duplicati/Library/Interface/BackendExtensions.cs +++ b/Duplicati/Library/Interface/BackendExtensions.cs @@ -39,7 +39,7 @@ namespace Duplicati.Library.Interface { using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancelToken); // If we can iterate successfully, even if it's empty, then the backend test is successful - await foreach (IFileEntry file in backend.ListAsync(cts.Token)) + await foreach (IFileEntry file in backend.ListAsync(cts.Token).ConfigureAwait(false)) { cts.Cancel(); return; diff --git a/Duplicati/Library/Main/Operation/CompactHandler.cs b/Duplicati/Library/Main/Operation/CompactHandler.cs index 5f245b916..b691e9ffe 100644 --- a/Duplicati/Library/Main/Operation/CompactHandler.cs +++ b/Duplicati/Library/Main/Operation/CompactHandler.cs @@ -160,7 +160,7 @@ namespace Duplicati.Library.Main.Operation using (var q = db.CreateBlockQueryHelper(transaction)) { - await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(volumesToDownload, m_result.TaskControl.ProgressToken)) + await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(volumesToDownload, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { using (tmpfile) { diff --git a/Duplicati/Library/Main/Operation/ListFilesHandler.cs b/Duplicati/Library/Main/Operation/ListFilesHandler.cs index dfdc414ed..263009748 100644 --- a/Duplicati/Library/Main/Operation/ListFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFilesHandler.cs @@ -176,7 +176,7 @@ namespace Duplicati.Library.Main.Operation long flindex = 1; var filteredListMap = filteredList.ToDictionary(x => x.Value.File.Name, x => x.Value); - await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(filteredList.Select(x => new RemoteVolumeMapper(x.Value)), cancellationToken)) + await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(filteredList.Select(x => new RemoteVolumeMapper(x.Value)), cancellationToken).ConfigureAwait(false)) { var flentry = filteredListMap[name]; using (tmpfile) @@ -237,7 +237,7 @@ namespace Duplicati.Library.Main.Operation { var list = new List(); var map = filteredList.ToDictionary(x => x.Value.File.Name, x => x); - await foreach (var (file, hash, size, name) in backendManager.GetFilesOverlappedAsync(filteredList.Select(x => new RemoteVolumeMapper(x.Value)), cancelToken)) + await foreach (var (file, hash, size, name) in backendManager.GetFilesOverlappedAsync(filteredList.Select(x => new RemoteVolumeMapper(x.Value)), cancelToken).ConfigureAwait(false)) { // We must obtain the partial/full status from the fileset file in the dlist files. // Without this, the restore dialog will show all versions as full, or all versions diff --git a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs index 1cbbfabcc..66481fb30 100644 --- a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs +++ b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs @@ -208,7 +208,7 @@ namespace Duplicati.Library.Main.Operation var blocksize = m_options.Blocksize; var hashes_pr_block = blocksize / m_options.BlockhashSize; - await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(filelistWork, m_result.TaskControl.ProgressToken)) + await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(filelistWork, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { var entry = new RemoteVolume(name, hash, size); try @@ -378,7 +378,7 @@ namespace Duplicati.Library.Main.Operation var progress = 0; - await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(indexfiles, m_result.TaskControl.ProgressToken)) + await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(indexfiles, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { try { @@ -516,7 +516,7 @@ namespace Duplicati.Library.Main.Operation } var progress = 0; - await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(lst, m_result.TaskControl.ProgressToken)) + await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(lst, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { try { diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 821e75381..a0b6b5eb5 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -424,7 +424,7 @@ namespace Duplicati.Library.Main.Operation } //Then we grab all remote volumes that have the missing blocks - await foreach (var (tmpfile, _, _, name) in backendManager.GetFilesOverlappedAsync(mbl.GetMissingBlockSources().ToList(), cancellationToken)) + await foreach (var (tmpfile, _, _, name) in backendManager.GetFilesOverlappedAsync(mbl.GetMissingBlockSources().ToList(), cancellationToken).ConfigureAwait(false)) { try { diff --git a/Duplicati/Library/Main/Operation/RestoreHandler.cs b/Duplicati/Library/Main/Operation/RestoreHandler.cs index 73620a62f..1383f1a8f 100644 --- a/Duplicati/Library/Main/Operation/RestoreHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreHandler.cs @@ -489,7 +489,7 @@ namespace Duplicati.Library.Main.Operation var brokenFiles = new List(); - await foreach (var (tmpfile, _, _, name) in backendManager.GetFilesOverlappedAsync(volumes, cancellationToken)) + await foreach (var (tmpfile, _, _, name) in backendManager.GetFilesOverlappedAsync(volumes, cancellationToken).ConfigureAwait(false)) { try { diff --git a/Duplicati/Library/Main/Operation/TestHandler.cs b/Duplicati/Library/Main/Operation/TestHandler.cs index 21f485cf3..f73b5d745 100644 --- a/Duplicati/Library/Main/Operation/TestHandler.cs +++ b/Duplicati/Library/Main/Operation/TestHandler.cs @@ -74,7 +74,7 @@ namespace Duplicati.Library.Main.Operation if (m_options.FullRemoteVerification != Options.RemoteTestStrategy.False) { - await foreach (var (tf, hash, size, name) in backend.GetFilesOverlappedAsync(files, CancellationToken.None)) + await foreach (var (tf, hash, size, name) in backend.GetFilesOverlappedAsync(files, CancellationToken.None).ConfigureAwait(false)) { var vol = new RemoteVolume(name, hash, size); try From ce60b807c8c1fa49aee59bc40947a35b273b5d43 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 18 Feb 2025 09:18:46 +0100 Subject: [PATCH 184/362] Added additional profiling logging --- .../Library/Main/Operation/RestoreHandler.cs | 47 +++++++++++-------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/Duplicati/Library/Main/Operation/RestoreHandler.cs b/Duplicati/Library/Main/Operation/RestoreHandler.cs index fb6e2ef64..86fd0030c 100644 --- a/Duplicati/Library/Main/Operation/RestoreHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreHandler.cs @@ -322,11 +322,14 @@ namespace Duplicati.Library.Main.Operation // Prepare the block and file list and create the directory structure m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateFileList); - PrepareBlockAndFileList(database, m_options, filter, m_result); + using (new Logging.Timer(LOGTAG, "PrepareBlockList", "PrepareBlockList")) + PrepareBlockAndFileList(database, m_options, filter, m_result); m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateTargetFolders); - await CreateDirectoryStructure(database, m_options, m_result).ConfigureAwait(false); + using (new Logging.Timer(LOGTAG, "CreateDirectory", "CreateDirectory")) + await CreateDirectoryStructure(database, m_options, m_result).ConfigureAwait(false); database.SetResult(m_result); + using var setup_log_timer = new Logging.Timer(LOGTAG, "RestoreNetworkSetup", "RestoreNetworkSetup"); // Create the channels between BlockManager and FileProcessor var fileprocessor_requests = new Channel[m_options.RestoreFileProcessors].Select(_ => ChannelManager.CreateChannel(buffersize: Restore.Channels.BufferSize)).ToArray(); var fileprocessor_responses = new Channel[m_options.RestoreFileProcessors].Select(_ => ChannelManager.CreateChannel(buffersize: Restore.Channels.BufferSize)).ToArray(); @@ -341,6 +344,8 @@ namespace Duplicati.Library.Main.Operation var volumedecryptors = Enumerable.Range(0, m_options.RestoreVolumeDecryptors).Select(i => Restore.VolumeDecryptor.Run(channels, m_options)).ToArray(); var volumedecompressors = Enumerable.Range(0, m_options.RestoreVolumeDecompressors).Select(i => Restore.VolumeDecompressor.Run(channels, m_options)).ToArray(); + setup_log_timer.Dispose(); + // Wait for the network to complete Task[] all = [ @@ -354,6 +359,7 @@ namespace Duplicati.Library.Main.Operation ]; // Start the progress updater + using (new Logging.Timer(LOGTAG, "RestoreNetworkWait", "RestoreNetworkWait")) using (var kill_updater = new CancellationTokenSource()) { var updater = Task.Run(async () => @@ -489,28 +495,29 @@ namespace Duplicati.Library.Main.Operation var brokenFiles = new List(); - await foreach (var (tmpfile, _, _, name) in backendManager.GetFilesOverlappedAsync(volumes, cancellationToken)) - { - try + using (new Logging.Timer(LOGTAG, "PatchWithBlocklist", "PatchWithBlocklist")) + await foreach (var (tmpfile, _, _, name) in backendManager.GetFilesOverlappedAsync(volumes, cancellationToken)) { - if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + try { - await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false); - return; - } + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + { + await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false); + return; + } - using (tmpfile) - using (var blocks = new BlockVolumeReader(GetCompressionModule(name), tmpfile, m_options)) - PatchWithBlocklist(database, blocks, m_options, m_result, m_blockbuffer, metadatastorage); + using (tmpfile) + using (var blocks = new BlockVolumeReader(GetCompressionModule(name), tmpfile, m_options)) + PatchWithBlocklist(database, blocks, m_options, m_result, m_blockbuffer, metadatastorage); + } + catch (Exception ex) + { + brokenFiles.Add(name); + Logging.Log.WriteErrorMessage(LOGTAG, "PatchingFailed", ex, "Failed to patch with remote file: \"{0}\", message: {1}", name, ex.Message); + if (ex is System.Threading.ThreadAbortException) + throw; + } } - catch (Exception ex) - { - brokenFiles.Add(name); - Logging.Log.WriteErrorMessage(LOGTAG, "PatchingFailed", ex, "Failed to patch with remote file: \"{0}\", message: {1}", name, ex.Message); - if (ex is System.Threading.ThreadAbortException) - throw; - } - } var fileErrors = 0L; From 17e2c0b9cf912e7f0a93ed6828acd2eac99763fc Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 18 Feb 2025 09:20:28 +0100 Subject: [PATCH 185/362] Added ConfigureAwait(false) --- .../Main/Operation/Restore/FileLister.cs | 2 +- .../Main/Operation/Restore/FileProcessor.cs | 36 +++++++++---------- .../Operation/Restore/VolumeDecompressor.cs | 4 +-- .../Main/Operation/Restore/VolumeDecryptor.cs | 2 +- .../Main/Operation/Restore/VolumeManager.cs | 6 ++-- 5 files changed, 25 insertions(+), 25 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/FileLister.cs b/Duplicati/Library/Main/Operation/Restore/FileLister.cs index 64f3dec02..15e7cc18b 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileLister.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileLister.cs @@ -67,7 +67,7 @@ namespace Duplicati.Library.Main.Operation.Restore sw_write?.Start(); foreach (var file in files) { - await self.Output.WriteAsync(file); + await self.Output.WriteAsync(file).ConfigureAwait(false); } sw_write?.Stop(); } diff --git a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs index 48d79df0e..abef8218f 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs @@ -82,7 +82,7 @@ namespace Duplicati.Library.Main.Operation.Restore { // Get the next file to restore. sw_file?.Start(); - var file = await self.Input.ReadAsync(); + var file = await self.Input.ReadAsync().ConfigureAwait(false); sw_file?.Stop(); // Get information about the blocks for the file @@ -93,7 +93,7 @@ namespace Duplicati.Library.Main.Operation.Restore sw_work?.Start(); // Verify the target file blocks that may already exist. - var (bytes_verified, missing_blocks, verified_blocks) = await VerifyTargetBlocks(file, blocks, filehasher, blockhasher, options, results); + var (bytes_verified, missing_blocks, verified_blocks) = await VerifyTargetBlocks(file, blocks, filehasher, blockhasher, options, results).ConfigureAwait(false); long bytes_written = 0; if (blocks.Length != missing_blocks.Count + verified_blocks.Count) { @@ -124,7 +124,7 @@ namespace Duplicati.Library.Main.Operation.Restore } else { - await CopyOldTargetBlocksToNewTarget(file, new_file, verified_blocks); + await CopyOldTargetBlocksToNewTarget(file, new_file, verified_blocks).ConfigureAwait(false); } } else @@ -141,14 +141,14 @@ namespace Duplicati.Library.Main.Operation.Restore { block.CacheDecrEvict = true; sw_req?.Start(); - await block_request.WriteAsync(block); + await block_request.WriteAsync(block).ConfigureAwait(false); sw_req?.Stop(); } if (missing_blocks.Count > 0 && options.UseLocalBlocks) { // Verify the local blocks at the original restore path that may be used to restore the file. - (bytes_written, missing_blocks) = await VerifyLocalBlocks(file, missing_blocks, blocks.Length, filehasher, blockhasher, options, results, block_request); + (bytes_written, missing_blocks) = await VerifyLocalBlocks(file, missing_blocks, blocks.Length, filehasher, blockhasher, options, results, block_request).ConfigureAwait(false); } bool empty_file_or_symlink = false; @@ -174,7 +174,7 @@ namespace Duplicati.Library.Main.Operation.Restore if (missing_blocks.Count != 0) { blocks[0].CacheDecrEvict = true; - await block_request.WriteAsync(blocks[0]); + await block_request.WriteAsync(blocks[0]).ConfigureAwait(false); } } } @@ -222,7 +222,7 @@ namespace Duplicati.Library.Main.Operation.Restore int j = 0; for (int i = 0; i < (int)Math.Min(missing_blocks.Count, burst); i++) { - await block_request.WriteAsync(missing_blocks[i]); + await block_request.WriteAsync(missing_blocks[i]).ConfigureAwait(false); } sw_req?.Stop(); @@ -241,12 +241,12 @@ namespace Duplicati.Library.Main.Operation.Restore // Read the block from the response and issue a new request, if more blocks are missing sw_work?.Stop(); sw_resp?.Start(); - var data = await block_response.ReadAsync(); + var data = await block_response.ReadAsync().ConfigureAwait(false); sw_resp?.Stop(); sw_req?.Start(); if (j < missing_blocks.Count - burst) { - await block_request.WriteAsync(missing_blocks[j + burst]); + await block_request.WriteAsync(missing_blocks[j + burst]).ConfigureAwait(false); } sw_req?.Stop(); sw_work?.Start(); @@ -261,7 +261,7 @@ namespace Duplicati.Library.Main.Operation.Restore else { // Write the block to the file - await fs.WriteAsync(data); + await fs.WriteAsync(data).ConfigureAwait(false); } // Keep track of metrics @@ -321,7 +321,7 @@ namespace Duplicati.Library.Main.Operation.Restore if (!options.SkipMetadata) { - empty_file_or_symlink |= await RestoreMetadata(db, file, block_request, block_response, options, sw_meta, sw_work, sw_req, sw_resp); + empty_file_or_symlink |= await RestoreMetadata(db, file, block_request, block_response, options, sw_meta, sw_work, sw_req, sw_resp).ConfigureAwait(false); } // TODO legacy restore doesn't count metadata restore as a restored file. @@ -379,8 +379,8 @@ namespace Duplicati.Library.Main.Operation.Restore fs_new.Seek(block.BlockOffset * block.BlockSize, SeekOrigin.Begin); var buffer = new byte[block.BlockSize]; - await fs_old.ReadAsync(buffer, 0, buffer.Length); - await fs_new.WriteAsync(buffer, 0, buffer.Length); + await fs_old.ReadAsync(buffer, 0, buffer.Length).ConfigureAwait(false); + await fs_new.WriteAsync(buffer, 0, buffer.Length).ConfigureAwait(false); ; } } @@ -470,10 +470,10 @@ namespace Duplicati.Library.Main.Operation.Restore { sw_work?.Stop(); sw_req?.Start(); - await block_request.WriteAsync(block); + await block_request.WriteAsync(block).ConfigureAwait(false); sw_req?.Stop(); sw_resp?.Start(); - var data = await block_response.ReadAsync(); + var data = await block_response.ReadAsync().ConfigureAwait(false); sw_resp?.Stop(); sw_work?.Start(); ms.Write(data, 0, data.Length); @@ -509,7 +509,7 @@ namespace Duplicati.Library.Main.Operation.Restore var buffer = new byte[options.Blocksize]; for (int i = 0; i < blocks.Length; i++) { - var read = await f.ReadAsync(buffer, 0, (int)blocks[i].BlockSize); + var read = await f.ReadAsync(buffer, 0, (int)blocks[i].BlockSize).ConfigureAwait(false); if (read == blocks[i].BlockSize) { // Block is present @@ -632,7 +632,7 @@ namespace Duplicati.Library.Main.Operation.Restore try { f_original.Seek(i * options.Blocksize, SeekOrigin.Begin); - read = await f_original.ReadAsync(buffer, 0, (int)blocks[j].BlockSize); + read = await f_original.ReadAsync(buffer, 0, (int)blocks[j].BlockSize).ConfigureAwait(false); } catch (Exception) { @@ -671,7 +671,7 @@ namespace Duplicati.Library.Main.Operation.Restore bytes_read += read; bytes_written += read; blocks[j].CacheDecrEvict = true; - await block_request.WriteAsync(blocks[j]); + await block_request.WriteAsync(blocks[j]).ConfigureAwait(false); verified_blocks.Add(blocks[j]); } } diff --git a/Duplicati/Library/Main/Operation/Restore/VolumeDecompressor.cs b/Duplicati/Library/Main/Operation/Restore/VolumeDecompressor.cs index f2e409e70..a1f4afa0f 100644 --- a/Duplicati/Library/Main/Operation/Restore/VolumeDecompressor.cs +++ b/Duplicati/Library/Main/Operation/Restore/VolumeDecompressor.cs @@ -65,7 +65,7 @@ namespace Duplicati.Library.Main.Operation.Restore { sw_read?.Start(); // Get the block request and volume from the `VolumeDecryptor` process. - var (block_request, volume_reader) = await self.Input.ReadAsync(); + var (block_request, volume_reader) = await self.Input.ReadAsync().ConfigureAwait(false); sw_read?.Stop(); sw_decompress?.Start(); @@ -86,7 +86,7 @@ namespace Duplicati.Library.Main.Operation.Restore sw_write?.Start(); // Send the block to the `BlockManager` process. - await self.Output.WriteAsync((block_request, data)); + await self.Output.WriteAsync((block_request, data)).ConfigureAwait(false); sw_write?.Stop(); } } diff --git a/Duplicati/Library/Main/Operation/Restore/VolumeDecryptor.cs b/Duplicati/Library/Main/Operation/Restore/VolumeDecryptor.cs index ae154b3d8..2dc5fa8e8 100644 --- a/Duplicati/Library/Main/Operation/Restore/VolumeDecryptor.cs +++ b/Duplicati/Library/Main/Operation/Restore/VolumeDecryptor.cs @@ -70,7 +70,7 @@ namespace Duplicati.Library.Main.Operation.Restore sw_write?.Start(); // Pass the decrypted volume to the `VolumeDecompressor` process. - await self.Output.WriteAsync((volume_id, volume, bvr)); + await self.Output.WriteAsync((volume_id, volume, bvr)).ConfigureAwait(false); sw_write?.Stop(); } } diff --git a/Duplicati/Library/Main/Operation/Restore/VolumeManager.cs b/Duplicati/Library/Main/Operation/Restore/VolumeManager.cs index 95d3c7b2d..ba14a643d 100644 --- a/Duplicati/Library/Main/Operation/Restore/VolumeManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/VolumeManager.cs @@ -76,7 +76,7 @@ namespace Duplicati.Library.Main.Operation.Restore { while (true) { - var msg = await self.VolumeRequestResponse.ReadAsync(); + var msg = await self.VolumeRequestResponse.ReadAsync().ConfigureAwait(false); switch (msg) { @@ -96,7 +96,7 @@ namespace Duplicati.Library.Main.Operation.Restore sw_request?.Start(); if (cache.TryGetValue(request.VolumeID, out BlockVolumeReader reader)) { - await self.DecompressRequest.WriteAsync((request, reader)); + await self.DecompressRequest.WriteAsync((request, reader)).ConfigureAwait(false); } else { @@ -129,7 +129,7 @@ namespace Duplicati.Library.Main.Operation.Restore sw_wakeup?.Start(); foreach (var request in in_flight[volume_id]) { - await self.DecompressRequest.WriteAsync((request, reader)); + await self.DecompressRequest.WriteAsync((request, reader)).ConfigureAwait(false); } in_flight.Remove(volume_id); sw_wakeup?.Stop(); From a0d5096ebc56ab81bc2555fd5b39b4e47b0884c8 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 18 Feb 2025 09:25:20 +0100 Subject: [PATCH 186/362] Reworked the "Volume loop" of the restore network to handle Backends themselves, rather than using the BackendManager. This allows for tighter control over which process performs which action, allowing them more easily to run in parallel. --- .../Main/Operation/Restore/Channels.cs | 5 +- .../Main/Operation/Restore/VolumeDecryptor.cs | 90 ++++++++++++++++++- .../Operation/Restore/VolumeDownloader.cs | 21 +++-- .../Main/Operation/Restore/VolumeManager.cs | 15 +--- .../Library/Main/Operation/RestoreHandler.cs | 4 +- 5 files changed, 110 insertions(+), 25 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/Channels.cs b/Duplicati/Library/Main/Operation/Restore/Channels.cs index 4d14ee055..9b83773da 100644 --- a/Duplicati/Library/Main/Operation/Restore/Channels.cs +++ b/Duplicati/Library/Main/Operation/Restore/Channels.cs @@ -19,7 +19,6 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System.Threading.Tasks; using CoCoL; using Duplicati.Library.Main.Volumes; using Duplicati.Library.Utility; @@ -46,12 +45,12 @@ namespace Duplicati.Library.Main.Operation.Restore /// /// Channel between and . /// - public readonly IChannel<(long, Task)> DownloadRequest = ChannelManager.CreateChannel<(long, Task)>(buffersize: BufferSize); + public readonly IChannel DownloadRequest = ChannelManager.CreateChannel(buffersize: BufferSize); /// /// Channel between and /// - public readonly IChannel<(long, TempFile)> DecryptRequest = ChannelManager.CreateChannel<(long, TempFile)>(buffersize: BufferSize); + public readonly IChannel<(long, string, TempFile)> DecryptRequest = ChannelManager.CreateChannel<(long, string, TempFile)>(buffersize: BufferSize); /// /// Channel between and diff --git a/Duplicati/Library/Main/Operation/Restore/VolumeDecryptor.cs b/Duplicati/Library/Main/Operation/Restore/VolumeDecryptor.cs index 2dc5fa8e8..59576fac9 100644 --- a/Duplicati/Library/Main/Operation/Restore/VolumeDecryptor.cs +++ b/Duplicati/Library/Main/Operation/Restore/VolumeDecryptor.cs @@ -21,6 +21,7 @@ using System; using System.Diagnostics; +using System.Linq; using System.Threading.Tasks; using CoCoL; using Duplicati.Library.Main.Volumes; @@ -57,15 +58,23 @@ namespace Duplicati.Library.Main.Operation.Restore Stopwatch sw_decrypt = options.InternalProfiling ? new() : null; try { + // Taken from BackendManager.GetOperation + using var encryption = options.NoEncryption + ? null + : (DynamicLoader.EncryptionLoader.GetModule(options.EncryptionModule, options.Passphrase, options.RawOptions) + ?? throw new Exception(Strings.BackendMananger.EncryptionModuleNotFound(options.EncryptionModule)) + ); + while (true) { // Get the block request and volume from the `VolumeDownloader` process. sw_read?.Start(); - var (volume_id, volume) = await self.Input.ReadAsync(); + var (volume_id, volume_name, volume) = await self.Input.ReadAsync().ConfigureAwait(false); sw_read?.Stop(); sw_decrypt?.Start(); - var bvr = new BlockVolumeReader(options.CompressionModule, volume, options); + var tmpfile = DecryptFile(volume, DetectEncryptionModule(volume_name, options, encryption)); + var bvr = new BlockVolumeReader(options.CompressionModule, tmpfile, options); sw_decrypt?.Stop(); sw_write?.Start(); @@ -90,6 +99,83 @@ namespace Duplicati.Library.Main.Operation.Restore } }); } + + // Taken from BackendManager.GetOperation + public static Library.Utility.TempFile DecryptFile(Library.Utility.TempFile tempFile, Interface.IEncryption? decrypter) + { + // Support no encryption + if (decrypter == null) + return tempFile; + + Library.Utility.TempFile? decryptTarget = null; + + // Always dispose the source file + using (tempFile) + using (new Logging.Timer(LOGTAG, "DecryptFile", "Decrypting " + tempFile)) + { + try + { + decryptTarget = new Library.Utility.TempFile(); + try { decrypter.Decrypt(tempFile, decryptTarget); } + // If we fail here, make sure that we throw a crypto exception + catch (System.Security.Cryptography.CryptographicException) { throw; } + catch (Exception ex) { throw new System.Security.Cryptography.CryptographicException(ex.Message, ex); } + + var result = decryptTarget; + decryptTarget = null; + return result; + } + finally + { + // Remove temp files on failure + decryptTarget?.Dispose(); + } + } + } + + private static Interface.IEncryption? DetectEncryptionModule(string remotefilename, Options options, Interface.IEncryption? encryption) + { + try + { + // Auto-guess the encryption module + var ext = (System.IO.Path.GetExtension(remotefilename) ?? "").TrimStart('.'); + if (!ext.Equals(encryption?.FilenameExtension, StringComparison.OrdinalIgnoreCase)) + { + // Check if the file is not encrypted + if (DynamicLoader.CompressionLoader.Keys.Contains(ext, StringComparer.OrdinalIgnoreCase)) + { + if (encryption != null) + Logging.Log.WriteVerboseMessage(LOGTAG, "AutomaticDecryptionDetection", "Filename extension \"{0}\" does not match encryption module \"{1}\", guessing that it is not encrypted", ext, options.EncryptionModule); + return null; + } + // Check if the file is encrypted with something else + else if (DynamicLoader.EncryptionLoader.Keys.Contains(ext, StringComparer.OrdinalIgnoreCase)) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "AutomaticDecryptionDetection", "Filename extension \"{0}\" does not match encryption module \"{1}\", attempting to use matching encryption module", ext, options.EncryptionModule); + + try + { + return DynamicLoader.EncryptionLoader.GetModule(ext, options.Passphrase, options.RawOptions) + ?? encryption; + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "AutomaticDecryptionDetection", ex, "Failed to load encryption module \"{0}\", using specified encryption module \"{1}\"", ext, options.EncryptionModule); + } + } + // Fallback, lets see what happens... + else + { + Logging.Log.WriteVerboseMessage(LOGTAG, "AutomaticDecryptionDetection", "Filename extension \"{0}\" does not match encryption module \"{1}\", attempting to use specified encryption module as no others match", ext, options.EncryptionModule); + } + } + + return encryption; + } + // If we fail here, make sure that we throw a crypto exception + catch (System.Security.Cryptography.CryptographicException) { throw; } + catch (Exception ex) { throw new System.Security.Cryptography.CryptographicException(ex.Message, ex); } + } } } \ No newline at end of file diff --git a/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs b/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs index 40a28d772..65c20c92a 100644 --- a/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs +++ b/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs @@ -22,8 +22,10 @@ using System; using System.Diagnostics; using System.Linq; +using System.Threading; using System.Threading.Tasks; using CoCoL; +using Duplicati.Library.Interface; using Duplicati.Library.Main.Database; using Duplicati.Library.Utility; @@ -43,9 +45,12 @@ namespace Duplicati.Library.Main.Operation.Restore /// /// Runs the volume downloader process. /// + /// The named channels for the restore operation. + /// The local restore database, used to find volume information given the volume ID. + /// The backend to use for downloading the volumes. /// The restore options. /// The restore results. - public static Task Run(Channels channels, LocalRestoreDatabase db, Options options, RestoreResults results) + public static Task Run(Channels channels, LocalRestoreDatabase db, IBackend backend, Options options, RestoreResults results) { return AutomationExtensions.RunTask( new @@ -65,19 +70,19 @@ namespace Duplicati.Library.Main.Operation.Restore { // Get the block request from the `BlockManager` process. sw_read?.Start(); - var (volume_id, waittask) = await self.Input.ReadAsync(); + var volume_id = await self.Input.ReadAsync().ConfigureAwait(false); sw_read?.Stop(); // Trigger the download. sw_wait?.Start(); - TempFile f = null; + TempFile f = new(); + var (volume_name, size, hash) = db.GetVolumeInfo(volume_id).First(); try { - f = await waittask.ConfigureAwait(false); + await backend.GetAsync(volume_name, f.Name, CancellationToken.None).ConfigureAwait(false); } catch (Exception) { - var (volume_name, _, _) = db.GetVolumeInfo(volume_id).First(); lock (results) results.BrokenRemoteFiles.Add(volume_name); @@ -87,7 +92,7 @@ namespace Duplicati.Library.Main.Operation.Restore // Pass the download handle (which may or may not have downloaded already) to the `VolumeDecryptor` process. sw_write?.Start(); - await self.Output.WriteAsync((volume_id, f)); + await self.Output.WriteAsync((volume_id, volume_name, f)).ConfigureAwait(false); sw_write?.Stop(); } } @@ -105,6 +110,10 @@ namespace Duplicati.Library.Main.Operation.Restore Logging.Log.WriteErrorMessage(LOGTAG, "DownloadError", ex, "Error during download"); throw; } + finally + { + backend.Dispose(); + } }); } } diff --git a/Duplicati/Library/Main/Operation/Restore/VolumeManager.cs b/Duplicati/Library/Main/Operation/Restore/VolumeManager.cs index ba14a643d..0c636973c 100644 --- a/Duplicati/Library/Main/Operation/Restore/VolumeManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/VolumeManager.cs @@ -20,7 +20,6 @@ // DEALINGS IN THE SOFTWARE. using CoCoL; -using Duplicati.Library.Main.Database; using Duplicati.Library.Main.Volumes; using Duplicati.Library.Utility; using System; @@ -46,11 +45,9 @@ namespace Duplicati.Library.Main.Operation.Restore /// /// Runs the volume manager process. /// - /// The local restore database, used to find the volume where a block is stored. - /// The backend manager, used to fetch the volumes from the backend. + /// The named channels for the restore operation. /// The restore options. - /// The restore results. - public static Task Run(Channels channels, LocalRestoreDatabase db, IBackendManager backend, Options options, RestoreResults results) + public static Task Run(Channels channels, Options options) { return AutomationExtensions.RunTask( new @@ -106,13 +103,7 @@ namespace Duplicati.Library.Main.Operation.Restore } else { - sw_query?.Start(); - var (volume_name, volume_size, volume_hash) = db.GetVolumeInfo(request.VolumeID).First(); - sw_query?.Stop(); - sw_backend?.Start(); - var handle = backend.GetAsync(volume_name, volume_hash, volume_size, results.TaskControl.TransferToken); - sw_backend?.Stop(); - await self.DownloadRequest.WriteAsync((request.VolumeID, handle)); + await self.DownloadRequest.WriteAsync(request.VolumeID).ConfigureAwait(false); in_flight[request.VolumeID] = [request]; } } diff --git a/Duplicati/Library/Main/Operation/RestoreHandler.cs b/Duplicati/Library/Main/Operation/RestoreHandler.cs index 86fd0030c..c119e47a3 100644 --- a/Duplicati/Library/Main/Operation/RestoreHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreHandler.cs @@ -339,8 +339,8 @@ namespace Duplicati.Library.Main.Operation var filelister = Restore.FileLister.Run(channels, database, m_options, m_result); var fileprocessors = Enumerable.Range(0, m_options.RestoreFileProcessors).Select(i => Restore.FileProcessor.Run(channels, database, fileprocessor_requests[i], fileprocessor_responses[i], m_options, m_result)).ToArray(); var blockmanager = Restore.BlockManager.Run(channels, database, m_options, fileprocessor_requests, fileprocessor_responses); - var volumecache = Restore.VolumeManager.Run(channels, database, backendManager, m_options, m_result); - var volumedownloaders = Enumerable.Range(0, m_options.RestoreVolumeDownloaders).Select(i => Restore.VolumeDownloader.Run(channels, database, m_options, m_result)).ToArray(); + var volumecache = Restore.VolumeManager.Run(channels, m_options); + var volumedownloaders = Enumerable.Range(0, m_options.RestoreVolumeDownloaders).Select(i => Restore.VolumeDownloader.Run(channels, database, backendManager.GetBackend(), m_options, m_result)).ToArray(); var volumedecryptors = Enumerable.Range(0, m_options.RestoreVolumeDecryptors).Select(i => Restore.VolumeDecryptor.Run(channels, m_options)).ToArray(); var volumedecompressors = Enumerable.Range(0, m_options.RestoreVolumeDecompressors).Select(i => Restore.VolumeDecompressor.Run(channels, m_options)).ToArray(); From 1c8396efd77e792c839cb435a3adff46feed7cd1 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 18 Feb 2025 09:39:57 +0100 Subject: [PATCH 187/362] Add random seed option to test data generation command --- Tools/TestDataGenerator/Commands/Create.cs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Tools/TestDataGenerator/Commands/Create.cs b/Tools/TestDataGenerator/Commands/Create.cs index 4eed98130..de576d996 100644 --- a/Tools/TestDataGenerator/Commands/Create.cs +++ b/Tools/TestDataGenerator/Commands/Create.cs @@ -22,6 +22,7 @@ public static class Create /// The maximum depth of subfolders to create in the target folder /// The maximum number of folders to create in the target folder /// The maximum length of each path segment in the target folder + /// The random seed to use for generating the test data record CommandInput( DirectoryInfo TargetFolder, int FileCount, @@ -32,7 +33,8 @@ public static class Create int MaxFanOut, int MaxDepth, int MaxFolderCount, - int MaxPathSegmentLength); + int MaxPathSegmentLength, + string RandomSeed); /// /// Creates the command @@ -82,6 +84,10 @@ public static class Create maxPathSegmentLengthOption.SetDefaultValue(15); command.AddOption(maxPathSegmentLengthOption); + var randomSeedOption = new Option("--random-seed", "The random seed to use for generating the test data"); + randomSeedOption.SetDefaultValue("Duplicati"); + command.AddOption(randomSeedOption); + command.Handler = CommandHandler.Create(Execute); return command; @@ -121,7 +127,7 @@ public static class Create throw new Exception("The maximum path segment length must be greater than zero"); Console.WriteLine($"Creating test data in {input.TargetFolder.FullName}"); - var rnd = new Random(); + var rnd = new Random(input.RandomSeed.GetHashCode()); var folders = GeneratePathStructure(rnd, input.TargetFolder.FullName, input.MaxDepth, input.MaxFanOut, input.MaxPathSegmentLength); while (folders.Count < input.MaxFolderCount) { From 211dbcd8e208a0cb5417aee721e1b8a0b4807010 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 18 Feb 2025 09:54:38 +0100 Subject: [PATCH 188/362] Fixed a bug where the file creation throws a permission exception, because a folder with the same name exists. --- Tools/TestDataGenerator/Commands/Create.cs | 4 ++++ Tools/TestDataGenerator/Commands/Shared.cs | 12 +++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/Tools/TestDataGenerator/Commands/Create.cs b/Tools/TestDataGenerator/Commands/Create.cs index de576d996..d1ff6ba7f 100644 --- a/Tools/TestDataGenerator/Commands/Create.cs +++ b/Tools/TestDataGenerator/Commands/Create.cs @@ -180,6 +180,10 @@ public static class Create { try { + // Sometimes with a large number of files, the file name clashes with a folder name. In that case, generate a new file name + while (Directory.Exists(file)) + file = Path.Combine(Path.GetDirectoryName(file), GenerateFileName(rnd, input.MaxPathSegmentLength)); + using var fs = new FileStream(file, FileMode.Create, FileAccess.Write); if (size > 0) { diff --git a/Tools/TestDataGenerator/Commands/Shared.cs b/Tools/TestDataGenerator/Commands/Shared.cs index ac9afad60..caf3e614b 100644 --- a/Tools/TestDataGenerator/Commands/Shared.cs +++ b/Tools/TestDataGenerator/Commands/Shared.cs @@ -81,6 +81,16 @@ public static class Shared return names; } + /// + /// Generates a random file name + /// + /// The random number generator to use + /// The maximum length of the file name + public static string GenerateFileName(Random rnd, int maxLength) + { + return GetPathSegment(rnd, maxLength) + FileExtensions[rnd.Next(FileExtensions.Count)]; + } + /// /// Generates a list of random file names /// @@ -92,7 +102,7 @@ public static class Shared { var names = new List(count); for (int i = 0; i < count; i++) - names.Add(GetPathSegment(rnd, maxLength) + FileExtensions[rnd.Next(FileExtensions.Count)]); + names.Add(GenerateFileName(rnd, maxLength)); return names; } From 7ca9a186481d9ca189acb8abbe2afc9f29424b8c Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 18 Feb 2025 09:59:25 +0100 Subject: [PATCH 189/362] Cannot assign a value to a variable in a foreach loop --- Tools/TestDataGenerator/Commands/Create.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Tools/TestDataGenerator/Commands/Create.cs b/Tools/TestDataGenerator/Commands/Create.cs index d1ff6ba7f..c6be766b5 100644 --- a/Tools/TestDataGenerator/Commands/Create.cs +++ b/Tools/TestDataGenerator/Commands/Create.cs @@ -178,13 +178,14 @@ public static class Create foreach (var (file, size) in filesWithSizes) { + var filename = file; try { // Sometimes with a large number of files, the file name clashes with a folder name. In that case, generate a new file name - while (Directory.Exists(file)) - file = Path.Combine(Path.GetDirectoryName(file), GenerateFileName(rnd, input.MaxPathSegmentLength)); + while (Directory.Exists(filename)) + filename = Path.Combine(Path.GetDirectoryName(filename) ?? "", GenerateFileName(rnd, input.MaxPathSegmentLength)); - using var fs = new FileStream(file, FileMode.Create, FileAccess.Write); + using var fs = new FileStream(filename, FileMode.Create, FileAccess.Write); if (size > 0) { fs.SetLength(size); @@ -203,7 +204,7 @@ public static class Create } catch (Exception ex) { - Console.WriteLine($"Error creating file {file}: {ex.Message}"); + Console.WriteLine($"Error creating file {filename}: {ex.Message}"); } } From f3c8412f30be348681045cccc73fadc4801ec61f Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 18 Feb 2025 10:20:44 +0100 Subject: [PATCH 190/362] Added throughput and ETA reporting --- Tools/TestDataGenerator/Commands/Create.cs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Tools/TestDataGenerator/Commands/Create.cs b/Tools/TestDataGenerator/Commands/Create.cs index c6be766b5..21ca748cf 100644 --- a/Tools/TestDataGenerator/Commands/Create.cs +++ b/Tools/TestDataGenerator/Commands/Create.cs @@ -196,10 +196,14 @@ public static class Create filesCreated++; fileSizeCreated += size; - if ((DateTime.UtcNow - lastUpdate) > updateInterval) + var now = DateTime.UtcNow; + if ((now - lastUpdate) > updateInterval) { - Console.WriteLine($"Created {filesCreated} of {files.Count} files ({SizeToHumanReadable(fileSizeCreated)} of {SizeToHumanReadable(totalSize)})"); - lastUpdate = DateTime.UtcNow; + long throughput = (long)Math.Floor(fileSizeCreated / (now - lastUpdate).TotalSeconds); + var time_left = TimeSpan.FromSeconds((input.MaxTotalSize - fileSizeCreated) / throughput); + string time_left_str = time_left.ToString(@"hh\:mm\:ss"); + Console.WriteLine($"Created {filesCreated} of {files.Count} files ({SizeToHumanReadable(fileSizeCreated)} of {SizeToHumanReadable(totalSize)}) ({SizeToHumanReadable(throughput)}/s) - ETA: {time_left_str}"); + lastUpdate = now; } } catch (Exception ex) From 057c698fa3de89484b87a2a00310f6b1e93ba757 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 18 Feb 2025 10:28:59 +0100 Subject: [PATCH 191/362] Throughput times used were incorrect --- Tools/TestDataGenerator/Commands/Create.cs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Tools/TestDataGenerator/Commands/Create.cs b/Tools/TestDataGenerator/Commands/Create.cs index 21ca748cf..fb9591bd4 100644 --- a/Tools/TestDataGenerator/Commands/Create.cs +++ b/Tools/TestDataGenerator/Commands/Create.cs @@ -175,6 +175,7 @@ public static class Create var filesCreated = 0L; var fileSizeCreated = 0L; + var generateStart = DateTime.UtcNow; foreach (var (file, size) in filesWithSizes) { @@ -196,14 +197,14 @@ public static class Create filesCreated++; fileSizeCreated += size; - var now = DateTime.UtcNow; - if ((now - lastUpdate) > updateInterval) + var generateNow = DateTime.UtcNow; + if ((generateNow - lastUpdate) > updateInterval) { - long throughput = (long)Math.Floor(fileSizeCreated / (now - lastUpdate).TotalSeconds); - var time_left = TimeSpan.FromSeconds((input.MaxTotalSize - fileSizeCreated) / throughput); - string time_left_str = time_left.ToString(@"hh\:mm\:ss"); - Console.WriteLine($"Created {filesCreated} of {files.Count} files ({SizeToHumanReadable(fileSizeCreated)} of {SizeToHumanReadable(totalSize)}) ({SizeToHumanReadable(throughput)}/s) - ETA: {time_left_str}"); - lastUpdate = now; + long throughput = (long)Math.Floor(fileSizeCreated / (generateNow - generateStart).TotalSeconds); + var timeLeft = TimeSpan.FromSeconds((input.MaxTotalSize - fileSizeCreated) / throughput); + string timeLeftString = timeLeft.ToString(@"hh\:mm\:ss"); + Console.WriteLine($"Created {filesCreated} of {files.Count} files ({SizeToHumanReadable(fileSizeCreated)} of {SizeToHumanReadable(totalSize)}) ({SizeToHumanReadable(throughput)}/s) - ETA: {timeLeftString}"); + lastUpdate = generateNow; } } catch (Exception ex) From ca1c765b610079f8224322e22c78941475c58698 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 18 Feb 2025 11:04:25 +0100 Subject: [PATCH 192/362] Fixed developer setup for ngclient on Windows. This fixes two issues that prevented the debug loader to auto-fetch and set up ngclient on Windows when running debug builds. --- .../Middlewares/StaticFilesMiddleware.Debug.cs | 13 ++++++++++++- .../Middlewares/StaticFilesMiddleware.cs | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Duplicati/WebserverCore/Middlewares/StaticFilesMiddleware.Debug.cs b/Duplicati/WebserverCore/Middlewares/StaticFilesMiddleware.Debug.cs index 57dc64693..dab8f0c08 100644 --- a/Duplicati/WebserverCore/Middlewares/StaticFilesMiddleware.Debug.cs +++ b/Duplicati/WebserverCore/Middlewares/StaticFilesMiddleware.Debug.cs @@ -41,6 +41,17 @@ internal static class NpmSpaHelper { var tempPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + // Handle move not working across drive boundaries on Windows + if (OperatingSystem.IsWindows() && Path.GetPathRoot(tempPath) != Path.GetPathRoot(targetPath)) + { + tempPath = Path.GetFullPath(targetPath); + if (targetPath.EndsWith(Path.DirectorySeparatorChar)) + tempPath = tempPath[..-1]; + tempPath += "-tmp"; + if (Directory.Exists(tempPath)) + Directory.Delete(tempPath, true); + } + try { // Download and extract the package @@ -156,7 +167,7 @@ internal static class NpmSpaHelper return null; // Package is not installed, install it - var packageFolder = Path.Combine(basepath, "node_modules", packageId); + var packageFolder = Path.GetFullPath(Path.Combine(basepath, "node_modules", packageId)); if (!Directory.Exists(packageFolder)) return InstallNpmPackage(packageUrl, packageFolder); diff --git a/Duplicati/WebserverCore/Middlewares/StaticFilesMiddleware.cs b/Duplicati/WebserverCore/Middlewares/StaticFilesMiddleware.cs index db69609da..7c6119076 100644 --- a/Duplicati/WebserverCore/Middlewares/StaticFilesMiddleware.cs +++ b/Duplicati/WebserverCore/Middlewares/StaticFilesMiddleware.cs @@ -143,7 +143,7 @@ public static class StaticFilesExtensions { // Serve the static file var file = new FileInfo(Path.Combine(spaConfig.BasePath, path.Value.Substring(spaConfig.Prefix.Length).TrimStart('/'))); - if (file.FullName.StartsWith(spaConfig.BasePath) && file.Exists && fileTypeMappings.Mappings.TryGetValue(Path.GetExtension(file.Extension), out var contentType)) + if (file.FullName.StartsWith(spaConfig.BasePath, Library.Utility.Utility.ClientFilenameStringComparison) && file.Exists && fileTypeMappings.Mappings.TryGetValue(Path.GetExtension(file.Extension), out var contentType)) { context.Response.ContentType = contentType; context.Response.StatusCode = 200; From 12d49f6338c2aac1efa60ae3dd3c90bf491f49c0 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 18 Feb 2025 11:07:39 +0100 Subject: [PATCH 193/362] Version bump for ngclient --- Duplicati/Server/webroot/ngclient/package-lock.json | 8 ++++---- Duplicati/Server/webroot/ngclient/package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Duplicati/Server/webroot/ngclient/package-lock.json b/Duplicati/Server/webroot/ngclient/package-lock.json index 78716466e..8324e94cb 100644 --- a/Duplicati/Server/webroot/ngclient/package-lock.json +++ b/Duplicati/Server/webroot/ngclient/package-lock.json @@ -9,13 +9,13 @@ "version": "1.0.0", "license": "MIT", "dependencies": { - "@duplicati/ngclient": "^0.0.80" + "@duplicati/ngclient": "^0.0.83" } }, "node_modules/@duplicati/ngclient": { - "version": "0.0.80", - "resolved": "https://registry.npmjs.org/@duplicati/ngclient/-/ngclient-0.0.80.tgz", - "integrity": "sha512-x8n8q3iFy9uh5tixtdn80M+95E1P81D0WKuCGnIv4mL1xlOKyod8vngfmabQrm8Vd2YcgB653sUc5gYxTKdtVA==", + "version": "0.0.83", + "resolved": "https://registry.npmjs.org/@duplicati/ngclient/-/ngclient-0.0.83.tgz", + "integrity": "sha512-/kvFgch7Y60HhFPCMZxT20GebVggBG98/PpTJ3R5xr4cQVeOtkJgZb9Yds4LNiIBhiNBQvk93CN0FrhEDtzB6w==", "license": "MIT" } } diff --git a/Duplicati/Server/webroot/ngclient/package.json b/Duplicati/Server/webroot/ngclient/package.json index 5c5f364cf..59c9cf022 100644 --- a/Duplicati/Server/webroot/ngclient/package.json +++ b/Duplicati/Server/webroot/ngclient/package.json @@ -4,6 +4,6 @@ "description": "Modern Angular Client for Duplicati", "license": "MIT", "dependencies": { - "@duplicati/ngclient": "^0.0.80" + "@duplicati/ngclient": "^0.0.83" } } From f61a1d0ad8bc398687e8904154a8d86841261cdc Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 18 Feb 2025 15:13:46 +0100 Subject: [PATCH 194/362] Fixed Windows VSS support --- Duplicati/Library/Snapshots/WindowsSnapshot.cs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Snapshots/WindowsSnapshot.cs b/Duplicati/Library/Snapshots/WindowsSnapshot.cs index 876554595..07de392c5 100644 --- a/Duplicati/Library/Snapshots/WindowsSnapshot.cs +++ b/Duplicati/Library/Snapshots/WindowsSnapshot.cs @@ -55,6 +55,11 @@ namespace Duplicati.Library.Snapshots /// private readonly VssBackupComponents _vssBackupComponents; + /// + /// The source folders included in the snapshot + /// + private readonly IReadOnlyList _sourceFolders; + /// /// Constructs a new backup snapshot, using all the required disks /// @@ -65,7 +70,7 @@ namespace Duplicati.Library.Snapshots : base(followSymlinks) { // For Windows, ensure we don't store paths with extended device path prefixes (i.e., @"\\?\" or @"\\?\UNC\") - sources = sources.Select(SystemIOWindows.RemoveExtendedDevicePathPrefix); + _sourceFolders = sources.Select(SystemIOWindows.RemoveExtendedDevicePathPrefix).ToList(); try { _vssBackupComponents = new VssBackupComponents(); @@ -82,7 +87,7 @@ namespace Duplicati.Library.Snapshots } _vssBackupComponents.SetupWriters(null, excludedWriters); - _vssBackupComponents.InitShadowVolumes(sources); + _vssBackupComponents.InitShadowVolumes(_sourceFolders); _vssBackupComponents.MapVolumesToSnapShots(); @@ -122,7 +127,7 @@ namespace Duplicati.Library.Snapshots /// The source files and folders public override IEnumerable EnumerateFilesystemEntries() { - foreach (var folder in _vssBackupComponents.SnapshotLocalPaths) + foreach (var folder in _sourceFolders) { if (folder.EndsWith(Path.DirectorySeparatorChar) || DirectoryExists(folder)) yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(folder), true, true); From 12d0c9710d038062c4905e7f2e6426403b302a93 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 18 Feb 2025 17:17:16 +0100 Subject: [PATCH 195/362] Added additional fix for VSS --- Duplicati/Library/Snapshots/WindowsSnapshot.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Snapshots/WindowsSnapshot.cs b/Duplicati/Library/Snapshots/WindowsSnapshot.cs index 07de392c5..8d14674ec 100644 --- a/Duplicati/Library/Snapshots/WindowsSnapshot.cs +++ b/Duplicati/Library/Snapshots/WindowsSnapshot.cs @@ -119,7 +119,7 @@ namespace Duplicati.Library.Snapshots /// /// Gets the source folders /// - public override IEnumerable SourceFolders => _vssBackupComponents.SnapshotLocalPaths; + public override IEnumerable SourceFolders => _sourceFolders; /// /// Enumerates the root source files and folders From def60a7e709158337a5d742dd66f148dd30a2fee Mon Sep 17 00:00:00 2001 From: "Marcelo C." Date: Tue, 18 Feb 2025 15:34:22 -0300 Subject: [PATCH 196/362] Adding B2 to CI - Added B2 Test - Updated ReadMe with backends status and variables --- .github/workflows/backendtests.yml | 60 +++++++++++++++- .../Duplicati.Backend.Tests/B2/B2Tests.cs | 46 +++++++++++++ LiveTests/Duplicati.Backend.Tests/README.md | 68 +++++++++++-------- 3 files changed, 145 insertions(+), 29 deletions(-) create mode 100644 LiveTests/Duplicati.Backend.Tests/B2/B2Tests.cs diff --git a/.github/workflows/backendtests.yml b/.github/workflows/backendtests.yml index 9d84de249..9b3148002 100644 --- a/.github/workflows/backendtests.yml +++ b/.github/workflows/backendtests.yml @@ -13,6 +13,7 @@ jobs: testcontainers_secrets_available: "${{ steps.check_testcontainers.outputs.available }}" google_secrets_available: "${{ steps.check_google.outputs.available }}" pcloud_secrets_available: "${{ steps.check_pcloud.outputs.available }}" + b2_secrets_available: "${{ steps.check_b2.outputs.available }}" steps: - id: check_pcloud name: Check pCloud secrets @@ -103,6 +104,22 @@ jobs: fi echo "Google check completed" + - id: check_b2 + name: Check B2 secrets + shell: bash + run: | + echo "Starting B2 secrets check..." + if [[ -n "${{ secrets.TESTCREDENTIAL_B2_BUCKET }}" ]] && \ + [[ -n "${{ secrets.TESTCREDENTIAL_B2_FOLDER }}" ]] && \ + [[ -n "${{ secrets.TESTCREDENTIAL_B2_USERNAME }}" ]] && \ + [[ -n "${{ secrets.TESTCREDENTIAL_B2_PASSWORD }}" ]]; then + echo "All B2 secrets found" + echo "available=true" >> $GITHUB_OUTPUT + else + echo "Missing some B2 secrets" + echo "available=false" >> $GITHUB_OUTPUT + fi + echo "B2 check completed" - name: Debug Output shell: bash run: | @@ -110,6 +127,9 @@ jobs: echo "Dropbox Available: ${{ steps.check_dropbox.outputs.available }}" echo "TestContainers Available: ${{ steps.check_testcontainers.outputs.available }}" echo "Google Available: ${{ steps.check_google.outputs.available }}" + echo "pCloud Available: ${{ steps.check_pcloud.outputs.available }}" + echo "Azure Available: ${{ steps.check_azure.outputs.available }}" + echo "B2 Available: ${{ steps.check_b2.outputs.available }}" test_ftp: needs: check_secrets if: needs.check_secrets.outputs.testcontainers_secrets_available == 'true' @@ -446,4 +466,42 @@ jobs: run: >- dotnet test --no-build --filter="ClassName~CIFSTests" --logger:"console;verbosity=detailed" - LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln \ No newline at end of file + LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln + test_b2: + needs: check_secrets + if: needs.check_secrets.outputs.b2_secrets_available == 'true' + name: Backblaze B2 Tests + runs-on: "${{ matrix.os }}" + strategy: + max-parallel: 1 + fail-fast: false + matrix: + os: + - ubuntu-latest + - windows-latest + - macos-latest + steps: + - name: Set up .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 8.x + - name: Checkout source + uses: actions/checkout@v4 + - name: Restore NuGet dependencies + run: >- + dotnet restore + LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln + - name: Build project + run: >- + dotnet build --no-restore + LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln + - name: Run B2 tests + env: + TESTCREDENTIAL_B2_BUCKET: "${{ secrets.TESTCREDENTIAL_B2_BUCKET }}" + TESTCREDENTIAL_B2_FOLDER: "${{ secrets.TESTCREDENTIAL_B2_FOLDER }}" + TESTCREDENTIAL_B2_USERNAME: "${{ secrets.TESTCREDENTIAL_B2_USERNAME }}" + TESTCREDENTIAL_B2_PASSWORD: "${{ secrets.TESTCREDENTIAL_B2_PASSWORD }}" + run: >- + dotnet test --no-build --filter="ClassName~B2Tests" + --logger:"console;verbosity=detailed" + LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln \ No newline at end of file diff --git a/LiveTests/Duplicati.Backend.Tests/B2/B2Tests.cs b/LiveTests/Duplicati.Backend.Tests/B2/B2Tests.cs new file mode 100644 index 000000000..d45bf2774 --- /dev/null +++ b/LiveTests/Duplicati.Backend.Tests/B2/B2Tests.cs @@ -0,0 +1,46 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +namespace Duplicati.Backend.Tests.B2; + +/// +/// Backblaze B2 Tests +/// +[TestClass] +public sealed class B2Tests : BaseTest +{ + [TestMethod] + public Task TestB2() + { + CheckRequiredEnvironment(["TESTCREDENTIAL_B2_BUCKET", "TESTCREDENTIAL_B2_FOLDER", "TESTCREDENTIAL_B2_USERNAME", "TESTCREDENTIAL_B2_PASSWORD"]); + + var exitCode = CommandLine.BackendTester.Program.Main( + new[] + { + $"b2://{Environment.GetEnvironmentVariable("TESTCREDENTIAL_B2_BUCKET")}/{Environment.GetEnvironmentVariable("TESTCREDENTIAL_B2_FOLDER")}?auth-username={Uri.EscapeDataString(Environment.GetEnvironmentVariable("TESTCREDENTIAL_B2_USERNAME")!)}&auth-password={Uri.EscapeDataString(Environment.GetEnvironmentVariable("TESTCREDENTIAL_B2_PASSWORD")!)}", + + }.Concat(Parameters.GlobalTestParameters).ToArray()); + + if (exitCode != 0) Assert.Fail("BackendTester is returning non-zero exit code, check logs for details"); + + return Task.CompletedTask; + } +} \ No newline at end of file diff --git a/LiveTests/Duplicati.Backend.Tests/README.md b/LiveTests/Duplicati.Backend.Tests/README.md index 4b1d21ed1..8427d1ffd 100644 --- a/LiveTests/Duplicati.Backend.Tests/README.md +++ b/LiveTests/Duplicati.Backend.Tests/README.md @@ -15,34 +15,34 @@ NUMBER_OF_FILES default is 20 ## Backends CI Status | Backend | CI Planned | CI Status | -|----------|------------|------------| -| pCloud | Planned | Live | -| WebDav | Planned | Live | -| Shhv2 | Planned | Live | -| S3 | Planned | Live | -| Ftp | Planned | Live | -| Planned | Planned | Live | -| Dropbox | Planned | Live | -| AlternativeFTP | Planned | Live | -| AzureBlob | Planned | Live | -| CIFS | Planned | Live | -| Google Drive | Planned | Live | -| TahoeLAFS | Pending Decision | | -| TescentCOS | Pending Decision | | -| Storj | Pending Decision | | -| Sia | Pending Decision | | -| SharePoint | Pending Decision | | -| Rclone | Pending Decision | | -| OneDrive | Pending Decision | | -| OpenStack | Pending Decision | | -| Mega | Pending Decision | | -| Jottacloud | Pending Decision | | -| IDrive2 | Planned | | -| File | Planned | | -| Box | Pending Decision | | -| CloudFiles | Pending Decision | | -| backBlaze | Planned | | -| AliyunOSS | Pending Decision | | +|----------|------------|-----------| +| pCloud | Planned | Live | +| WebDav | Planned | Live | +| Shhv2 | Planned | Live | +| S3 | Planned | Live | +| Ftp | Planned | Live | +| Planned | Planned | Live | +| Dropbox | Planned | Live | +| AlternativeFTP | Planned | Live | +| AzureBlob | Planned | Live | +| CIFS | Planned | Live | +| Google Drive | Planned | Live | +| TahoeLAFS | Pending Decision | | +| TescentCOS | Pending Decision | | +| Storj | Pending Decision | | +| Sia | Pending Decision | | +| SharePoint | Pending Decision | | +| Rclone | Pending Decision | | +| OneDrive | Pending Decision | | +| OpenStack | Pending Decision | | +| Mega | Pending Decision | | +| Jottacloud | Pending Decision | | +| IDrive2 | Planned | | +| File | Planned | | +| Box | Pending Decision | | +| CloudFiles | Pending Decision | | +| backBlaze | Planned | Live | +| AliyunOSS | Pending Decision | | ## Backends that do not require Environment variables @@ -56,6 +56,17 @@ Please note that TestContainers token has to be configured in secrets/Github act ## Backends that require Environment variables +## Backblaze B2 + +Backblaze B2 credentials are mapped to the following environment variables: + +``` +TESTCREDENTIAL_B2_BUCKET +TESTCREDENTIAL_B2_FOLDER +TESTCREDENTIAL_B2_USERNAME +TESTCREDENTIAL_B2_PASSWORD +``` + ## Google Drive: Google Drive credentials are mapped to the following environment variables: @@ -64,6 +75,7 @@ Google Drive credentials are mapped to the following environment variables: TESTCREDENTIAL_GOOGLEDRIVE_FOLDER TESTCREDENTIAL_GOOGLEDRIVE_TOKEN ``` + ## S3 S3 credentials are mapped to the following environment variables: From 73d0ad905f099854a9b1722dd4a3e200dd47bf5f Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 18 Feb 2025 22:12:37 +0100 Subject: [PATCH 197/362] Fixed USN support --- .../Backup/FileEnumerationProcess.cs | 53 ++-- .../Snapshots/RecursiveFilterHelper.cs | 229 --------------- .../Library/Snapshots/UsnJournalService.cs | 278 +++++++++++++++++- 3 files changed, 307 insertions(+), 253 deletions(-) delete mode 100644 Duplicati/Library/Snapshots/RecursiveFilterHelper.cs diff --git a/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs b/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs index a1ef2ec65..a7db2b2ce 100644 --- a/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs @@ -33,6 +33,7 @@ using Duplicati.Library.Interface; using System.Runtime.CompilerServices; using Duplicati.Library.Snapshots; using Duplicati.Library.SourceProvider; +using Duplicati.Library.Common.IO; namespace Duplicati.Library.Main.Operation.Backup { @@ -96,22 +97,30 @@ namespace Duplicati.Library.Main.Operation.Backup if (ignorenames != null && ignorenames.Length == 0) ignorenames = null; - // If we have a specific list, use that instead of enumerating the filesystem - async IAsyncEnumerable ExpandSources(IEnumerable list) - { - foreach (var s in list) - { - var r = await sourceProvider.GetEntry(s, false, token).ConfigureAwait(false); - if (r != null) - yield return r; - } - } + // Shared filter function with bound variables + ValueTask FilterEntry(ISourceFileEntry entry) + => SourceFileEntryFilter(entry, blacklistPaths, hardlinkPolicy, symlinkPolicy, hardlinkmap, fileAttributeFilter, enumeratefilter, ignorenames, mixinqueue, token); + + // Prepare the work list IAsyncEnumerable worklist; + + // If we have a specific list, use that instead of enumerating the filesystem if (changedfilelist != null && changedfilelist.Any()) { - worklist = ExpandSources(changedfilelist).WhereAwait(x => - SourceFileEntryFilter(x, blacklistPaths, hardlinkPolicy, symlinkPolicy, hardlinkmap, fileAttributeFilter, enumeratefilter, ignorenames, mixinqueue, token)); + async IAsyncEnumerable ExpandSources(IEnumerable list) + { + foreach (var s in list) + { + var r = await sourceProvider.GetEntry(s, s.EndsWith(Path.DirectorySeparatorChar), token).ConfigureAwait(false); + if (r != null) + { + //TODO: Set r.IsRoot = true for source elements + yield return r; + } + } + } + worklist = ExpandSources(changedfilelist).WhereAwait(FilterEntry); } else if (journalService != null) { @@ -127,17 +136,27 @@ namespace Duplicati.Library.Main.Operation.Backup if (fileProviders.Count > 1) throw new InvalidOperationException("Multiple file providers found, but USN only supports one"); - worklist = RecursiveFilterHelper.GetModifiedSources( - ExpandSources(journalService.VolumeDataList.SelectMany(x => x.Folders.Concat(x.Files))), - fileProviders.First().SnapshotService, - x => SourceFileEntryFilter(x, blacklistPaths, hardlinkPolicy, symlinkPolicy, hardlinkmap, fileAttributeFilter, enumeratefilter, ignorenames, mixinqueue, token), + // TODO: This is not as effecient as possible. + // If the root folder is marked changed by USN, the expansion with RecurseEntries + // will cause a full regular scan. It should be possible to *only* process the + // changed elements as returned from the USN journal. + // It should be possible to remove RecurseEntries from the GetModifiedSources() + // enumeration result. + // Such a change requires significant testing as there are many pitfalls with USN. + worklist = RecurseEntries(journalService.GetModifiedSources(FilterEntry, token), + FilterEntry, token + ) + .Concat( + RecurseEntries(journalService.GetFullScanSources(token), + FilterEntry, + token) ); } else { worklist = RecurseEntries(sourceProvider.Enumerate(token), - x => SourceFileEntryFilter(x, blacklistPaths, hardlinkPolicy, symlinkPolicy, hardlinkmap, fileAttributeFilter, enumeratefilter, ignorenames, mixinqueue, token), + FilterEntry, token ); } diff --git a/Duplicati/Library/Snapshots/RecursiveFilterHelper.cs b/Duplicati/Library/Snapshots/RecursiveFilterHelper.cs deleted file mode 100644 index 1e8fb9f02..000000000 --- a/Duplicati/Library/Snapshots/RecursiveFilterHelper.cs +++ /dev/null @@ -1,229 +0,0 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - -#nullable enable - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using Duplicati.Library.Interface; -using System.Threading.Tasks; -using System.Runtime.CompilerServices; - -namespace Duplicati.Library.Snapshots; - -public static class RecursiveFilterHelper -{ - /// - /// The log tag to use - /// - private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(RecursiveFilterHelper)); - - /// - /// Filters sources, returning sub-set having been modified since last - /// change, as specified by journalData. - /// - /// List of sources - /// Snapshot service - /// Filter callback to exclude filtered items - /// Cancellation token - /// Filtered sources - public static async IAsyncEnumerable GetModifiedSources(IAsyncEnumerable entries, ISnapshotService snapshot, Func> filter, [EnumeratorCancellation] CancellationToken token) - { - // prepare cache for includes (value = true) and excludes (value = false, will be populated - // on-demand) - var cache = new Dictionary(); - await foreach (var source in entries) - { - if (token.IsCancellationRequested) - break; - - cache[source.Path] = true; - } - - // Check the simplified folders, and their parent folders against the exclusion filter. - // This is needed because the filter may exclude "C:\A\", but this won't match the more - // specific "C:\A\B\" in our list, even though it's meant to be excluded. - // The reason why the filter doesn't exclude it is because during a regular (non-USN) full scan, - // FilterHandler.EnumerateFilesAndFolders() works top-down, and won't even enumerate child - // folders. - // The sources are needed to stop evaluating parent folders above the specified source folders - await foreach (var folder in FilterExcludedFolders(entries.Where(x => x.IsFolder), snapshot, filter, cache, token)) - { - if (token.IsCancellationRequested) - break; - - if (!snapshot.DirectoryExists(folder.Path)) - continue; - - yield return folder; - } - - // The simplified file list also needs to be checked against the exclusion filter, as it - // may contain entries excluded due to attributes, but also because they are below excluded - // folders, which themselves aren't in the folder list from step 1. - // Note that the simplified file list may contain entries that have been deleted! They need to - // be kept in the list (unless excluded by the filter) in order for the backup handler to record their - // deletion. - await foreach (var files in FilterExcludedFiles(entries.Where(x => !x.IsFolder), snapshot, filter, cache, token)) - { - if (token.IsCancellationRequested) - break; - - if (!snapshot.FileExists(files.Path)) - continue; - - yield return files; - } - } - - /// - /// Filter supplied files, removing any files which itself, or one - /// of its parent folders, is excluded by the filter. - /// - /// Files to filter - /// Snapshot service - /// Exclusion filter - /// Cache of included and excluded files / folders - /// Cancellation token - /// Filtered files - private static async IAsyncEnumerable FilterExcludedFiles( - IAsyncEnumerable files, - ISnapshotService snapshot, - Func> filter, - IDictionary cache, - [EnumeratorCancellation] CancellationToken token) - { - await foreach (var file in files.WithCancellation(token).ConfigureAwait(false)) - { - try - { - if (!await filter(file)) - continue; - - var parentPath = Utility.Utility.GetParent(file.Path, true); - if (!string.IsNullOrWhiteSpace(parentPath)) - { - var parent = snapshot.GetFilesystemEntry(parentPath, true); - if (await IsFolderOrAncestorsExcluded(parent, snapshot, filter, cache, token)) - continue; - } - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "FilterExcludedFilesError", ex, "Error while filtering file: {0}", ex.Message); - continue; - } - - yield return file; - } - } - - /// - /// Filter supplied folders, removing any folder which itself, or one - /// of its ancestors, is excluded by the filter. - /// - /// Folder to filter - /// Snapshot service - /// Exclusion filter - /// Cache of excluded folders - /// Cancellation token - /// Filtered folders - private static async IAsyncEnumerable FilterExcludedFolders( - IAsyncEnumerable folders, - ISnapshotService snapshot, - Func> filter, - IDictionary cache, - [EnumeratorCancellation] CancellationToken token) - { - await foreach (var folder in folders.WithCancellation(token).ConfigureAwait(false)) - { - try - { - if (await IsFolderOrAncestorsExcluded(folder, snapshot, filter, cache, token)) - continue; - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "FilterExcludedFoldersError", ex, "Error while filtering folder: {0}", ex.Message); - continue; - } - - yield return folder; - } - } - - /// - /// Tests if specified folder, or any of its ancestors, is excluded by the filter - /// - /// Folder to test - /// Filter - /// Cache of excluded folders (optional) - /// True if excluded, false otherwise - private static async ValueTask IsFolderOrAncestorsExcluded( - ISourceFileEntry folder, - ISnapshotService snapshot, - Func> filter, - IDictionary cache, - CancellationToken token) - { - List? parents = null; - while (folder != null) - { - if (token.IsCancellationRequested) - break; - - // first check cache - if (cache.TryGetValue(folder.Path, out var include)) - { - if (include) - return false; - - break; // hit! - } - - parents ??= []; // create on-demand - - // remember folder for cache - parents.Add(folder.Path); - - - if (!await filter(folder)) - break; // excluded - - var parentPath = Utility.Utility.GetParent(folder.Path, true); - if (string.IsNullOrWhiteSpace(parentPath)) - break; - - folder = snapshot.GetFilesystemEntry(parentPath, true); - } - - if (folder != null) - { - // update cache - parents?.ForEach(p => cache[p] = false); - } - - return folder != null; - } -} - diff --git a/Duplicati/Library/Snapshots/UsnJournalService.cs b/Duplicati/Library/Snapshots/UsnJournalService.cs index 47cd08baa..09583cef8 100644 --- a/Duplicati/Library/Snapshots/UsnJournalService.cs +++ b/Duplicati/Library/Snapshots/UsnJournalService.cs @@ -19,6 +19,7 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +#nullable enable using System; using System.Collections.Generic; @@ -29,6 +30,8 @@ using System.Threading; using Duplicati.Library.Interface; using Duplicati.Library.Common.IO; using Duplicati.Library.Utility; +using System.Threading.Tasks; +using System.Runtime.CompilerServices; namespace Duplicati.Library.Snapshots { @@ -70,6 +73,9 @@ namespace Duplicati.Library.Snapshots m_token = token; } + /// + /// The volume data lists for the mapped root drives + /// public IEnumerable VolumeDataList => m_volumeDataDict.Select(e => e.Value); /// @@ -90,7 +96,7 @@ namespace Duplicati.Library.Snapshots // get hash identifying current source filter / sources configuration var configHash = Utility.Utility.ByteArrayAsHexString(MD5HashHelper.GetHash(new string[] { - emitFilter == null ? string.Empty : emitFilter.ToString(), + emitFilter?.ToString() ?? string.Empty, string.Join("; ", m_snapshot.SourceFolders), fileAttributeFilter.ToString(), skipFilesLargerThan.ToString() @@ -218,6 +224,261 @@ namespace Duplicati.Library.Snapshots return result; } + /// + /// Returns all sources that should be fully scanned + /// + /// Cancellation token + /// Filtered sources + public async IAsyncEnumerable GetFullScanSources([EnumeratorCancellation] CancellationToken token) + { + // Make the method async enumerable + await Task.CompletedTask; + + foreach(var volumeData in m_volumeDataDict.Values.Where(x => x.IsFullScan)) + { + if (volumeData.Folders != null) + { + foreach(var folderPath in volumeData.Folders) + { + if (token.IsCancellationRequested) + yield break; + + var folder = m_snapshot.GetFilesystemEntry(folderPath, true); + if (folder != null) + yield return folder; + else + Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "FolderLookupError", null, "No entry while locating source folder {0}", folderPath); + } + } + + if (volumeData.Files != null) + { + foreach(var filePath in volumeData.Files) + { + if (token.IsCancellationRequested) + yield break; + + var file = m_snapshot.GetFilesystemEntry(filePath, false); + if (file != null) + yield return file; + else + Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "FolderLookupError", null, "No entry while locating source file {0}", filePath); + } + } + } + } + + + /// + /// Filters sources, returning sub-set having been modified since last + /// change, as specified by journalData. + /// + /// Filter callback to exclude filtered items + /// Cancellation token + /// Filtered sources + public async IAsyncEnumerable GetModifiedSources(Func> filter, [EnumeratorCancellation] CancellationToken token) + { + foreach(var volumeData in m_volumeDataDict.Values.Where(x => !x.IsFullScan)) + { + // prepare cache for includes (value = true) and excludes (value = false, will be populated + // on-demand) + var cache = new Dictionary(); + foreach (var source in m_snapshot.SourceFolders) + { + if (token.IsCancellationRequested) + yield break; + + cache[source] = true; + } + + // Check the simplified folders, and their parent folders against the exclusion filter. + // This is needed because the filter may exclude "C:\A\", but this won't match the more + // specific "C:\A\B\" in our list, even though it's meant to be excluded. + // The reason why the filter doesn't exclude it is because during a regular (non-USN) full scan, + // FilterHandler.EnumerateFilesAndFolders() works top-down, and won't even enumerate child + // folders. + // The sources are needed to stop evaluating parent folders above the specified source folders + await foreach (var folder in FilterExcludedFolders(volumeData.Folders, m_snapshot, filter, cache, token).ConfigureAwait(false)) + { + if (token.IsCancellationRequested) + yield break; + + if (!m_snapshot.DirectoryExists(folder.Path)) + continue; + + yield return folder; + } + + // The simplified file list also needs to be checked against the exclusion filter, as it + // may contain entries excluded due to attributes, but also because they are below excluded + // folders, which themselves aren't in the folder list from step 1. + // Note that the simplified file list may contain entries that have been deleted! They need to + // be kept in the list (unless excluded by the filter) in order for the backup handler to record their + // deletion. + await foreach (var files in FilterExcludedFiles(volumeData.Files, m_snapshot, filter, cache, token).ConfigureAwait(false)) + { + if (token.IsCancellationRequested) + yield break; + + if (!m_snapshot.FileExists(files.Path)) + continue; + + yield return files; + } + } + } + + /// + /// Filter supplied files, removing any files which itself, or one + /// of its parent folders, is excluded by the filter. + /// + /// Files to filter + /// Snapshot service + /// Exclusion filter + /// Cache of included and excluded files / folders + /// Cancellation token + /// Filtered files + private static async IAsyncEnumerable FilterExcludedFiles( + IEnumerable? files, + ISnapshotService snapshot, + Func> filter, + IDictionary cache, + [EnumeratorCancellation] CancellationToken token) + { + if (files == null) + yield break; + + foreach (var filePath in files) + { + ISourceFileEntry? file; + try + { + file = snapshot.GetFilesystemEntry(filePath, false); + if (file == null) + continue; + + if (!await filter(file)) + continue; + + var parentPath = Utility.Utility.GetParent(file.Path, true); + if (!string.IsNullOrWhiteSpace(parentPath)) + { + var parent = snapshot.GetFilesystemEntry(parentPath, true); + if (parent == null) + continue; + + if (await IsFolderOrAncestorsExcluded(parent, snapshot, filter, cache, token)) + continue; + } + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "FilterExcludedFilesError", ex, "Error while filtering file: {0}", ex.Message); + continue; + } + + if (file != null) + yield return file; + } + } + + /// + /// Filter supplied folders, removing any folder which itself, or one + /// of its ancestors, is excluded by the filter. + /// + /// Folder to filter + /// Snapshot service + /// Exclusion filter + /// Cache of excluded folders + /// Cancellation token + /// Filtered folders + private static async IAsyncEnumerable FilterExcludedFolders( + IEnumerable? folders, + ISnapshotService snapshot, + Func> filter, + IDictionary cache, + [EnumeratorCancellation] CancellationToken token) + { + if (folders == null) + yield break; + + foreach (var folderPath in folders) + { + ISourceFileEntry? folder; + try + { + folder = snapshot.GetFilesystemEntry(folderPath, true); + if (folder == null) + continue; + if (await IsFolderOrAncestorsExcluded(folder, snapshot, filter, cache, token)) + continue; + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "FilterExcludedFoldersError", ex, "Error while filtering folder: {0}", ex.Message); + continue; + } + + yield return folder; + } + } + + /// + /// Tests if specified folder, or any of its ancestors, is excluded by the filter + /// + /// Folder to test + /// Filter + /// Cache of excluded folders (optional) + /// True if excluded, false otherwise + private static async ValueTask IsFolderOrAncestorsExcluded( + ISourceFileEntry inputFolder, + ISnapshotService snapshot, + Func> filter, + IDictionary cache, + CancellationToken token) + { + List? parents = null; + ISourceFileEntry? folder = inputFolder; + + while (folder != null) + { + if (token.IsCancellationRequested) + break; + + // first check cache + if (cache.TryGetValue(folder.Path, out var include)) + { + if (include) + return false; + + break; // hit! + } + + parents ??= []; // create on-demand + + // remember folder for cache + parents.Add(folder.Path); + + + if (!await filter(folder)) + break; // excluded + + var parentPath = Utility.Utility.GetParent(folder.Path, true); + if (string.IsNullOrWhiteSpace(parentPath)) + break; + + folder = snapshot.GetFilesystemEntry(parentPath, true); + } + + if (folder != null) + { + // update cache + parents?.ForEach(p => cache[p] = false); + } + + return folder != null; + } + /// /// Sort sources by root volume /// @@ -260,9 +521,12 @@ namespace Duplicati.Library.Snapshots if (volumeData.IsFullScan) return true; // do not append from previous set, already scanned - if (volumeData.Files.Contains(path)) + if (volumeData.Files != null && volumeData.Files.Contains(path)) return true; // do not append from previous set, already scanned + if (volumeData.Folders == null) + return false; + foreach (var folder in volumeData.Folders) { if (m_token.IsCancellationRequested) @@ -289,23 +553,23 @@ namespace Duplicati.Library.Snapshots /// /// Volume /// - public string Volume { get; set; } + public required string Volume { get; set; } /// /// Set of potentially modified files /// - public HashSet Files { get; internal set; } + public HashSet? Files { get; internal set; } /// /// Set of folders that are potentially modified, or whose children /// are potentially modified /// - public List Folders { get; internal set; } + public List? Folders { get; internal set; } /// /// Journal data to use for next backup /// - public USNJournalDataEntry JournalData { get; internal set; } + public USNJournalDataEntry? JournalData { get; internal set; } /// /// If true, a full scan for this volume was required @@ -315,6 +579,6 @@ namespace Duplicati.Library.Snapshots /// /// Optional exception message for volume /// - public Exception Exception { get; internal set; } + public Exception? Exception { get; internal set; } } } From ac603a4d8f707cf8079c29eb1522c7cfe2fb9bbc Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 18 Feb 2025 22:32:37 +0100 Subject: [PATCH 198/362] Renamed SourceFolders to SourceEntries as it can contain files as well --- Duplicati/Library/Snapshots/ISnapshotService.cs | 4 ++-- Duplicati/Library/Snapshots/LinuxSnapshot.cs | 10 +++++----- Duplicati/Library/Snapshots/NoSnapshotLinux.cs | 12 ++++++------ Duplicati/Library/Snapshots/NoSnapshotWindows.cs | 14 +++++++------- Duplicati/Library/Snapshots/SnapshotBase.cs | 4 ++-- Duplicati/Library/Snapshots/UsnJournalService.cs | 14 +++++++------- Duplicati/Library/Snapshots/WindowsSnapshot.cs | 10 +++++----- 7 files changed, 34 insertions(+), 34 deletions(-) diff --git a/Duplicati/Library/Snapshots/ISnapshotService.cs b/Duplicati/Library/Snapshots/ISnapshotService.cs index fa95438b8..72542e465 100644 --- a/Duplicati/Library/Snapshots/ISnapshotService.cs +++ b/Duplicati/Library/Snapshots/ISnapshotService.cs @@ -36,9 +36,9 @@ namespace Duplicati.Library.Snapshots public interface ISnapshotService : IDisposable { /// - /// Gets the source folders + /// Gets the source entries /// - IEnumerable SourceFolders { get; } + IEnumerable SourceEntries { get; } /// /// Enumerates the root source files and folders diff --git a/Duplicati/Library/Snapshots/LinuxSnapshot.cs b/Duplicati/Library/Snapshots/LinuxSnapshot.cs index 5df6084d6..05cf884d7 100644 --- a/Duplicati/Library/Snapshots/LinuxSnapshot.cs +++ b/Duplicati/Library/Snapshots/LinuxSnapshot.cs @@ -110,7 +110,7 @@ namespace Duplicati.Library.Snapshots /// /// Gets the source folders /// - public override IEnumerable SourceFolders => m_snapShots.Select(e => e.LocalPath); + public override IEnumerable SourceEntries => m_entries.Select(x => x.Key); /// /// Enumerates the root source files and folders @@ -118,12 +118,12 @@ namespace Duplicati.Library.Snapshots /// The source files and folders public override IEnumerable EnumerateFilesystemEntries() { - foreach (var snap in m_snapShots) + foreach (var sourceEntry in SourceEntries) { - if (DirectoryExists(snap.LocalPath) || snap.LocalPath.EndsWith(System.IO.Path.DirectorySeparatorChar)) - yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(snap.LocalPath), true, true); + if (DirectoryExists(sourceEntry) || sourceEntry.EndsWith(System.IO.Path.DirectorySeparatorChar)) + yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(sourceEntry), true, true); else - yield return new SnapshotSourceFileEntry(this, snap.LocalPath, false, true); + yield return new SnapshotSourceFileEntry(this, sourceEntry, false, true); } } diff --git a/Duplicati/Library/Snapshots/NoSnapshotLinux.cs b/Duplicati/Library/Snapshots/NoSnapshotLinux.cs index 87fff66d1..6aaa1ecd6 100644 --- a/Duplicati/Library/Snapshots/NoSnapshotLinux.cs +++ b/Duplicati/Library/Snapshots/NoSnapshotLinux.cs @@ -69,25 +69,25 @@ namespace Duplicati.Library.Snapshots /// /// The list of folders to create snapshots of /// - private readonly IEnumerable m_folders; + private readonly IEnumerable m_sources; /// /// Initializes a new instance of the class. /// - /// The list of folders to create snapshots of + /// The list of sources to create snapshots of /// A flag indicating if advisory locks should be ignored /// A flag indicating if symlinks should be followed - public NoSnapshotLinux(IEnumerable folders, bool ignoreAdvisoryLocks, bool followSymlinks) + public NoSnapshotLinux(IEnumerable sources, bool ignoreAdvisoryLocks, bool followSymlinks) : base(followSymlinks) { - m_folders = folders; + m_sources = sources; m_ignoreAdvisoryLocks = ignoreAdvisoryLocks; } /// /// Gets the source folders /// - public override IEnumerable SourceFolders => m_folders; + public override IEnumerable SourceEntries => m_sources; /// /// Enumerates the root source files and folders @@ -95,7 +95,7 @@ namespace Duplicati.Library.Snapshots /// The source files and folders public override IEnumerable EnumerateFilesystemEntries() { - foreach (var folder in m_folders) + foreach (var folder in m_sources) { if (DirectoryExists(folder) || folder.EndsWith(Path.DirectorySeparatorChar)) yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(folder), true, true); diff --git a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs index b4a3b16e0..029ae3791 100644 --- a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs +++ b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs @@ -37,17 +37,17 @@ namespace Duplicati.Library.Snapshots /// /// The list of folders to create snapshots of /// - private readonly IEnumerable m_folders; + private readonly IEnumerable m_sources; /// /// Initializes a new instance of the class. /// - /// The list of folders to create snapshots of + /// The list of entries to create snapshots of /// A flag indicating if symlinks should be followed - public NoSnapshotWindows(IEnumerable folders, bool followSymlinks) + public NoSnapshotWindows(IEnumerable sources, bool followSymlinks) : base(followSymlinks) { - m_folders = folders; + m_sources = sources; } /// @@ -77,8 +77,8 @@ namespace Duplicati.Library.Snapshots /// /// Gets the source folders /// - public override IEnumerable SourceFolders - => m_folders; + public override IEnumerable SourceEntries + => m_sources; /// /// Enumerates the root source files and folders @@ -86,7 +86,7 @@ namespace Duplicati.Library.Snapshots /// The source files and folders public override IEnumerable EnumerateFilesystemEntries() { - foreach (var folder in m_folders.Select(SystemIOWindows.RemoveExtendedDevicePathPrefix)) + foreach (var folder in m_sources.Select(SystemIOWindows.RemoveExtendedDevicePathPrefix)) { if (DirectoryExists(folder) || folder.EndsWith(System.IO.Path.DirectorySeparatorChar)) yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(folder), true, true); diff --git a/Duplicati/Library/Snapshots/SnapshotBase.cs b/Duplicati/Library/Snapshots/SnapshotBase.cs index 7fcab2761..1d8f5a253 100644 --- a/Duplicati/Library/Snapshots/SnapshotBase.cs +++ b/Duplicati/Library/Snapshots/SnapshotBase.cs @@ -56,9 +56,9 @@ namespace Duplicati.Library.Snapshots protected bool FollowSymlinks { get; } /// - /// Gets the source folders + /// Gets the source entries /// - public abstract IEnumerable SourceFolders { get; } + public abstract IEnumerable SourceEntries { get; } /// /// Enuemrates the root source files and folders diff --git a/Duplicati/Library/Snapshots/UsnJournalService.cs b/Duplicati/Library/Snapshots/UsnJournalService.cs index 09583cef8..8f1298472 100644 --- a/Duplicati/Library/Snapshots/UsnJournalService.cs +++ b/Duplicati/Library/Snapshots/UsnJournalService.cs @@ -97,7 +97,7 @@ namespace Duplicati.Library.Snapshots // get hash identifying current source filter / sources configuration var configHash = Utility.Utility.ByteArrayAsHexString(MD5HashHelper.GetHash(new string[] { emitFilter?.ToString() ?? string.Empty, - string.Join("; ", m_snapshot.SourceFolders), + string.Join("; ", m_snapshot.SourceEntries), fileAttributeFilter.ToString(), skipFilesLargerThan.ToString() })); @@ -106,7 +106,7 @@ namespace Duplicati.Library.Snapshots var journalDataDict = prevJournalData.ToDictionary(data => data.Volume); // iterate over volumes - foreach (var sourcesPerVolume in SortByVolume(m_snapshot.SourceFolders)) + foreach (var sourcesPerVolume in SortByVolume(m_snapshot.SourceEntries)) { Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "UsnInitialize", "Reading USN journal for volume: {0}", sourcesPerVolume.Key); @@ -234,11 +234,11 @@ namespace Duplicati.Library.Snapshots // Make the method async enumerable await Task.CompletedTask; - foreach(var volumeData in m_volumeDataDict.Values.Where(x => x.IsFullScan)) + foreach (var volumeData in m_volumeDataDict.Values.Where(x => x.IsFullScan)) { if (volumeData.Folders != null) { - foreach(var folderPath in volumeData.Folders) + foreach (var folderPath in volumeData.Folders) { if (token.IsCancellationRequested) yield break; @@ -253,7 +253,7 @@ namespace Duplicati.Library.Snapshots if (volumeData.Files != null) { - foreach(var filePath in volumeData.Files) + foreach (var filePath in volumeData.Files) { if (token.IsCancellationRequested) yield break; @@ -278,12 +278,12 @@ namespace Duplicati.Library.Snapshots /// Filtered sources public async IAsyncEnumerable GetModifiedSources(Func> filter, [EnumeratorCancellation] CancellationToken token) { - foreach(var volumeData in m_volumeDataDict.Values.Where(x => !x.IsFullScan)) + foreach (var volumeData in m_volumeDataDict.Values.Where(x => !x.IsFullScan)) { // prepare cache for includes (value = true) and excludes (value = false, will be populated // on-demand) var cache = new Dictionary(); - foreach (var source in m_snapshot.SourceFolders) + foreach (var source in m_snapshot.SourceEntries) { if (token.IsCancellationRequested) yield break; diff --git a/Duplicati/Library/Snapshots/WindowsSnapshot.cs b/Duplicati/Library/Snapshots/WindowsSnapshot.cs index 8d14674ec..08bd5e1ae 100644 --- a/Duplicati/Library/Snapshots/WindowsSnapshot.cs +++ b/Duplicati/Library/Snapshots/WindowsSnapshot.cs @@ -58,7 +58,7 @@ namespace Duplicati.Library.Snapshots /// /// The source folders included in the snapshot /// - private readonly IReadOnlyList _sourceFolders; + private readonly IReadOnlyList _sourceEntries; /// /// Constructs a new backup snapshot, using all the required disks @@ -70,7 +70,7 @@ namespace Duplicati.Library.Snapshots : base(followSymlinks) { // For Windows, ensure we don't store paths with extended device path prefixes (i.e., @"\\?\" or @"\\?\UNC\") - _sourceFolders = sources.Select(SystemIOWindows.RemoveExtendedDevicePathPrefix).ToList(); + _sourceEntries = sources.Select(SystemIOWindows.RemoveExtendedDevicePathPrefix).ToList(); try { _vssBackupComponents = new VssBackupComponents(); @@ -87,7 +87,7 @@ namespace Duplicati.Library.Snapshots } _vssBackupComponents.SetupWriters(null, excludedWriters); - _vssBackupComponents.InitShadowVolumes(_sourceFolders); + _vssBackupComponents.InitShadowVolumes(_sourceEntries); _vssBackupComponents.MapVolumesToSnapShots(); @@ -119,7 +119,7 @@ namespace Duplicati.Library.Snapshots /// /// Gets the source folders /// - public override IEnumerable SourceFolders => _sourceFolders; + public override IEnumerable SourceEntries => _sourceEntries; /// /// Enumerates the root source files and folders @@ -127,7 +127,7 @@ namespace Duplicati.Library.Snapshots /// The source files and folders public override IEnumerable EnumerateFilesystemEntries() { - foreach (var folder in _sourceFolders) + foreach (var folder in _sourceEntries) { if (folder.EndsWith(Path.DirectorySeparatorChar) || DirectoryExists(folder)) yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(folder), true, true); From 88f91b5b9e4fc3ff43f960ebbb812dbe2f047095 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 18 Feb 2025 22:33:47 +0100 Subject: [PATCH 199/362] Removed extra sln file --- .../Duplicati.GUI.TrayIcon.sln | 24 ------------------- 1 file changed, 24 deletions(-) delete mode 100644 Duplicati/GUI/Duplicati.GUI.TrayIcon/Duplicati.GUI.TrayIcon.sln diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Duplicati.GUI.TrayIcon.sln b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Duplicati.GUI.TrayIcon.sln deleted file mode 100644 index 9e8f17693..000000000 --- a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Duplicati.GUI.TrayIcon.sln +++ /dev/null @@ -1,24 +0,0 @@ -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.5.2.0 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.GUI.TrayIcon", "Duplicati.GUI.TrayIcon.csproj", "{E35D4E94-4FFB-4503-2107-3B0B0BCECB00}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Any CPU = Debug|Any CPU - Release|Any CPU = Release|Any CPU - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {E35D4E94-4FFB-4503-2107-3B0B0BCECB00}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {E35D4E94-4FFB-4503-2107-3B0B0BCECB00}.Debug|Any CPU.Build.0 = Debug|Any CPU - {E35D4E94-4FFB-4503-2107-3B0B0BCECB00}.Release|Any CPU.ActiveCfg = Release|Any CPU - {E35D4E94-4FFB-4503-2107-3B0B0BCECB00}.Release|Any CPU.Build.0 = Release|Any CPU - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection - GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {5FCDB925-2807-4C6C-A0C5-FBF8B201B580} - EndGlobalSection -EndGlobal From cf5b4bb24dc5617179df78fea57532ff862fcb95 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 19 Feb 2025 14:00:14 +0100 Subject: [PATCH 200/362] Added support for Wmic based Windows VSS snapshots. Refactored code a bit to be easier to understand. --- .../Operation/Backup/CountFilesHandler.cs | 2 +- .../Backup/FileEnumerationProcess.cs | 3 +- .../Library/Main/Operation/BackupHandler.cs | 1 + Duplicati/Library/Main/Options.cs | 19 ++ Duplicati/Library/Main/Strings.cs | 2 + .../Library/Modules/Builtin/HyperVOptions.cs | 43 ++-- .../Library/Modules/Builtin/MSSQLOptions.cs | 45 ++-- Duplicati/Library/Snapshots/Program.cs | 2 - .../Library/Snapshots/SnapshotProvider.cs | 30 +++ .../Library/Snapshots/{ => USN}/USNJournal.cs | 2 +- .../Snapshots/{ => USN}/UsnJournalService.cs | 2 +- .../Library/Snapshots/{ => USN}/Win32USN.cs | 42 ++-- .../Snapshots/Windows/AlphaVssBackup.cs | 200 ++++++++++++++++ .../Snapshots/{ => Windows}/HyperVUtility.cs | 11 +- .../Snapshots/Windows/ISnapshotProvider.cs | 118 ++++++++++ .../Snapshots/{ => Windows}/MSSQLUtility.cs | 49 ++-- .../Windows/SnapshotManager.cs} | 180 ++++++--------- .../Windows/WmicShadowCopyManager.cs | 215 ++++++++++++++++++ .../Snapshots/Windows/WmicVssBackup.cs | 139 +++++++++++ .../Library/Snapshots/WindowsSnapshot.cs | 21 +- .../WebserverCore/Endpoints/V1/HyperV.cs | 5 +- Duplicati/WebserverCore/Endpoints/V1/MSSql.cs | 3 +- 22 files changed, 914 insertions(+), 220 deletions(-) create mode 100644 Duplicati/Library/Snapshots/SnapshotProvider.cs rename Duplicati/Library/Snapshots/{ => USN}/USNJournal.cs (97%) rename Duplicati/Library/Snapshots/{ => USN}/UsnJournalService.cs (97%) rename Duplicati/Library/Snapshots/{ => USN}/Win32USN.cs (97%) create mode 100644 Duplicati/Library/Snapshots/Windows/AlphaVssBackup.cs rename Duplicati/Library/Snapshots/{ => Windows}/HyperVUtility.cs (95%) create mode 100644 Duplicati/Library/Snapshots/Windows/ISnapshotProvider.cs rename Duplicati/Library/Snapshots/{ => Windows}/MSSQLUtility.cs (94%) rename Duplicati/Library/{Common/IO/VssBackupComponents.cs => Snapshots/Windows/SnapshotManager.cs} (61%) create mode 100644 Duplicati/Library/Snapshots/Windows/WmicShadowCopyManager.cs create mode 100644 Duplicati/Library/Snapshots/Windows/WmicVssBackup.cs diff --git a/Duplicati/Library/Main/Operation/Backup/CountFilesHandler.cs b/Duplicati/Library/Main/Operation/Backup/CountFilesHandler.cs index ffefa0abc..1d16f29d9 100644 --- a/Duplicati/Library/Main/Operation/Backup/CountFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/Backup/CountFilesHandler.cs @@ -24,7 +24,7 @@ using System.Threading.Tasks; using Duplicati.Library.Utility; using CoCoL; using Duplicati.Library.Interface; -using Duplicati.Library.Snapshots; +using Duplicati.Library.Snapshots.USN; namespace Duplicati.Library.Main.Operation.Backup { diff --git a/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs b/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs index a7db2b2ce..a181907ff 100644 --- a/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs @@ -31,9 +31,8 @@ using System.Threading; using Duplicati.Library.Main.Operation.Common; using Duplicati.Library.Interface; using System.Runtime.CompilerServices; -using Duplicati.Library.Snapshots; using Duplicati.Library.SourceProvider; -using Duplicati.Library.Common.IO; +using Duplicati.Library.Snapshots.USN; namespace Duplicati.Library.Main.Operation.Backup { diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 84fc89b69..059bbc8df 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -36,6 +36,7 @@ using Duplicati.Library.Main.Operation.Common; using System.Data; using Duplicati.Library.DynamicLoader; using Duplicati.Library.SourceProvider; +using Duplicati.Library.Snapshots.USN; namespace Duplicati.Library.Main.Operation { diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index c0d681aaa..4d32c8ae3 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -345,6 +345,7 @@ namespace Duplicati.Library.Main new CommandLineArgument("upload-unchanged-backups", CommandLineArgument.ArgumentType.Boolean, Strings.Options.UploadUnchangedBackupsShort, Strings.Options.UploadUnchangedBackupsLong, "false"), new CommandLineArgument("snapshot-policy", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.SnapshotpolicyShort, Strings.Options.SnapshotpolicyLong, "off", null, Enum.GetNames(typeof(OptimizationStrategy))), + new CommandLineArgument("snapshot-provider", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.SnapshotproviderShort, Strings.Options.SnapshotproviderLong, OperatingSystem.IsWindows() ? Snapshots.SnapshotProvider.AlphaVSS.ToString() : Snapshots.SnapshotProvider.LVM.ToString(), null, (OperatingSystem.IsWindows() ? [Snapshots.SnapshotProvider.AlphaVSS, Snapshots.SnapshotProvider.Wmic] : new [] { Snapshots.SnapshotProvider.LVM }).Select(x => x.ToString()).ToArray()), new CommandLineArgument("vss-exclude-writers", CommandLineArgument.ArgumentType.String, Strings.Options.VssexcludewritersShort, Strings.Options.VssexcludewritersLong, "{e8132975-6f93-4464-a53e-1050253ae220}"), new CommandLineArgument("vss-use-mapping", CommandLineArgument.ArgumentType.Boolean, Strings.Options.VssusemappingShort, Strings.Options.VssusemappingLong, "false"), new CommandLineArgument("usn-policy", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.UsnpolicyShort, Strings.Options.UsnpolicyLong, "off", null, Enum.GetNames(typeof(OptimizationStrategy))), @@ -1025,6 +1026,24 @@ namespace Duplicati.Library.Main } } + /// + /// Gets the snapshot strategy to use + /// + public Snapshots.SnapshotProvider SnapShotProvider + { + get + { + if (!m_options.TryGetValue("snapshot-provider", out var provider)) + provider = ""; + + Snapshots.SnapshotProvider r; + if (!Enum.TryParse(provider, true, out r)) + r = OperatingSystem.IsWindows() ? Snapshots.SnapshotProvider.AlphaVSS : Snapshots.SnapshotProvider.LVM; + + return r; + } + } + /// /// Gets a flag indicating if advisory locking should be ignored /// diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs index 6ee298520..862d064c2 100644 --- a/Duplicati/Library/Main/Strings.cs +++ b/Duplicati/Library/Main/Strings.cs @@ -116,6 +116,8 @@ namespace Duplicati.Library.Main.Strings public static string EnablemoduleShort { get { return LC.L(@"Enable one or more modules"); } } public static string SnapshotpolicyLong { get { return LC.L(@"This setting controls the usage of snapshots, which allows Duplicati to backup files that are locked by other programs. If this is set to ""off"", Duplicati will not attempt to create a disk snapshot. Setting this to ""auto"" makes Duplicati attempt to create a snapshot, and fail silently if that was not allowed or supported (note that the OS may still log system warnings). A setting of ""on"" will also make Duplicati attempt to create a snapshot, but will produce a warning message in the log if it fails. Setting it to ""required"" will make Duplicati abort the backup if the snapshot creation fails. On windows this uses the Volume Shadow Copy Services (VSS) and requires administrative privileges. On Linux this uses Logical Volume Management (LVM) and requires root privileges."); } } public static string SnapshotpolicyShort { get { return LC.L(@"Control the use of disk snapshots"); } } + public static string SnapshotproviderLong { get { return LC.L(@"The snapshot provider implementation for Windows. The AlphaVSS is the most feature complete, but is not supported on Arm64. The WMIC based snapshot has less features, but is more portable. On Linux, only LVM is supported"); } } + public static string SnapshotproviderShort { get { return LC.L(@"The snapshot provider implementation to use"); } } public static string AsynchronousuploadfolderLong { get { return LC.L(@"The pre-generated volumes will be placed into the temporary folder by default. This option can set a different folder for placing the temporary volumes. Despite the name, this also works for synchronous runs."); } } public static string AsynchronousuploadfolderShort { get { return LC.L(@"The path where ready volumes are placed until uploaded"); } } public static string AsynchronousuploadlimitLong { get { return LC.L(@"When performing asynchronous uploads, Duplicati will create volumes that can be uploaded. To prevent Duplicati from generating too many volumes, this option limits the number of pending uploads. Set to zero to disable the limit. The volume(s) that are being created are not counted in this limit. Use the option --concurrency-compressors=1 to limit the number of volumes being created."); } } diff --git a/Duplicati/Library/Modules/Builtin/HyperVOptions.cs b/Duplicati/Library/Modules/Builtin/HyperVOptions.cs index 2df703c2e..67bda3d9e 100644 --- a/Duplicati/Library/Modules/Builtin/HyperVOptions.cs +++ b/Duplicati/Library/Modules/Builtin/HyperVOptions.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -24,8 +24,8 @@ using System.Collections.Generic; using System.Linq; using System.Runtime.Versioning; using System.Text.RegularExpressions; -using Duplicati.Library.Common; using Duplicati.Library.Snapshots; +using Duplicati.Library.Snapshots.Windows; namespace Duplicati.Library.Modules.Builtin { @@ -149,7 +149,8 @@ namespace Duplicati.Library.Modules.Builtin Logging.Log.WriteWarningMessage(LOGTAG, "HyperVOnServerOnly", null, "This is client version of Windows. Hyper-V VSS writer is present only on Server version. Backup will continue, but will be crash consistent only in opposite to application consistent in Server version"); Logging.Log.WriteInformationMessage(LOGTAG, "StartingHyperVQuery", "Starting to gather Hyper-V information"); - hypervUtility.QueryHyperVGuestsInfo(true); + var provider = Utility.Utility.ParseEnumOption(changedOptions.AsReadOnly(), "snapshot-provider", SnapshotProvider.AlphaVSS); + hypervUtility.QueryHyperVGuestsInfo(provider, true); Logging.Log.WriteInformationMessage(LOGTAG, "HyperVMachineCount", "Found {0} virtual machines on Hyper-V", hypervUtility.Guests.Count); foreach (var guest in hypervUtility.Guests) diff --git a/Duplicati/Library/Modules/Builtin/MSSQLOptions.cs b/Duplicati/Library/Modules/Builtin/MSSQLOptions.cs index d41f84009..058b119c5 100644 --- a/Duplicati/Library/Modules/Builtin/MSSQLOptions.cs +++ b/Duplicati/Library/Modules/Builtin/MSSQLOptions.cs @@ -1,31 +1,31 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; using System.Linq; using System.Runtime.Versioning; using System.Text.RegularExpressions; -using Duplicati.Library.Common; using Duplicati.Library.Snapshots; +using Duplicati.Library.Snapshots.Windows; namespace Duplicati.Library.Modules.Builtin { @@ -144,7 +144,8 @@ namespace Duplicati.Library.Modules.Builtin } Logging.Log.WriteInformationMessage(LOGTAG, "StartingMsSqlQuery", "Starting to gather Microsoft SQL Server information", Logging.LogMessageType.Information); - mssqlUtility.QueryDBsInfo(); + var provider = Utility.Utility.ParseEnumOption(changedOptions.AsReadOnly(), "snapshot-provider", SnapshotProvider.AlphaVSS); + mssqlUtility.QueryDBsInfo(provider); Logging.Log.WriteInformationMessage(LOGTAG, "MsSqlDatabaseCount", "Found {0} databases on Microsoft SQL Server", mssqlUtility.DBs.Count); foreach(var db in mssqlUtility.DBs) diff --git a/Duplicati/Library/Snapshots/Program.cs b/Duplicati/Library/Snapshots/Program.cs index 6afd2647d..8e48df343 100644 --- a/Duplicati/Library/Snapshots/Program.cs +++ b/Duplicati/Library/Snapshots/Program.cs @@ -21,9 +21,7 @@ using System; using System.Collections.Generic; -using System.Linq; using Duplicati.Library.AutoUpdater; -using Duplicati.Library.Common; using Duplicati.Library.Utility; namespace Duplicati.Library.Snapshots diff --git a/Duplicati/Library/Snapshots/SnapshotProvider.cs b/Duplicati/Library/Snapshots/SnapshotProvider.cs new file mode 100644 index 000000000..c1e8984fa --- /dev/null +++ b/Duplicati/Library/Snapshots/SnapshotProvider.cs @@ -0,0 +1,30 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +namespace Duplicati.Library.Snapshots +{ + public enum SnapshotProvider + { + AlphaVSS, + Wmic, + LVM + } +} diff --git a/Duplicati/Library/Snapshots/USNJournal.cs b/Duplicati/Library/Snapshots/USN/USNJournal.cs similarity index 97% rename from Duplicati/Library/Snapshots/USNJournal.cs rename to Duplicati/Library/Snapshots/USN/USNJournal.cs index 2608395d5..73346ae59 100644 --- a/Duplicati/Library/Snapshots/USNJournal.cs +++ b/Duplicati/Library/Snapshots/USN/USNJournal.cs @@ -31,7 +31,7 @@ using System.Runtime.Versioning; using Duplicati.Library.Common.IO; using Microsoft.Win32.SafeHandles; -namespace Duplicati.Library.Snapshots +namespace Duplicati.Library.Snapshots.USN { /// /// Class that encapsulates USN journal access to a single volume diff --git a/Duplicati/Library/Snapshots/UsnJournalService.cs b/Duplicati/Library/Snapshots/USN/UsnJournalService.cs similarity index 97% rename from Duplicati/Library/Snapshots/UsnJournalService.cs rename to Duplicati/Library/Snapshots/USN/UsnJournalService.cs index 8f1298472..d44bad907 100644 --- a/Duplicati/Library/Snapshots/UsnJournalService.cs +++ b/Duplicati/Library/Snapshots/USN/UsnJournalService.cs @@ -33,7 +33,7 @@ using Duplicati.Library.Utility; using System.Threading.Tasks; using System.Runtime.CompilerServices; -namespace Duplicati.Library.Snapshots +namespace Duplicati.Library.Snapshots.USN { [SupportedOSPlatform("windows")] public class UsnJournalService diff --git a/Duplicati/Library/Snapshots/Win32USN.cs b/Duplicati/Library/Snapshots/USN/Win32USN.cs similarity index 97% rename from Duplicati/Library/Snapshots/Win32USN.cs rename to Duplicati/Library/Snapshots/USN/Win32USN.cs index ed40e8c3f..3a9ebf7c2 100644 --- a/Duplicati/Library/Snapshots/Win32USN.cs +++ b/Duplicati/Library/Snapshots/USN/Win32USN.cs @@ -1,30 +1,30 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. using System; using System.Runtime.InteropServices; using System.Runtime.Versioning; using Microsoft.Win32.SafeHandles; -namespace Duplicati.Library.Snapshots +namespace Duplicati.Library.Snapshots.USN { //The signatures in this file are from http://pinvoke.net diff --git a/Duplicati/Library/Snapshots/Windows/AlphaVssBackup.cs b/Duplicati/Library/Snapshots/Windows/AlphaVssBackup.cs new file mode 100644 index 000000000..85d65f49b --- /dev/null +++ b/Duplicati/Library/Snapshots/Windows/AlphaVssBackup.cs @@ -0,0 +1,200 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Alphaleonis.Win32.Vss; +using Duplicati.Library.Common.IO; + +namespace Duplicati.Library.Snapshots.Windows; + +/// +/// Implementation of a snapshot, using AlphaVSS +/// +internal class AlphaVssBackup : ISnapshotProvider +{ + /// + /// The implementation that is being wrapped + /// + private IVssBackupComponents _components; + /// + /// Creates a new snapshot implementation + /// + /// The components to use + private AlphaVssBackup(IVssBackupComponents components) + { + _components = components; + } + + /// + public void EnableWriterClasses(Guid[] guids) + => _components.EnableWriterClasses(guids); + + /// + public void DisableWriterClasses(Guid[] guids) + => _components.DisableWriterClasses(guids); + + /// + public void GatherWriterMetadata() + => _components.GatherWriterMetadata(); + /// + public void FreeWriterMetadata() + => _components.FreeWriterMetadata(); + + /// + public void VerifyWriters(Guid[] guids) + { + foreach (var writerGUID in guids) + { + if (!_components.WriterMetadata.Any(o => o.WriterId.Equals(writerGUID))) + { + throw new Exception(string.Format("Writer with GUID {0} was not added to VSS writer set.", writerGUID.ToString())); + } + } + } + + /// + /// Stub for returning the snapshot info + /// + /// The path where the snapshot is located + private sealed record SnapshotInfo(string SnapshotDeviceObject) : ISnapshotInfo; + + /// + public ISnapshotInfo GetSnapshotProperties(Guid shadowId) + => new SnapshotInfo(_components.GetSnapshotProperties(shadowId).SnapshotDeviceObject); + + /// + public void StartSnapshotSet() + => _components.StartSnapshotSet(); + + /// + public void PrepareForBackup() + => _components.PrepareForBackup(); + + /// + public void DoSnapshotSet() + => _components.DoSnapshotSet(); + + /// + public bool IsVolumeSupported(string drive) + => _components.IsVolumeSupported(drive); + + /// + public Guid AddToSnapshotSet(string drive) + => _components.AddToSnapshotSet(drive); + + /// + public void BackupComplete() + => _components.BackupComplete(); + + /// + public void DeleteSnapshot(Guid shadowId, bool forceDelete) + => _components.DeleteSnapshot(shadowId, forceDelete); + + /// + public static ISnapshotProvider Create() + => new AlphaVssBackup(GetVssBackupComponents()); + + /// + /// Creates a new AlphaVssBackupComponents instance and adds the context + /// + /// The configured instance + private static IVssBackupComponents GetVssBackupComponents() + { + //Prepare the backup + IVssBackupComponents vssBackupComponents = CreateVssBackupComponents(); + vssBackupComponents.InitializeForBackup(null); + vssBackupComponents.SetContext(VssSnapshotContext.Backup); + vssBackupComponents.SetBackupState(false, true, VssBackupType.Full, false); + + return vssBackupComponents; + } + + /// + /// Loads the AlphaVSS implementation for the current OS + /// + /// The implementation + private static IVssBackupComponents CreateVssBackupComponents() + { + var vss = VssFactoryProvider.Default.GetVssFactory(); + if (vss == null) + throw new InvalidOperationException(); + + return vss.CreateVssBackupComponents(); + } + + /// + public IEnumerable ParseWriterMetaData(Guid[] writers) + { + // check if writers got enabled + foreach (var writerGUID in writers) + { + var writer = _components.WriterMetadata.First(o => o.WriterId.Equals(writerGUID)); + foreach (var component in writer.Components) + { + yield return new WriterMetaData + { + Guid = writerGUID, + Name = component.ComponentName, + LogicalPath = component.LogicalPath, + Paths = GetPathsFromComponent(component) + }; + } + } + } + + /// + /// Returns the paths from the writer metadata + /// + /// The component to get the paths from + /// The list of paths + private List GetPathsFromComponent(IVssWMComponent component) + { + var paths = new List(); + + foreach (var file in component.Files) + { + if (file.FileSpecification.Contains("*")) + { + if (Directory.Exists(Util.AppendDirSeparator(file.Path))) + paths.Add(Util.AppendDirSeparator(file.Path)); + } + else + { + var fileWithSpec = SystemIO.IO_OS.PathCombine(file.Path, file.FileSpecification); + if (File.Exists(fileWithSpec)) + paths.Add(fileWithSpec); + } + } + return paths; + + } + + /// + public void Dispose() + { + _components.Dispose(); + } +} \ No newline at end of file diff --git a/Duplicati/Library/Snapshots/HyperVUtility.cs b/Duplicati/Library/Snapshots/Windows/HyperVUtility.cs similarity index 95% rename from Duplicati/Library/Snapshots/HyperVUtility.cs rename to Duplicati/Library/Snapshots/Windows/HyperVUtility.cs index 4e7b268a6..9e0f16524 100644 --- a/Duplicati/Library/Snapshots/HyperVUtility.cs +++ b/Duplicati/Library/Snapshots/Windows/HyperVUtility.cs @@ -26,7 +26,7 @@ using System.Management; using System.Runtime.Versioning; using Duplicati.Library.Common.IO; -namespace Duplicati.Library.Snapshots +namespace Duplicati.Library.Snapshots.Windows { public class HyperVGuest : IEquatable { @@ -160,8 +160,9 @@ namespace Duplicati.Library.Snapshots /// Query Hyper-V for all Virtual Machines info /// /// Specify if returned data should contain VM paths + /// The provider to use for VSS /// List of Hyper-V Machines - public void QueryHyperVGuestsInfo(bool bIncludePaths = false) + public void QueryHyperVGuestsInfo(SnapshotProvider provider, bool bIncludePaths = false) { if (!IsHyperVInstalled) return; @@ -178,7 +179,7 @@ namespace Duplicati.Library.Snapshots if (bIncludePaths) { - foreach (var o in GetAllVMsPathsVSS()) + foreach (var o in GetAllVMsPathsVSS(provider)) { foreach (var mObject in moCollection) { @@ -218,9 +219,9 @@ namespace Duplicati.Library.Snapshots /// For all Hyper-V guests it enumerate all associated paths using VSS data /// /// A collection of VMs and paths - private static IEnumerable GetAllVMsPathsVSS() + private static IEnumerable GetAllVMsPathsVSS(SnapshotProvider provider) { - using (var vssBackupComponents = new VssBackupComponents()) + using (var vssBackupComponents = new SnapshotManager(provider)) { var writerGUIDS = new[] { HyperVWriterGuid }; diff --git a/Duplicati/Library/Snapshots/Windows/ISnapshotProvider.cs b/Duplicati/Library/Snapshots/Windows/ISnapshotProvider.cs new file mode 100644 index 000000000..db6a0965a --- /dev/null +++ b/Duplicati/Library/Snapshots/Windows/ISnapshotProvider.cs @@ -0,0 +1,118 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +using System; +using System.Collections.Generic; + +namespace Duplicati.Library.Snapshots.Windows; + +/// +/// Interface for the snapshot information +/// +internal interface ISnapshotInfo +{ + /// + /// The path where the snapshot is located + /// + string SnapshotDeviceObject { get; } +} + +/// +/// Interface for a snapshot provider +/// +internal interface ISnapshotProvider : IDisposable +{ + /// + /// Explicitly enables the selected writer GUIDs + /// + /// The GUIDs to enable + void EnableWriterClasses(Guid[] guids); + /// + /// Explicitly disable the selected writer GUIDs + /// + /// The GUIDs to disable + void DisableWriterClasses(Guid[] guids); + /// + /// Gather writer metadata prior to the snapshot + /// + void GatherWriterMetadata(); + /// + /// Release any allocated metadata + /// + void FreeWriterMetadata(); + /// + /// Verify that the writes are enabled on the snapshot + /// + /// The writer GUIDs to verify as enabled + void VerifyWriters(Guid[] guids); + + /// + /// Gets information of a snapshot + /// + /// The shadow copy id + /// The snapshot information + ISnapshotInfo GetSnapshotProperties(Guid shadowId); + + /// + /// Creates a snapshot + /// + void StartSnapshotSet(); + /// + /// Starts the snapshot as a backup snapshot + /// + void PrepareForBackup(); + /// + /// Finalizes the snapshot creation + /// + void DoSnapshotSet(); + /// + /// Checks if the volume is supported by the snapshot + /// + /// The drive to check + /// true if the volume is supported; false otherwise + bool IsVolumeSupported(string drive); + /// + /// Adds a drive to the snapshot set + /// + /// The drive to add + /// The shadow id + Guid AddToSnapshotSet(string drive); + + /// + /// Completes the snapshot after the backup has completed + /// + void BackupComplete(); + /// + /// Releases all resources held by the snapshot + /// + /// The snapshot shadow id + /// Flag to choose force deletion + void DeleteSnapshot(Guid shadowId, bool forceDelete); + + /// + /// Parses write metadata + /// + /// The writers to return information from + /// The writer metadata + IEnumerable ParseWriterMetaData(Guid[] writers); +} \ No newline at end of file diff --git a/Duplicati/Library/Snapshots/MSSQLUtility.cs b/Duplicati/Library/Snapshots/Windows/MSSQLUtility.cs similarity index 94% rename from Duplicati/Library/Snapshots/MSSQLUtility.cs rename to Duplicati/Library/Snapshots/Windows/MSSQLUtility.cs index 896c24686..7cbf8444e 100644 --- a/Duplicati/Library/Snapshots/MSSQLUtility.cs +++ b/Duplicati/Library/Snapshots/Windows/MSSQLUtility.cs @@ -1,31 +1,30 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. -using Duplicati.Library.Common; -using Duplicati.Library.Common.IO; +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + using System; using System.Collections.Generic; using System.Linq; using System.Runtime.Versioning; -namespace Duplicati.Library.Snapshots +namespace Duplicati.Library.Snapshots.Windows { public class MSSQLDB : IEquatable { @@ -152,14 +151,14 @@ namespace Duplicati.Library.Snapshots /// For all MS SQL databases it enumerate all associated paths using VSS data /// /// A collection of DBs and paths - public void QueryDBsInfo() + public void QueryDBsInfo(SnapshotProvider provider) { if (!IsMSSQLInstalled) return; m_DBs.Clear(); - using (var vssBackupComponents = new VssBackupComponents()) + using (var vssBackupComponents = new SnapshotManager(provider)) { var writerGUIDS = new [] { MSSQLWriterGuid }; try diff --git a/Duplicati/Library/Common/IO/VssBackupComponents.cs b/Duplicati/Library/Snapshots/Windows/SnapshotManager.cs similarity index 61% rename from Duplicati/Library/Common/IO/VssBackupComponents.cs rename to Duplicati/Library/Snapshots/Windows/SnapshotManager.cs index 8d8619a2a..78d21fa91 100644 --- a/Duplicati/Library/Common/IO/VssBackupComponents.cs +++ b/Duplicati/Library/Snapshots/Windows/SnapshotManager.cs @@ -21,13 +21,16 @@ using System; using System.Collections.Generic; -using System.IO; using System.Linq; using System.Runtime.Versioning; using Alphaleonis.Win32.Vss; +using Duplicati.Library.Common.IO; -namespace Duplicati.Library.Common.IO +namespace Duplicati.Library.Snapshots.Windows { + /// + /// Metadata from a writer instance + /// public class WriterMetaData { public string Name { get; set; } @@ -36,16 +39,21 @@ namespace Duplicati.Library.Common.IO public List Paths { get; set; } } + /// + /// The manager for the Windows snapshot + /// [SupportedOSPlatform("windows")] - public class VssBackupComponents : IDisposable + public class SnapshotManager : IDisposable { /// /// The tag used for logging messages /// - public static readonly string LOGTAG = Logging.Log.LogTagFromType(); + private static readonly string LOGTAG = Logging.Log.LogTagFromType(); - - private IVssBackupComponents _vssBackupComponents; + /// + /// The snapshot implementation + /// + private ISnapshotProvider _snapshotProvider; /// /// The list of snapshot ids for each volume, key is the path root, e.g. C:\. @@ -69,26 +77,39 @@ namespace Duplicati.Library.Common.IO /// private List _mappedDrives; - public VssBackupComponents() + /// + /// Creates a new snapshot manager + /// + /// The provider to use + public SnapshotManager(SnapshotProvider provider) { - _vssBackupComponents = VssBackupComponentsHelper.GetVssBackupComponents(); + _snapshotProvider = provider switch { + SnapshotProvider.AlphaVSS => AlphaVssBackup.Create(), + SnapshotProvider.Wmic => new WmicVssBackup(), + _ => throw new ArgumentException($"Invalid provider: {provider}") + }; } + /// + /// Setup writers on the snapshot + /// + /// The explicitly included writers + /// The explicitly excluded writers public void SetupWriters(Guid[] includedWriters, Guid[] excludedWriters) { if (includedWriters != null && includedWriters.Length > 0) - _vssBackupComponents.EnableWriterClasses(includedWriters); + _snapshotProvider.EnableWriterClasses(includedWriters); if (excludedWriters != null && excludedWriters.Length > 0) - _vssBackupComponents.DisableWriterClasses(excludedWriters); + _snapshotProvider.DisableWriterClasses(excludedWriters); try { - _vssBackupComponents.GatherWriterMetadata(); + _snapshotProvider.GatherWriterMetadata(); } finally { - _vssBackupComponents.FreeWriterMetadata(); + _snapshotProvider.FreeWriterMetadata(); } if (includedWriters == null) @@ -97,15 +118,12 @@ namespace Duplicati.Library.Common.IO } // check if writers got enabled - foreach (var writerGUID in includedWriters) - { - if (!_vssBackupComponents.WriterMetadata.Any(o => o.WriterId.Equals(writerGUID))) - { - throw new Exception(string.Format("Writer with GUID {0} was not added to VSS writer set.", writerGUID.ToString())); - } - } + _snapshotProvider.VerifyWriters(includedWriters); } + /// + /// Maps drives from snapshot paths + /// public void MapDrives() { _mappedDrives = new List(); @@ -124,6 +142,9 @@ namespace Duplicati.Library.Common.IO } } + /// + /// Makes a map of the snapshots for easy lookup + /// public void MapVolumesToSnapShots() { _volumeMap = new Dictionary(StringComparer.OrdinalIgnoreCase); @@ -133,43 +154,17 @@ namespace Duplicati.Library.Common.IO // Attempting to get attributes on this path will fail, so we need to append a double backslash // Strangely, the double backslash is not needed when acessing a subfolder, here a single backslash is enough, // but double backslash works for both cases due to path normalization. - _volumeMap.Add(kvp.Key, Util.AppendDirSeparator(_vssBackupComponents.GetSnapshotProperties(kvp.Value).SnapshotDeviceObject) + "\\"); + _volumeMap.Add(kvp.Key, Util.AppendDirSeparator(_snapshotProvider.GetSnapshotProperties(kvp.Value).SnapshotDeviceObject) + "\\"); } _volumeReverseMap = _volumeMap.ToDictionary(x => x.Value, x => x.Key); } - public Dictionary SnapshotDeviceAndVolumes - { - get - { - return _volumeReverseMap; - } - } - - public IEnumerable SnapshotLocalPaths => _volumeMap.Keys; - - private List GetPathsFromComponent(IVssWMComponent component) - { - var paths = new List(); - - foreach (var file in component.Files) - { - if (file.FileSpecification.Contains("*")) - { - if (Directory.Exists(Util.AppendDirSeparator(file.Path))) - paths.Add(Util.AppendDirSeparator(file.Path)); - } - else - { - var fileWithSpec = SystemIO.IO_OS.PathCombine(file.Path, file.FileSpecification); - if (File.Exists(fileWithSpec)) - paths.Add(fileWithSpec); - } - } - return paths; - - } + /// + /// Returns the map of snapshot path and logical path + /// + public Dictionary SnapshotDeviceAndVolumes + => _volumeReverseMap; /// /// Enumerable to iterate over components of the given writers. @@ -178,38 +173,30 @@ namespace Duplicati.Library.Common.IO /// Anonymous object containing GUID, component name, logical path and paths of the component. /// Writers. public IEnumerable ParseWriterMetaData(Guid[] writers) - { - // check if writers got enabled - foreach (var writerGUID in writers) - { - var writer = _vssBackupComponents.WriterMetadata.First(o => o.WriterId.Equals(writerGUID)); - foreach (var component in writer.Components) - { - yield return new WriterMetaData - { - Guid = writerGUID, - Name = component.ComponentName, - LogicalPath = component.LogicalPath, - Paths = GetPathsFromComponent(component) - }; - } - } - } - + => _snapshotProvider.ParseWriterMetaData(writers); + /// + /// Prepares the snapshot and notifies all writers + /// + /// The soruces to include public void InitShadowVolumes(IEnumerable sources) { - _vssBackupComponents.StartSnapshotSet(); + _snapshotProvider.StartSnapshotSet(); - CheckSupportedVolumes(sources); + CheckAndAddSupportedVolumes(sources); //Make all writers aware that we are going to do the backup - _vssBackupComponents.PrepareForBackup(); + _snapshotProvider.PrepareForBackup(); //Create the shadow volumes - _vssBackupComponents.DoSnapshotSet(); + _snapshotProvider.DoSnapshotSet(); } + /// + /// Gets the snapshot path from the local path + /// + /// The local path + /// The snapshot path public string GetVolumeFromCache(string path) { if (!_volumeMap.TryGetValue(path, out var volumePath)) @@ -218,7 +205,11 @@ namespace Duplicati.Library.Common.IO return volumePath; } - public void CheckSupportedVolumes(IEnumerable sources) + /// + /// Checks if the volumes can be added and creates snapshots as needed + /// + /// The sources to create snapshots for + private void CheckAndAddSupportedVolumes(IEnumerable sources) { //Figure out which volumes are in the set _volumes = new Dictionary(StringComparer.OrdinalIgnoreCase); @@ -227,16 +218,17 @@ namespace Duplicati.Library.Common.IO var drive = SystemIO.IO_OS.GetPathRoot(s); if (!_volumes.ContainsKey(drive)) { - if (!_vssBackupComponents.IsVolumeSupported(drive)) + if (!_snapshotProvider.IsVolumeSupported(drive)) { throw new VssVolumeNotSupportedException(drive); } - _volumes.Add(drive, _vssBackupComponents.AddToSnapshotSet(drive)); + _volumes.Add(drive, _snapshotProvider.AddToSnapshotSet(drive)); } } } + /// public void Dispose() { try @@ -258,7 +250,7 @@ namespace Duplicati.Library.Common.IO try { - _vssBackupComponents?.BackupComplete(); + _snapshotProvider?.BackupComplete(); } catch (Exception ex) { @@ -267,13 +259,13 @@ namespace Duplicati.Library.Common.IO try { - if (_vssBackupComponents != null) + if (_snapshotProvider != null) { foreach (var g in _volumes.Values) { try { - _vssBackupComponents.DeleteSnapshot(g, false); + _snapshotProvider.DeleteSnapshot(g, false); } catch (Exception ex) { @@ -287,35 +279,11 @@ namespace Duplicati.Library.Common.IO Logging.Log.WriteVerboseMessage(LOGTAG, "VSSSnapShotDeleteCleanError", ex, "Failed during VSS esnapshot closing"); } - if (_vssBackupComponents != null) + if (_snapshotProvider != null) { - _vssBackupComponents.Dispose(); - _vssBackupComponents = null; + _snapshotProvider.Dispose(); + _snapshotProvider = null; } } } - - - public static class VssBackupComponentsHelper - { - public static IVssBackupComponents GetVssBackupComponents() - { - //Prepare the backup - IVssBackupComponents vssBackupComponents = CreateVssBackupComponents(); - vssBackupComponents.InitializeForBackup(null); - vssBackupComponents.SetContext(VssSnapshotContext.Backup); - vssBackupComponents.SetBackupState(false, true, VssBackupType.Full, false); - - return vssBackupComponents; - } - - public static IVssBackupComponents CreateVssBackupComponents() - { - var vss = VssFactoryProvider.Default.GetVssFactory(); - if (vss == null) - throw new InvalidOperationException(); - - return vss.CreateVssBackupComponents(); - } - } } diff --git a/Duplicati/Library/Snapshots/Windows/WmicShadowCopyManager.cs b/Duplicati/Library/Snapshots/Windows/WmicShadowCopyManager.cs new file mode 100644 index 000000000..45a1ddd6b --- /dev/null +++ b/Duplicati/Library/Snapshots/Windows/WmicShadowCopyManager.cs @@ -0,0 +1,215 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Runtime.Versioning; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +#nullable enable + +namespace Duplicati.Library.Snapshots.Windows; + +/// +/// A shadow copy manager using the wmic commandline tool +/// +[SupportedOSPlatform("windows")] +internal class WmicShadowCopyManager : IDisposable +{ + /// + /// The tag used for logging messages + /// + public static readonly string LOGTAG = Logging.Log.LogTagFromType(); + + /// + /// A single shadow copy + /// + /// The shadow ID as a string + /// The shadow ID as a GUID + /// The drive that the snapshot is for + /// The path that contains the snapshot + public class WmicShadowCopy(string shadowId, Guid parsedId, string originalDrive, string mappedPath) : IDisposable + { + /// + /// Gets the shadow ID + /// + public string ShadowID { get; } = shadowId; + /// + /// Gets the shadow ID + /// + public Guid ParsedId { get; } = parsedId; + /// + /// Gets the drive that was originally mapped + /// + public string OriginalDrive { get; } = originalDrive; + /// + /// Gets the path where the snapshot is found + /// + public string MappedPath { get; } = mappedPath; + + /// + /// Flag keeping track of the snapshot deletion state + /// + private bool _snapshotDeleted; + + /// + public void Dispose() + { + DeleteShadowCopy(); + } + + /// + /// Deletes the shadow copy + /// + private void DeleteShadowCopy() + { + if (_snapshotDeleted) + return; + if (!string.IsNullOrEmpty(ShadowID)) + { + _snapshotDeleted = true; + Logging.Log.WriteVerboseMessage(LOGTAG, "DeleteShadowCopy", $"Deleting Shadow Copy: {ShadowID}"); + DeleteShadow(ShadowID); + } + } + } + + /// + /// The list of the currently registered shadow copies + /// + private List _shadowCopies = new List(); + + /// + /// Gets the list of the currently registered shadow copies + /// + public IEnumerable ShadowCopies => _shadowCopies; + + /// + /// Creates a new snapshot for the given drive + /// + /// The drive to create the snapshot for + /// The created snapshot + public WmicShadowCopy Add(string drive) + { + var shadowId = CreateShadowCopy(drive); + if (string.IsNullOrEmpty(shadowId)) + throw new InvalidOperationException("Failed to create shadow copy"); + + var shadowPath = GetShadowPath(shadowId); + if (string.IsNullOrEmpty(shadowPath)) + { + DeleteShadow(shadowId); + throw new InvalidOperationException("Failed to get shadow copy path"); + } + + var snapshot = new WmicShadowCopy(shadowId, Guid.Parse(shadowId), drive, shadowPath); + _shadowCopies.Add(snapshot); + + return snapshot; + } + + /// + public void Dispose() + { + foreach (var shadow in ShadowCopies) + { + shadow.Dispose(); + } + _shadowCopies.Clear(); + } + + /// + /// Creates a shadow copy + /// + /// The drive to create the snapshot for + /// The shadow id + private static string? CreateShadowCopy(string drive) + { + string output = ExecuteCommand("wmic", $"shadowcopy call create Volume='{drive}'", 10000); + + // Extract ShadowID using regex + Match match = Regex.Match(output, @"ShadowID\s*=\s*""({[0-9A-F\-]+})"""); + if (match.Success) + { + return match.Groups[1].Value; + } + + Logging.Log.WriteErrorMessage(LOGTAG, "ShadowCopyFailed", null, "Failed to create shadow copy for {0}: {1}", drive, output); + return null; + } + + /// + /// Gets the path where the shadow copy is mounted + /// + /// The shadow copy id + /// The path where the copy is mounted + private static string? GetShadowPath(string shadowId) + { + string output = ExecuteCommand("wmic", "shadowcopy get ID, DeviceObject", 5000); + + // Extract DeviceObject corresponding to the ShadowID + string pattern = $@"(\\\\\?\\GLOBALROOT\\Device\\HarddiskVolumeShadowCopy\d+)\s+{shadowId}"; + Match match = Regex.Match(output, pattern); + + if (match.Success) + { + return match.Groups[1].Value; + } + + Logging.Log.WriteErrorMessage(LOGTAG, "ShadowCopyFailed", null, "Failed to get shadow copy path for {0}: {1}", shadowId, output); + return null; + } + + /// + /// Delete a shadow copy + /// + /// The shadow id + private static void DeleteShadow(string shadowId) + => ExecuteCommand("wmic", $"shadowcopy where ID=\"{shadowId}\" delete", 5000); + + /// + /// Executes a command + /// + /// The binary to execute + /// The arguments to use + /// The timeout + /// The output of the command + private static string ExecuteCommand(string fileName, string arguments, int timeoutMs) + { + try + { + using (var process = new Process()) + { + process.StartInfo.FileName = fileName; + process.StartInfo.Arguments = arguments; + process.StartInfo.RedirectStandardOutput = true; + process.StartInfo.RedirectStandardError = true; + process.StartInfo.UseShellExecute = false; + process.StartInfo.CreateNoWindow = true; + + process.Start(); + + var outputTask = process.StandardOutput.ReadToEndAsync(); + var errorTask = process.StandardError.ReadToEndAsync(); + + if (!process.WaitForExit(timeoutMs)) + { + process.Kill(); + throw new TimeoutException($"Command '{fileName} {arguments}' timed out after {timeoutMs}ms."); + } + + var output = outputTask.Result; + var error = errorTask.Result; + + if (!string.IsNullOrWhiteSpace(error)) + Logging.Log.WriteWarningMessage(LOGTAG, "ShadowCopyFailed", null, "Failed to execute command: {0} {1}: {2}", fileName, arguments, error); + + return output; + } + } + catch (Exception ex) + { + Logging.Log.WriteErrorMessage(LOGTAG, "ShadowCopyFailed", ex, "Failed to execute command: {0} {1}", fileName, arguments); + return string.Empty; + } + } +} \ No newline at end of file diff --git a/Duplicati/Library/Snapshots/Windows/WmicVssBackup.cs b/Duplicati/Library/Snapshots/Windows/WmicVssBackup.cs new file mode 100644 index 000000000..37d901dd8 --- /dev/null +++ b/Duplicati/Library/Snapshots/Windows/WmicVssBackup.cs @@ -0,0 +1,139 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Management; +using System.Runtime.Versioning; + +namespace Duplicati.Library.Snapshots.Windows; + +/// +/// Snapshot provider based on WMIC +/// +[SupportedOSPlatform("windows")] +internal class WmicVssBackup : ISnapshotProvider +{ + /// + /// The manager that contains the actual interface to WMIC + /// + private WmicShadowCopyManager _manager = new WmicShadowCopyManager(); + + /// + public Guid AddToSnapshotSet(string drive) + => _manager.Add(drive).ParsedId; + + /// + public void BackupComplete() + { + } + + /// + public void DeleteSnapshot(Guid shadowId, bool forceDelete) + { + foreach (var x in _manager.ShadowCopies.Where(x => x.ParsedId == shadowId)) + x.Dispose(); + } + + /// + public void DisableWriterClasses(Guid[] guids) + { + } + + /// + public void Dispose() + { + _manager.Dispose(); + } + + /// + public void DoSnapshotSet() + { + } + + /// + public void EnableWriterClasses(Guid[] guids) + { + } + + /// + public void FreeWriterMetadata() + { + } + + /// + public void GatherWriterMetadata() + { + } + + /// + /// Stub for returning the snapshot info + /// + /// The path where the snapshot is located + private sealed record SnapshotInfo (string SnapshotDeviceObject) : ISnapshotInfo; + + /// + public ISnapshotInfo GetSnapshotProperties(Guid shadowId) + => new SnapshotInfo(_manager.ShadowCopies.First(x => x.ParsedId == shadowId).MappedPath); + + /// + public bool IsVolumeSupported(string drive) + { + if (string.IsNullOrWhiteSpace(drive)) + return false; + + try + { + var query = $"SELECT * FROM Win32_ShadowCopy WHERE VolumeName LIKE '%{drive[0]}%'"; + var searcher = new ManagementObjectSearcher(query); + var results = searcher.Get(); + + return results.Count > 0; + } + catch (Exception ex) + { + Console.WriteLine($"Error checking VSS support: {ex.Message}"); + return false; + } + } + + /// + public IEnumerable ParseWriterMetaData(Guid[] writers) + => Enumerable.Empty(); + + /// + public void PrepareForBackup() + { + } + + /// + public void StartSnapshotSet() + { + } + + /// + public void VerifyWriters(Guid[] guids) + { + } +} \ No newline at end of file diff --git a/Duplicati/Library/Snapshots/WindowsSnapshot.cs b/Duplicati/Library/Snapshots/WindowsSnapshot.cs index 08bd5e1ae..1c1e70416 100644 --- a/Duplicati/Library/Snapshots/WindowsSnapshot.cs +++ b/Duplicati/Library/Snapshots/WindowsSnapshot.cs @@ -19,7 +19,6 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. - using System; using System.Collections.Generic; using System.IO; @@ -27,6 +26,7 @@ using System.Linq; using System.Runtime.Versioning; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; +using Duplicati.Library.Snapshots.Windows; namespace Duplicati.Library.Snapshots { @@ -53,7 +53,7 @@ namespace Duplicati.Library.Snapshots /// /// The main reference to the backup controller /// - private readonly VssBackupComponents _vssBackupComponents; + private readonly SnapshotManager _snapshotManager; /// /// The source folders included in the snapshot @@ -73,7 +73,8 @@ namespace Duplicati.Library.Snapshots _sourceEntries = sources.Select(SystemIOWindows.RemoveExtendedDevicePathPrefix).ToList(); try { - _vssBackupComponents = new VssBackupComponents(); + var provider = Utility.Utility.ParseEnumOption(options.AsReadOnly(), "snapshot-provider", SnapshotProvider.AlphaVSS); + _snapshotManager = new SnapshotManager(provider); // Default to exclude the System State writer var excludedWriters = new Guid[] { new Guid("{e8132975-6f93-4464-a53e-1050253ae220}") }; @@ -85,16 +86,16 @@ namespace Duplicati.Library.Snapshots .Select(x => new Guid(x)) .ToArray(); } - _vssBackupComponents.SetupWriters(null, excludedWriters); + _snapshotManager.SetupWriters(null, excludedWriters); - _vssBackupComponents.InitShadowVolumes(_sourceEntries); + _snapshotManager.InitShadowVolumes(_sourceEntries); - _vssBackupComponents.MapVolumesToSnapShots(); + _snapshotManager.MapVolumesToSnapShots(); //If we should map the drives, we do that now and update the volumeMap if (Utility.Utility.ParseBoolOption(options.AsReadOnly(), "vss-use-mapping")) { - _vssBackupComponents.MapDrives(); + _snapshotManager.MapDrives(); } } catch (Exception ex1) @@ -279,7 +280,7 @@ namespace Duplicati.Library.Snapshots if (!Path.IsPathRooted(snapshotPath)) throw new InvalidOperationException(); - foreach (var kvp in _vssBackupComponents.SnapshotDeviceAndVolumes) + foreach (var kvp in _snapshotManager.SnapshotDeviceAndVolumes) { if (snapshotPath.StartsWith(kvp.Key, Utility.Utility.ClientFilenameStringComparison)) return IO_WIN.PathCombine(kvp.Value, snapshotPath.Substring(kvp.Key.Length)); @@ -298,7 +299,7 @@ namespace Duplicati.Library.Snapshots throw new InvalidOperationException(); var root = IO_WIN.GetPathRoot(localPath); - var volumePath = _vssBackupComponents.GetVolumeFromCache(root); + var volumePath = _snapshotManager.GetVolumeFromCache(root); // Note that using a simple Path.Combine() for the following code // can result in invalid snapshot paths; e.g., if localPath is @@ -333,7 +334,7 @@ namespace Duplicati.Library.Snapshots { if (disposing) { - _vssBackupComponents?.Dispose(); + _snapshotManager?.Dispose(); } base.Dispose(disposing); diff --git a/Duplicati/WebserverCore/Endpoints/V1/HyperV.cs b/Duplicati/WebserverCore/Endpoints/V1/HyperV.cs index cde404bae..98f8fed9e 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/HyperV.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/HyperV.cs @@ -21,6 +21,7 @@ using System.Runtime.Versioning; using System.Security.Principal; using Duplicati.Library.Snapshots; +using Duplicati.Library.Snapshots.Windows; using Duplicati.WebserverCore.Abstractions; using Duplicati.WebserverCore.Exceptions; using Microsoft.AspNetCore.Mvc; @@ -58,7 +59,7 @@ public class HyperV : IEndpointV1 { if (string.IsNullOrEmpty(key)) { - hypervUtility.QueryHyperVGuestsInfo(); + hypervUtility.QueryHyperVGuestsInfo(SnapshotProvider.AlphaVSS); return hypervUtility.Guests.Select(x => new Dto.TreeNodeDto() { id = x.ID.ToString(), @@ -77,7 +78,7 @@ public class HyperV : IEndpointV1 } else { - hypervUtility.QueryHyperVGuestsInfo(true); + hypervUtility.QueryHyperVGuestsInfo(SnapshotProvider.AlphaVSS, true); var foundVMs = hypervUtility.Guests.FindAll(x => x.ID.Equals(new Guid(key))); if (foundVMs.Count != 1) diff --git a/Duplicati/WebserverCore/Endpoints/V1/MSSql.cs b/Duplicati/WebserverCore/Endpoints/V1/MSSql.cs index 0a2b98679..b4f4e6607 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/MSSql.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/MSSql.cs @@ -21,6 +21,7 @@ using System.Runtime.Versioning; using System.Security.Principal; using Duplicati.Library.Snapshots; +using Duplicati.Library.Snapshots.Windows; using Duplicati.WebserverCore.Abstractions; using Duplicati.WebserverCore.Exceptions; using Microsoft.AspNetCore.Mvc; @@ -56,7 +57,7 @@ public class MSSql : IEndpointV1 try { - mssqlUtility.QueryDBsInfo(); + mssqlUtility.QueryDBsInfo(SnapshotProvider.AlphaVSS); if (string.IsNullOrEmpty(key)) { From 36ba547a92b8f98797aa98dd4175ea30c81371c6 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 19 Feb 2025 14:20:03 +0100 Subject: [PATCH 201/362] Added the @ prefix to backend paths --- Duplicati/Library/Backend/CIFS/CIFS.cs | 2 +- Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs | 2 +- Duplicati/Library/Backend/S3/S3Backend.cs | 2 +- Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs | 2 +- Duplicati/Library/Interface/IFolderEnabledBackend.cs | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Duplicati/Library/Backend/CIFS/CIFS.cs b/Duplicati/Library/Backend/CIFS/CIFS.cs index eafcdfc45..549721edf 100644 --- a/Duplicati/Library/Backend/CIFS/CIFS.cs +++ b/Duplicati/Library/Backend/CIFS/CIFS.cs @@ -190,7 +190,7 @@ public class CIFSBackend : IStreamingBackend, IFolderEnabledBackend writeBufferSize ); - _sourcePathKey = $"{ProtocolKey}://{_connectionParameters.ServerName}/~{_connectionParameters.AuthUser}/{_connectionParameters.Path}/"; + _sourcePathKey = $"@{ProtocolKey}://{_connectionParameters.ServerName}/~{_connectionParameters.AuthUser}/{_connectionParameters.Path}/"; _sourceProvider = new BackendSourceProvider(this); } diff --git a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs index f495ddf52..a3c5d47be 100644 --- a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs +++ b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs @@ -90,7 +90,7 @@ namespace Duplicati.Library.Backend m_s3Client = new S3AwsClient(accessKeyId, accessKeySecret, null, host, null, true, false, options); - m_sourcePathKey = $"{ProtocolKey}://{host}/{m_bucket}/{m_prefix}/"; + m_sourcePathKey = $"@{ProtocolKey}://{host}/{m_bucket}/{m_prefix}/"; m_sourceProvider = new BackendSourceProvider(this); } diff --git a/Duplicati/Library/Backend/S3/S3Backend.cs b/Duplicati/Library/Backend/S3/S3Backend.cs index e6d4d99bb..92d4029d1 100644 --- a/Duplicati/Library/Backend/S3/S3Backend.cs +++ b/Duplicati/Library/Backend/S3/S3Backend.cs @@ -256,7 +256,7 @@ namespace Duplicati.Library.Backend throw new UserInformationException(Strings.S3Backend.UnknownS3ClientError(s3ClientOptionValue), "UnknownS3Client"); } - m_sourcePathKey = $"{ProtocolKey}://{hostname}/{m_bucket}/{m_prefix}/"; + m_sourcePathKey = $"@{ProtocolKey}://{hostname}/{m_bucket}/{m_prefix}/"; m_sourceProvider = new BackendSourceProvider(this); } diff --git a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs index d8057f651..e9abbe27e 100644 --- a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs +++ b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs @@ -118,7 +118,7 @@ namespace Duplicati.Library.Backend if (!string.IsNullOrWhiteSpace(timeoutstr)) m_keepaliveinterval = Library.Utility.Timeparser.ParseTimeSpan(timeoutstr); - m_sourcePathKey = $"{ProtocolKey}://{m_server}:{m_port}/~{m_username}/{m_path}/"; + m_sourcePathKey = $"@{ProtocolKey}://{m_server}:{m_port}/~{m_username}/{m_path}/"; m_sourceProvider = new BackendSourceProvider(this); } diff --git a/Duplicati/Library/Interface/IFolderEnabledBackend.cs b/Duplicati/Library/Interface/IFolderEnabledBackend.cs index 4f83178ad..4b9c6047a 100644 --- a/Duplicati/Library/Interface/IFolderEnabledBackend.cs +++ b/Duplicati/Library/Interface/IFolderEnabledBackend.cs @@ -35,7 +35,7 @@ namespace Duplicati.Library.Interface; public interface IFolderEnabledBackend : IBackend { /// - /// Returns the key for this backend instance's path + /// Returns the key for this backend instance's path, should be prefixed with a "@" /// string PathKey { get; } From bf808452923a0950831fd5f76ceed2ebfdfbf298 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 19 Feb 2025 15:46:22 +0100 Subject: [PATCH 202/362] Fixed a rare startup error --- Duplicati/Library/RestAPI/Scheduler.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/RestAPI/Scheduler.cs b/Duplicati/Library/RestAPI/Scheduler.cs index 281efc0fd..037af95cf 100644 --- a/Duplicati/Library/RestAPI/Scheduler.cs +++ b/Duplicati/Library/RestAPI/Scheduler.cs @@ -152,7 +152,7 @@ namespace Duplicati.Server /// public List WorkerQueue { - get { return (from t in m_worker?.CurrentTasks where t != null select t)?.ToList() ?? []; } + get { return m_worker?.CurrentTasks?.Where(t => t != null)?.ToList() ?? []; } } /// From b1488dae0cd2e849fb8c6c7f935dcfff8f1ccecf Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 19 Feb 2025 15:48:00 +0100 Subject: [PATCH 203/362] Switched to `vssadmin` for listing VSS enabled drives --- .../Windows/WmicShadowCopyManager.cs | 27 ++++++++++++++++++- .../Snapshots/Windows/WmicVssBackup.cs | 20 +++++--------- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Snapshots/Windows/WmicShadowCopyManager.cs b/Duplicati/Library/Snapshots/Windows/WmicShadowCopyManager.cs index 45a1ddd6b..ab54530b8 100644 --- a/Duplicati/Library/Snapshots/Windows/WmicShadowCopyManager.cs +++ b/Duplicati/Library/Snapshots/Windows/WmicShadowCopyManager.cs @@ -18,7 +18,7 @@ internal class WmicShadowCopyManager : IDisposable /// /// The tag used for logging messages /// - public static readonly string LOGTAG = Logging.Log.LogTagFromType(); + private static readonly string LOGTAG = Logging.Log.LogTagFromType(); /// /// A single shadow copy @@ -159,6 +159,31 @@ internal class WmicShadowCopyManager : IDisposable return null; } + /// + /// Returns the drives that are vss enabled + /// + /// + public static HashSet GetVssCapableDrivesViaVssadmin() + { + var vssDrives = new HashSet(StringComparer.OrdinalIgnoreCase); + + try + { + var output = ExecuteCommand("vssadmin", "list volumes", 5000); + + // Regex to match drive letters in output + var matches = Regex.Matches(output, @"Volume\s+path:\s*([A-Z]:)\\?\s", RegexOptions.IgnoreCase); + foreach (Match match in matches) + vssDrives.Add(match.Groups[1].Value.Substring(0, 1)); + } + catch (Exception ex) + { + Logging.Log.WriteErrorMessage(LOGTAG, "ShadowCopyListFailed", ex, "Failed to list volumes"); + } + + return vssDrives; + } + /// /// Delete a shadow copy /// diff --git a/Duplicati/Library/Snapshots/Windows/WmicVssBackup.cs b/Duplicati/Library/Snapshots/Windows/WmicVssBackup.cs index 37d901dd8..bf7568ed0 100644 --- a/Duplicati/Library/Snapshots/Windows/WmicVssBackup.cs +++ b/Duplicati/Library/Snapshots/Windows/WmicVssBackup.cs @@ -24,7 +24,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Management; using System.Runtime.Versioning; namespace Duplicati.Library.Snapshots.Windows; @@ -40,6 +39,11 @@ internal class WmicVssBackup : ISnapshotProvider /// private WmicShadowCopyManager _manager = new WmicShadowCopyManager(); + /// + /// Gets the VSS enabled drives + /// + private Lazy> _vssEnabledDrives = new Lazy>(() => WmicShadowCopyManager.GetVssCapableDrivesViaVssadmin()); + /// public Guid AddToSnapshotSet(string drive) => _manager.Add(drive).ParsedId; @@ -102,20 +106,8 @@ internal class WmicVssBackup : ISnapshotProvider { if (string.IsNullOrWhiteSpace(drive)) return false; - - try - { - var query = $"SELECT * FROM Win32_ShadowCopy WHERE VolumeName LIKE '%{drive[0]}%'"; - var searcher = new ManagementObjectSearcher(query); - var results = searcher.Get(); - return results.Count > 0; - } - catch (Exception ex) - { - Console.WriteLine($"Error checking VSS support: {ex.Message}"); - return false; - } + return _vssEnabledDrives.Value.Contains(drive.Substring(0, 1)); } /// From 864ce59c4a159bdf0181e2e4c0e143dce51e48cb Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 19 Feb 2025 15:55:13 +0100 Subject: [PATCH 204/362] Reworded pending uploads to active to better reflect that the uploads are in progress --- .../Main/Backend/BackendManager.Handler.cs | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Duplicati/Library/Main/Backend/BackendManager.Handler.cs b/Duplicati/Library/Main/Backend/BackendManager.Handler.cs index 915d12f4d..aa1185405 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.Handler.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.Handler.cs @@ -91,7 +91,7 @@ partial class BackendManager /// /// The list of active uploads /// - private readonly List activeUploads = new(); + private readonly List activeUploads = []; /// /// The pool of backends currently created /// @@ -198,11 +198,11 @@ partial class BackendManager } /// - /// Ensures that there are at most N - 1 pending uploads + /// Ensures that there are at most N - 1 active uploads /// - /// The maximum number of pending uploads + /// The maximum number of active uploads /// An awaitable task - private async Task EnsureAtMostNPendingUploads(int n) + private async Task EnsureAtMostNActiveUploads(int n) { while (activeUploads.Count >= n) { @@ -234,7 +234,7 @@ partial class BackendManager // Allow PUT operations to be queued, if requested if (op is PutOperation putOp && !putOp.WaitForComplete) { - await EnsureAtMostNPendingUploads(maxParallelUploads).ConfigureAwait(false); + await EnsureAtMostNActiveUploads(maxParallelUploads).ConfigureAwait(false); // Operation is accepted into queue, so we can signal completion putOp.SetComplete(true); @@ -242,8 +242,8 @@ partial class BackendManager } else { - // Wait for all pending uploads to complete - await EnsureAtMostNPendingUploads(1).ConfigureAwait(false); + // Wait for all of the active uploads to complete + await EnsureAtMostNActiveUploads(1).ConfigureAwait(false); // Execute the operation await ExecuteWithRetry(op, tcs.Token).ConfigureAwait(false); } @@ -259,25 +259,25 @@ partial class BackendManager } finally { - // Terminate any pending uploads + // Terminate any active uploads tcs.Cancel(); if (activeUploads.Count > 0) { - Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerDisposeWhileActive", null, "Terminating {0} pending uploads", activeUploads.Count); + Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerDisposeWhileActive", null, "Terminating {0} active uploads", activeUploads.Count); - // Wait for all pending uploads to complete + // Wait for all active uploads to complete await Task.WhenAny(Task.Delay(1000), Task.WhenAll(activeUploads)).ConfigureAwait(false); for (int i = activeUploads.Count - 1; i >= 0; i--) { var t = activeUploads[i]; activeUploads.RemoveAt(i); if (t.IsCompleted && !t.IsCanceled && t.Exception != null) - Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerDisposeError", t.Exception, "Error in pending upload: {0}", t.Exception.Message); + Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerDisposeError", t.Exception, "Error in active upload: {0}", t.Exception.Message); } if (activeUploads.Count > 0) - Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerDisposeError", null, "Terminating, but {0} pending uploads are still active", activeUploads.Count); + Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerDisposeError", null, "Terminating, but {0} active uploads are still active", activeUploads.Count); } // Dispose of any remaining backends From e2e930b2d7ff3082f0ebcf7278334e450b67200e Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 19 Feb 2025 15:55:59 +0100 Subject: [PATCH 205/362] Reworked BackendManager.Handler to handle multiple downloads in parallel --- .../Main/Backend/BackendManager.Handler.cs | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/Duplicati/Library/Main/Backend/BackendManager.Handler.cs b/Duplicati/Library/Main/Backend/BackendManager.Handler.cs index aa1185405..1115d6de7 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.Handler.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.Handler.cs @@ -88,6 +88,10 @@ partial class BackendManager /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(); + /// + /// The list of active downloads + /// + private readonly List activeDownloads = []; /// /// The list of active uploads /// @@ -105,6 +109,10 @@ partial class BackendManager /// private readonly ExecuteContext context; /// + /// The maximum number of parallel downloads + /// + private readonly int maxParallelDownloads; + /// /// The maximum number of parallel uploads /// private readonly int maxParallelUploads; @@ -155,6 +163,8 @@ partial class BackendManager this.backendUrl = backendUrl; this.context = context; + // TODO Currently, only the restore process uses parallel downloads. If others need it as well, maybe use another option. + maxParallelDownloads = Math.Max(1, context.Options.RestoreVolumeDownloaders); maxParallelUploads = Math.Max(1, context.Options.AsynchronousConcurrentUploadLimit); maxRetries = context.Options.NumberOfRetries; retryDelay = context.Options.RetryDelay; @@ -180,6 +190,23 @@ partial class BackendManager ); } + /// + /// Reclaims completed downloads + /// + /// An awaitable task + private async Task ReclaimCompletedDownloads() + { + for (int i = activeDownloads.Count - 1; i >= 0; i--) + { + if (activeDownloads[i].IsCompleted) + { + // Make sure the task is awaited so we capture any exceptions + await activeDownloads[i].ConfigureAwait(false); + activeDownloads.RemoveAt(i); + } + } + } + /// /// Reclaims completed uploads /// @@ -197,6 +224,20 @@ partial class BackendManager } } + /// + /// Ensures that there are at most N - 1 active downloads + /// + /// + /// + private async Task EnsureAtMostNActiveDownloads(int n) + { + while (activeDownloads.Count >= n) + { + await Task.WhenAny(activeDownloads).ConfigureAwait(false); + await ReclaimCompletedDownloads().ConfigureAwait(false); + } + } + /// /// Ensures that there are at most N - 1 active uploads /// @@ -240,6 +281,13 @@ partial class BackendManager putOp.SetComplete(true); activeUploads.Add(ExecuteWithRetry(putOp, tcs.Token)); } + else if (op is GetOperation getOp && !getOp.WaitForComplete) + { + await EnsureAtMostNActiveDownloads(maxParallelDownloads).ConfigureAwait(false); + + // Operation is accepted into queue, so we can signal completion + activeDownloads.Add(ExecuteWithRetry(getOp, tcs.Token)); + } else { // Wait for all of the active uploads to complete From 2c068202811fad46a79ff43fd9999abcac790bb0 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 19 Feb 2025 16:08:18 +0100 Subject: [PATCH 206/362] Made the BackendManager.GetOperations' decryption methods static and public so the restores VolumeDecryptor can use the same code. --- .../Backend/BackendManager.GetOperation.cs | 40 +- .../Library/Main/Backend/BackendManager.cs | 12 + Duplicati/Library/Main/IBackendManager.cs | 9 + .../Main/Operation/Restore/VolumeDecryptor.cs | 88 +- .../Library/Main/Operation/RestoreHandler.cs | 2360 ++++++++--------- 5 files changed, 1229 insertions(+), 1280 deletions(-) diff --git a/Duplicati/Library/Main/Backend/BackendManager.GetOperation.cs b/Duplicati/Library/Main/Backend/BackendManager.GetOperation.cs index 3d3855ef4..0381f1833 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.GetOperation.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.GetOperation.cs @@ -64,11 +64,6 @@ partial class BackendManager { TempFile? tmpfile = null; Context.Statwriter.SendEvent(BackendActionType.Get, BackendEventType.Started, RemoteFilename, Size); - using var encryption = Context.Options.NoEncryption - ? null - : (DynamicLoader.EncryptionLoader.GetModule(Context.Options.EncryptionModule, Context.Options.Passphrase, Context.Options.RawOptions) - ?? throw new Exception(Strings.BackendMananger.EncryptionModuleNotFound(Context.Options.EncryptionModule)) - ); try { @@ -94,7 +89,7 @@ partial class BackendManager } // Perform decryption after hash validation, if needed - tmpfile = DecryptFile(tmpfile, DetectEncryptionModule(encryption)); + tmpfile = DecryptFile(tmpfile, RemoteFilename, Context.Options); return (tmpfile, fileHash, dataSizeDownloaded); } @@ -161,40 +156,40 @@ partial class BackendManager /// /// The default encryption module /// The encryption module to use - private IEncryption? DetectEncryptionModule(IEncryption? encryption) + private static IEncryption? DetectEncryptionModule(string filename, Options options, IEncryption? encryption) { try { // Auto-guess the encryption module - var ext = (System.IO.Path.GetExtension(RemoteFilename) ?? "").TrimStart('.'); + var ext = (System.IO.Path.GetExtension(filename) ?? "").TrimStart('.'); if (!ext.Equals(encryption?.FilenameExtension, StringComparison.OrdinalIgnoreCase)) { // Check if the file is not encrypted if (DynamicLoader.CompressionLoader.Keys.Contains(ext, StringComparer.OrdinalIgnoreCase)) { if (encryption != null) - Logging.Log.WriteVerboseMessage(LOGTAG, "AutomaticDecryptionDetection", "Filename extension \"{0}\" does not match encryption module \"{1}\", guessing that it is not encrypted", ext, Context.Options.EncryptionModule); + Logging.Log.WriteVerboseMessage(LOGTAG, "AutomaticDecryptionDetection", "Filename extension \"{0}\" does not match encryption module \"{1}\", guessing that it is not encrypted", ext, options.EncryptionModule); return null; } // Check if the file is encrypted with something else else if (DynamicLoader.EncryptionLoader.Keys.Contains(ext, StringComparer.OrdinalIgnoreCase)) { - Logging.Log.WriteVerboseMessage(LOGTAG, "AutomaticDecryptionDetection", "Filename extension \"{0}\" does not match encryption module \"{1}\", attempting to use matching encryption module", ext, Context.Options.EncryptionModule); + Logging.Log.WriteVerboseMessage(LOGTAG, "AutomaticDecryptionDetection", "Filename extension \"{0}\" does not match encryption module \"{1}\", attempting to use matching encryption module", ext, options.EncryptionModule); try { - return DynamicLoader.EncryptionLoader.GetModule(ext, Context.Options.Passphrase, Context.Options.RawOptions) + return DynamicLoader.EncryptionLoader.GetModule(ext, options.Passphrase, options.RawOptions) ?? encryption; } catch (Exception ex) { - Logging.Log.WriteWarningMessage(LOGTAG, "AutomaticDecryptionDetection", ex, "Failed to load encryption module \"{0}\", using specified encryption module \"{1}\"", ext, Context.Options.EncryptionModule); + Logging.Log.WriteWarningMessage(LOGTAG, "AutomaticDecryptionDetection", ex, "Failed to load encryption module \"{0}\", using specified encryption module \"{1}\"", ext, options.EncryptionModule); } } // Fallback, lets see what happens... else { - Logging.Log.WriteVerboseMessage(LOGTAG, "AutomaticDecryptionDetection", "Filename extension \"{0}\" does not match encryption module \"{1}\", attempting to use specified encryption module as no others match", ext, Context.Options.EncryptionModule); + Logging.Log.WriteVerboseMessage(LOGTAG, "AutomaticDecryptionDetection", "Filename extension \"{0}\" does not match encryption module \"{1}\", attempting to use specified encryption module as no others match", ext, options.EncryptionModule); } } @@ -213,7 +208,7 @@ partial class BackendManager /// The encrypted file /// Then encryption module to use, or null for no encryption /// The decrypted file - private TempFile DecryptFile(TempFile tempFile, IEncryption? decrypter) + private static TempFile DecryptFile(TempFile tempFile, IEncryption? decrypter) { // Support no encryption if (decrypter == null) @@ -244,5 +239,22 @@ partial class BackendManager } } } + + /// + /// Decrypts a file using the specified options + /// + /// The file to decrypt + /// The name of the file. Used for detecting encryption algorithm if not specified in options or if it differs from the options + /// The Duplicati options + /// The decrypted file + public static TempFile DecryptFile(TempFile tmpfile, string filename, Options options) + { + using var encryption = options.NoEncryption + ? null + : (DynamicLoader.EncryptionLoader.GetModule(options.EncryptionModule, options.Passphrase, options.RawOptions) + ?? throw new Exception(Strings.BackendMananger.EncryptionModuleNotFound(options.EncryptionModule)) + ); + return DecryptFile(tmpfile, DetectEncryptionModule(filename, options, encryption)); + } } } \ No newline at end of file diff --git a/Duplicati/Library/Main/Backend/BackendManager.cs b/Duplicati/Library/Main/Backend/BackendManager.cs index 5813cdf79..a9762e5be 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.cs @@ -139,6 +139,18 @@ internal partial class BackendManager : IBackendManager return Convert.ToBase64String(hasher.ComputeHash(fs)); } + /// + /// Decrypts a file using the specified options + /// + /// The file to decrypt + /// The name of the file. Used for detecting encryption algorithm if not specified in options or if it differs from the options + /// The Duplicati options + /// The decrypted file + public TempFile DecryptFile(TempFile volume, string volume_name, Options options) + { + return GetOperation.DecryptFile(volume, volume_name, options); + } + /// /// Deletes a remote file /// diff --git a/Duplicati/Library/Main/IBackendManager.cs b/Duplicati/Library/Main/IBackendManager.cs index df92577a3..2d01d1c8b 100644 --- a/Duplicati/Library/Main/IBackendManager.cs +++ b/Duplicati/Library/Main/IBackendManager.cs @@ -54,6 +54,15 @@ internal interface IBackendManager : IDisposable /// An enumerable of file entries Task> ListAsync(CancellationToken cancelToken); + /// + /// Decrypts the given file and returns the decrypted file + /// + /// The file to decrypt + /// The name of the file. Used for detecting encryption algorithm if not specified in options or if it differs from the options + /// The Duplicati options + /// The decrypted file + TempFile DecryptFile(TempFile volume, string volume_name, Options options); + /// /// Deletes a file on the backend /// diff --git a/Duplicati/Library/Main/Operation/Restore/VolumeDecryptor.cs b/Duplicati/Library/Main/Operation/Restore/VolumeDecryptor.cs index 59576fac9..2555cbf0e 100644 --- a/Duplicati/Library/Main/Operation/Restore/VolumeDecryptor.cs +++ b/Duplicati/Library/Main/Operation/Restore/VolumeDecryptor.cs @@ -43,7 +43,7 @@ namespace Duplicati.Library.Main.Operation.Restore /// Runs the volume decryptor process. /// /// The restore options. - public static Task Run(Channels channels, Options options) + public static Task Run(Channels channels, IBackendManager backend, Options options) { return AutomationExtensions.RunTask( new @@ -58,13 +58,6 @@ namespace Duplicati.Library.Main.Operation.Restore Stopwatch sw_decrypt = options.InternalProfiling ? new() : null; try { - // Taken from BackendManager.GetOperation - using var encryption = options.NoEncryption - ? null - : (DynamicLoader.EncryptionLoader.GetModule(options.EncryptionModule, options.Passphrase, options.RawOptions) - ?? throw new Exception(Strings.BackendMananger.EncryptionModuleNotFound(options.EncryptionModule)) - ); - while (true) { // Get the block request and volume from the `VolumeDownloader` process. @@ -73,7 +66,7 @@ namespace Duplicati.Library.Main.Operation.Restore sw_read?.Stop(); sw_decrypt?.Start(); - var tmpfile = DecryptFile(volume, DetectEncryptionModule(volume_name, options, encryption)); + var tmpfile = backend.DecryptFile(volume, volume_name, options); var bvr = new BlockVolumeReader(options.CompressionModule, tmpfile, options); sw_decrypt?.Stop(); @@ -99,83 +92,6 @@ namespace Duplicati.Library.Main.Operation.Restore } }); } - - // Taken from BackendManager.GetOperation - public static Library.Utility.TempFile DecryptFile(Library.Utility.TempFile tempFile, Interface.IEncryption? decrypter) - { - // Support no encryption - if (decrypter == null) - return tempFile; - - Library.Utility.TempFile? decryptTarget = null; - - // Always dispose the source file - using (tempFile) - using (new Logging.Timer(LOGTAG, "DecryptFile", "Decrypting " + tempFile)) - { - try - { - decryptTarget = new Library.Utility.TempFile(); - try { decrypter.Decrypt(tempFile, decryptTarget); } - // If we fail here, make sure that we throw a crypto exception - catch (System.Security.Cryptography.CryptographicException) { throw; } - catch (Exception ex) { throw new System.Security.Cryptography.CryptographicException(ex.Message, ex); } - - var result = decryptTarget; - decryptTarget = null; - return result; - } - finally - { - // Remove temp files on failure - decryptTarget?.Dispose(); - } - } - } - - private static Interface.IEncryption? DetectEncryptionModule(string remotefilename, Options options, Interface.IEncryption? encryption) - { - try - { - // Auto-guess the encryption module - var ext = (System.IO.Path.GetExtension(remotefilename) ?? "").TrimStart('.'); - if (!ext.Equals(encryption?.FilenameExtension, StringComparison.OrdinalIgnoreCase)) - { - // Check if the file is not encrypted - if (DynamicLoader.CompressionLoader.Keys.Contains(ext, StringComparer.OrdinalIgnoreCase)) - { - if (encryption != null) - Logging.Log.WriteVerboseMessage(LOGTAG, "AutomaticDecryptionDetection", "Filename extension \"{0}\" does not match encryption module \"{1}\", guessing that it is not encrypted", ext, options.EncryptionModule); - return null; - } - // Check if the file is encrypted with something else - else if (DynamicLoader.EncryptionLoader.Keys.Contains(ext, StringComparer.OrdinalIgnoreCase)) - { - Logging.Log.WriteVerboseMessage(LOGTAG, "AutomaticDecryptionDetection", "Filename extension \"{0}\" does not match encryption module \"{1}\", attempting to use matching encryption module", ext, options.EncryptionModule); - - try - { - return DynamicLoader.EncryptionLoader.GetModule(ext, options.Passphrase, options.RawOptions) - ?? encryption; - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "AutomaticDecryptionDetection", ex, "Failed to load encryption module \"{0}\", using specified encryption module \"{1}\"", ext, options.EncryptionModule); - } - } - // Fallback, lets see what happens... - else - { - Logging.Log.WriteVerboseMessage(LOGTAG, "AutomaticDecryptionDetection", "Filename extension \"{0}\" does not match encryption module \"{1}\", attempting to use specified encryption module as no others match", ext, options.EncryptionModule); - } - } - - return encryption; - } - // If we fail here, make sure that we throw a crypto exception - catch (System.Security.Cryptography.CryptographicException) { throw; } - catch (Exception ex) { throw new System.Security.Cryptography.CryptographicException(ex.Message, ex); } - } } } \ No newline at end of file diff --git a/Duplicati/Library/Main/Operation/RestoreHandler.cs b/Duplicati/Library/Main/Operation/RestoreHandler.cs index c119e47a3..4b9bdc07a 100644 --- a/Duplicati/Library/Main/Operation/RestoreHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreHandler.cs @@ -1,1180 +1,1180 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - -using System; -using System.Collections.Generic; -using System.Linq; -using Duplicati.Library.Interface; -using Duplicati.Library.Common.IO; -using Duplicati.Library.Main.Database; -using Duplicati.Library.Main.Volumes; -using Duplicati.Library.Utility; -using CoCoL; -using System.Threading.Tasks; -using System.Threading; - -namespace Duplicati.Library.Main.Operation -{ - internal class RestoreHandler - { - /// - /// The tag used for logging - /// - private static readonly string LOGTAG = Logging.Log.LogTagFromType(); - - private readonly Options m_options; - private byte[] m_blockbuffer; - private readonly RestoreResults m_result; - private static readonly string DIRSEP = Util.DirectorySeparatorString; - - public RestoreHandler(Options options, RestoreResults result) - { - m_options = options; - m_result = result; - } - - /// - /// Gets the compression module by parsing the filename - /// - /// The filename to parse - /// The compression module - public static string GetCompressionModule(string filename) - { - var tmp = VolumeBase.ParseFilename(filename); - if (tmp == null) - throw new UserInformationException(string.Format("Unable to parse filename to valid entry: {0}", filename), "FailedToParseRemoteName"); - - return tmp.CompressionModule; - } - - public static RecreateDatabaseHandler.NumberedFilterFilelistDelegate FilterNumberedFilelist(DateTime time, long[] versions, bool singleTimeMatch = false) - { - if (time.Kind == DateTimeKind.Unspecified) - throw new Exception("Unspecified datetime instance, must be either local or UTC"); - - // Make sure the resolution is the same (i.e. no milliseconds) - if (time.Ticks > 0) - time = Library.Utility.Utility.DeserializeDateTime(Library.Utility.Utility.SerializeDateTime(time)).ToUniversalTime(); - - return - _lst => - { - // Unwrap, so we do not query the remote storage twice - var lst = (from n in _lst - where n.FileType == RemoteVolumeType.Files - orderby n.Time descending - select n).ToArray(); - - var numbers = lst.Zip(Enumerable.Range(0, lst.Length), (a, b) => new KeyValuePair(b, a)).ToList(); - - if (time.Ticks > 0 && versions != null && versions.Length > 0) - return from n in numbers - where (singleTimeMatch ? n.Value.Time == time : n.Value.Time <= time) && versions.Contains(n.Key) - select n; - else if (time.Ticks > 0) - return from n in numbers - where (singleTimeMatch ? n.Value.Time == time : n.Value.Time <= time) - select n; - else if (versions != null && versions.Length > 0) - return from n in numbers - where versions.Contains(n.Key) - select n; - else - return numbers; - }; - } - - public void Run(string[] paths, IBackendManager backendManager, Library.Utility.IFilter filter) - { - m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_Begin); - - // If we have both target paths and a filter, combine them into a single filter - filter = JoinedFilterExpression.Join(new FilterExpression(paths), filter); - - LocalRestoreDatabase db = null; - TempFile tmpdb = null; - try - { - if (!m_options.NoLocalDb && SystemIO.IO_OS.FileExists(m_options.Dbpath)) - { - db = new LocalRestoreDatabase(m_options.Dbpath); - db.SetResult(m_result); - } - else - { - Logging.Log.WriteInformationMessage(LOGTAG, "NoLocalDatabase", "No local database, building a temporary database"); - tmpdb = new TempFile(); - RecreateDatabaseHandler.NumberedFilterFilelistDelegate filelistfilter = FilterNumberedFilelist(m_options.Time, m_options.Version); - db = new LocalRestoreDatabase(tmpdb); - m_result.RecreateDatabaseResults = new RecreateDatabaseResults(m_result); - using (new Logging.Timer(LOGTAG, "RecreateTempDbForRestore", "Recreate temporary database for restore")) - new RecreateDatabaseHandler(m_options, (RecreateDatabaseResults)m_result.RecreateDatabaseResults) - .DoRun(backendManager, db, false, filter, filelistfilter, null).Await(); - - if (!m_options.SkipMetadata) - ApplyStoredMetadata(m_options, new RestoreHandlerMetadataStorage()); - - //If we have --version set, we need to adjust, as the db has only the required versions - //TODO: Bit of a hack to set options that way - if (m_options.Version != null && m_options.Version.Length > 0) - m_options.RawOptions["version"] = string.Join(",", Enumerable.Range(0, m_options.Version.Length).Select(x => x.ToString())); - } - - if (m_options.RestoreLegacy) - DoRun(backendManager, db, filter, m_result.TaskControl.ProgressToken).Await(); - else - DoRunNew(backendManager, db, filter, m_result.TaskControl.ProgressToken).Await(); - - db.WriteResults(); - } - finally - { - db?.Dispose(); - tmpdb?.Dispose(); - } - } - - private static void PatchWithBlocklist(LocalRestoreDatabase database, BlockVolumeReader blocks, Options options, RestoreResults result, byte[] blockbuffer, RestoreHandlerMetadataStorage metadatastorage) - { - var blocksize = options.Blocksize; - var updateCounter = 0L; - var fullblockverification = options.FullBlockVerification; - - using (var blockhasher = HashFactory.CreateHasher(options.BlockHashAlgorithm)) - using (var blockmarker = database.CreateBlockMarker()) - using (var volumekeeper = database.GetMissingBlockData(blocks, options.Blocksize)) - { - foreach (var restorelist in volumekeeper.FilesWithMissingBlocks) - { - var targetpath = restorelist.Path; - - if (options.Dryrun) - { - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPatchFile", "Would patch file with remote data: {0}", targetpath); - } - else - { - Logging.Log.WriteVerboseMessage(LOGTAG, "PatchingFile", "Patching file with remote data: {0}", targetpath); - - try - { - var folderpath = SystemIO.IO_OS.PathGetDirectoryName(targetpath); - if (!options.Dryrun && !SystemIO.IO_OS.DirectoryExists(folderpath)) - { - Logging.Log.WriteWarningMessage(LOGTAG, "CreateMissingFolder", null, "Creating missing folder {0} for file {1}", folderpath, targetpath); - SystemIO.IO_OS.DirectoryCreate(folderpath); - } - - // TODO: Much faster if we iterate the volume and checks what blocks are used, - // because the compressors usually like sequential reading - using (var file = SystemIO.IO_OS.FileOpenWrite(targetpath)) - foreach (var targetblock in restorelist.Blocks) - { - file.Position = targetblock.Offset; - var size = blocks.ReadBlock(targetblock.Key, blockbuffer); - if (targetblock.Size == size) - { - var valid = !fullblockverification; - if (!valid) - { - var key = Convert.ToBase64String(blockhasher.ComputeHash(blockbuffer, 0, size)); - if (targetblock.Key == key) - valid = true; - else - Logging.Log.WriteWarningMessage(LOGTAG, "InvalidBlock", null, "Invalid block detected for {0}, expected hash: {1}, actual hash: {2}", targetpath, targetblock.Key, key); - } - - if (valid) - { - file.Write(blockbuffer, 0, size); - blockmarker.SetBlockRestored(restorelist.FileID, targetblock.Offset / blocksize, targetblock.Key, size, false); - } - } - else - { - Logging.Log.WriteWarningMessage(LOGTAG, "WrongBlockSize", null, "Block with hash {0} should have size {1} but has size {2}", targetblock.Key, targetblock.Size, size); - } - } - - if ((++updateCounter) % 20 == 0) - blockmarker.UpdateProcessed(result.OperationProgressUpdater); - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "PatchFailed", ex, "Failed to patch file: \"{0}\", message: {1}, message: {1}", targetpath, ex.Message); - if (options.UnittestMode) - throw; - } - } - } - - if (!options.SkipMetadata) - { - foreach (var restoremetadata in volumekeeper.MetadataWithMissingBlocks) - { - var targetpath = restoremetadata.Path; - Logging.Log.WriteVerboseMessage(LOGTAG, "RecordingMetadata", "Recording metadata from remote data: {0}", targetpath); - - try - { - // TODO: When we support multi-block metadata this needs to deal with it - using (var ms = new System.IO.MemoryStream()) - { - foreach (var targetblock in restoremetadata.Blocks) - { - ms.Position = targetblock.Offset; - var size = blocks.ReadBlock(targetblock.Key, blockbuffer); - if (targetblock.Size == size) - { - ms.Write(blockbuffer, 0, size); - blockmarker.SetBlockRestored(restoremetadata.FileID, targetblock.Offset / blocksize, targetblock.Key, size, true); - } - } - - ms.Position = 0; - metadatastorage.Add(targetpath, ms); - //blockmarker.RecordMetadata(restoremetadata.FileID, ms); - } - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "MetatdataRecordFailed", ex, "Failed to record metadata for file: \"{0}\", message: {1}", targetpath, ex.Message); - if (options.UnittestMode) - throw; - } - } - } - blockmarker.UpdateProcessed(result.OperationProgressUpdater); - blockmarker.Commit(); - } - } - - private static void ApplyStoredMetadata(Options options, RestoreHandlerMetadataStorage metadatastorage) - { - foreach (var metainfo in metadatastorage.Records) - { - var targetpath = metainfo.Key; - - if (options.Dryrun) - { - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPatchMetadata", "Would patch metadata with remote data: {0}", targetpath); - } - else - { - Logging.Log.WriteVerboseMessage(LOGTAG, "PatchingMetadata", "Patching metadata with remote data: {0}", targetpath); - try - { - var folderpath = Duplicati.Library.Utility.Utility.GetParent(targetpath, false); - if (!options.Dryrun && !SystemIO.IO_OS.DirectoryExists(folderpath)) - { - Logging.Log.WriteWarningMessage(LOGTAG, "CreateMissingFolder", null, "Creating missing folder {0} for target {1}", folderpath, targetpath); - SystemIO.IO_OS.DirectoryCreate(folderpath); - } - - ApplyMetadata(targetpath, metainfo.Value, options.RestorePermissions, options.RestoreSymlinkMetadata, options.Dryrun); - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "MetadataWriteFailed", ex, "Failed to apply metadata to file: \"{0}\", message: {1}", targetpath, ex.Message); - if (options.UnittestMode) - throw; - } - } - } - } - - /// - /// Perform the restore operation. - /// This is the new implementation, which utilizes a CSP network of processes to perform the restore. - /// - /// The database containing information about the restore. - /// The filter of which files to restore. - private async Task DoRunNew(IBackendManager backendManager, LocalRestoreDatabase database, Library.Utility.IFilter filter, CancellationToken cancellationToken) - { - // Perform initial setup - Utility.UpdateOptionsFromDb(database, m_options); - Utility.VerifyOptionsAndUpdateDatabase(database, m_options); - - // Verify the backend if necessary - if (!m_options.NoBackendverification) - { - m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PreRestoreVerify); - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, database, m_result.BackendWriter, false, null).ConfigureAwait(false); - } - - // Prepare the block and file list and create the directory structure - m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateFileList); - using (new Logging.Timer(LOGTAG, "PrepareBlockList", "PrepareBlockList")) - PrepareBlockAndFileList(database, m_options, filter, m_result); - m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateTargetFolders); - using (new Logging.Timer(LOGTAG, "CreateDirectory", "CreateDirectory")) - await CreateDirectoryStructure(database, m_options, m_result).ConfigureAwait(false); - database.SetResult(m_result); - - using var setup_log_timer = new Logging.Timer(LOGTAG, "RestoreNetworkSetup", "RestoreNetworkSetup"); - // Create the channels between BlockManager and FileProcessor - var fileprocessor_requests = new Channel[m_options.RestoreFileProcessors].Select(_ => ChannelManager.CreateChannel(buffersize: Restore.Channels.BufferSize)).ToArray(); - var fileprocessor_responses = new Channel[m_options.RestoreFileProcessors].Select(_ => ChannelManager.CreateChannel(buffersize: Restore.Channels.BufferSize)).ToArray(); - - // Create the process network - Restore.Channels channels = new(); - var filelister = Restore.FileLister.Run(channels, database, m_options, m_result); - var fileprocessors = Enumerable.Range(0, m_options.RestoreFileProcessors).Select(i => Restore.FileProcessor.Run(channels, database, fileprocessor_requests[i], fileprocessor_responses[i], m_options, m_result)).ToArray(); - var blockmanager = Restore.BlockManager.Run(channels, database, m_options, fileprocessor_requests, fileprocessor_responses); - var volumecache = Restore.VolumeManager.Run(channels, m_options); - var volumedownloaders = Enumerable.Range(0, m_options.RestoreVolumeDownloaders).Select(i => Restore.VolumeDownloader.Run(channels, database, backendManager.GetBackend(), m_options, m_result)).ToArray(); - var volumedecryptors = Enumerable.Range(0, m_options.RestoreVolumeDecryptors).Select(i => Restore.VolumeDecryptor.Run(channels, m_options)).ToArray(); - var volumedecompressors = Enumerable.Range(0, m_options.RestoreVolumeDecompressors).Select(i => Restore.VolumeDecompressor.Run(channels, m_options)).ToArray(); - - setup_log_timer.Dispose(); - - // Wait for the network to complete - Task[] all = - [ - filelister, - ..fileprocessors, - blockmanager, - volumecache, - ..volumedownloaders, - ..volumedecryptors, - ..volumedecompressors - ]; - - // Start the progress updater - using (new Logging.Timer(LOGTAG, "RestoreNetworkWait", "RestoreNetworkWait")) - using (var kill_updater = new CancellationTokenSource()) - { - var updater = Task.Run(async () => - { - while (!kill_updater.Token.IsCancellationRequested) - { - m_result.OperationProgressUpdater.UpdatefilesProcessed(m_result.RestoredFiles, m_result.SizeOfRestoredFiles); - await Task.Delay(1000, kill_updater.Token).ConfigureAwait(false); - } - }, kill_updater.Token); - - await Task.WhenAll(all).ConfigureAwait(false); - kill_updater.Cancel(); - } - - if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - return; - - m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PostRestoreVerify); - - // If any errors occurred, log them - if (m_result.BrokenRemoteFiles.Count > 0 || m_result.BrokenLocalFiles.Count > 0) - { - var nl = Environment.NewLine; - int maxN = 10; - long remoteFirstN = Math.Min(maxN, m_result.BrokenRemoteFiles.Count); - string remoteFirst = remoteFirstN < m_result.BrokenRemoteFiles.Count ? $"first {maxN} " : string.Empty; - long localFirstN = Math.Min(maxN, m_result.BrokenLocalFiles.Count); - string localFirst = localFirstN < m_result.BrokenLocalFiles.Count ? $"first {maxN} " : string.Empty; - - string remoteMessage = m_result.BrokenRemoteFiles.Count > 0 ? $"Failed to download {m_result.BrokenRemoteFiles.Count} remote files." : string.Empty; - string remoteList = m_result.BrokenRemoteFiles.Count > 0 ? $"The following {remoteFirst}remote files failed to download, which may be the cause:{nl}{string.Join(nl, m_result.BrokenRemoteFiles.Take(maxN))}{nl}" : string.Empty; - string localMessage = m_result.BrokenLocalFiles.Count > 0 ? $"Failed to restore {m_result.BrokenLocalFiles.Count} local files." : string.Empty; - string localList = m_result.BrokenLocalFiles.Count > 0 ? $"The following {localFirst}local files failed to restore:{nl}{string.Join(nl, m_result.BrokenLocalFiles.Take(maxN))}{nl}" : string.Empty; - - Logging.Log.WriteErrorMessage(LOGTAG, "RestoreFailures", null, $"{remoteMessage}{nl}{localMessage}{nl}{remoteList}{nl}{localList}"); - } - else if (m_result.RestoredFiles == 0) - Logging.Log.WriteWarningMessage(LOGTAG, "NoFilesRestored", null, "Restore completed without errors but no files were restored"); - - // Drop the temp tables - database.DropRestoreTable(); - await backendManager.WaitForEmptyAsync(database, null, CancellationToken.None).ConfigureAwait(false); - - // Report that the restore is complete - m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_Complete); - m_result.EndTime = DateTime.UtcNow; - } - - /// - /// Perform the restore operation. - /// This is the legacy implementation, which performs the restore in a single thread. Kept as in case the new implementation fails. - /// - /// The database containing information about the restore. - /// The filter of which files to restore. - private async Task DoRun(IBackendManager backendManager, LocalRestoreDatabase database, Library.Utility.IFilter filter, CancellationToken cancellationToken) - { - //In this case, we check that the remote storage fits with the database. - //We can then query the database and find the blocks that we need to do the restore - //using (var database = new LocalRestoreDatabase(dbparent)) - using (var metadatastorage = new RestoreHandlerMetadataStorage()) - { - database.SetResult(m_result); - Utility.UpdateOptionsFromDb(database, m_options); - Utility.VerifyOptionsAndUpdateDatabase(database, m_options); - m_blockbuffer = new byte[m_options.Blocksize]; - - if (!m_options.NoBackendverification) - { - m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PreRestoreVerify); - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, database, m_result.BackendWriter, false, null).ConfigureAwait(false); - } - - //Figure out what files are to be patched, and what blocks are needed - m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateFileList); - using (new Logging.Timer(LOGTAG, "PrepareBlockList", "PrepareBlockList")) - PrepareBlockAndFileList(database, m_options, filter, m_result); - - //Make the entire output setup - m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateTargetFolders); - using (new Logging.Timer(LOGTAG, "CreateDirectory", "CreateDirectory")) - await CreateDirectoryStructure(database, m_options, m_result); - - //If we are patching an existing target folder, do not touch stuff that is already updated - m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_ScanForExistingFiles); - using (var blockhasher = HashFactory.CreateHasher(m_options.BlockHashAlgorithm)) - using (var filehasher = HashFactory.CreateHasher(m_options.FileHashAlgorithm)) - using (new Logging.Timer(LOGTAG, "ScanForExistingTargetBlocks", "ScanForExistingTargetBlocks")) - await ScanForExistingTargetBlocks(database, m_blockbuffer, blockhasher, filehasher, m_options, m_result).ConfigureAwait(false); - - //Look for existing blocks in the original source files only - if (m_options.UseLocalBlocks && !string.IsNullOrEmpty(m_options.Restorepath)) - { - using (var blockhasher = HashFactory.CreateHasher(m_options.BlockHashAlgorithm)) - using (new Logging.Timer(LOGTAG, "ScanForExistingSourceBlocksFast", "ScanForExistingSourceBlocksFast")) - { - m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_ScanForLocalBlocks); - await ScanForExistingSourceBlocksFast(database, m_options, m_blockbuffer, blockhasher, m_result).ConfigureAwait(false); - } - } - - if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - { - await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false); - return; - } - - // If other local files already have the blocks we want, we use them instead of downloading - if (m_options.UseLocalBlocks) - { - m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PatchWithLocalBlocks); - using (var blockhasher = HashFactory.CreateHasher(m_options.BlockHashAlgorithm)) - using (new Logging.Timer(LOGTAG, "PatchWithLocalBlocks", "PatchWithLocalBlocks")) - await ScanForExistingSourceBlocks(database, m_options, m_blockbuffer, blockhasher, m_result, metadatastorage).ConfigureAwait(false); - } - - if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - { - await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false); - return; - } - - // Fill BLOCKS with remote sources - List volumes; - using (new Logging.Timer(LOGTAG, "GetMissingVolumes", "GetMissingVolumes")) - volumes = database.GetMissingVolumes().ToList(); - - if (volumes.Count > 0) - { - Logging.Log.WriteInformationMessage(LOGTAG, "RemoteFileCount", "{0} remote files are required to restore", volumes.Count); - m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_DownloadingRemoteFiles); - } - - var brokenFiles = new List(); - - using (new Logging.Timer(LOGTAG, "PatchWithBlocklist", "PatchWithBlocklist")) - await foreach (var (tmpfile, _, _, name) in backendManager.GetFilesOverlappedAsync(volumes, cancellationToken)) - { - try - { - if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - { - await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false); - return; - } - - using (tmpfile) - using (var blocks = new BlockVolumeReader(GetCompressionModule(name), tmpfile, m_options)) - PatchWithBlocklist(database, blocks, m_options, m_result, m_blockbuffer, metadatastorage); - } - catch (Exception ex) - { - brokenFiles.Add(name); - Logging.Log.WriteErrorMessage(LOGTAG, "PatchingFailed", ex, "Failed to patch with remote file: \"{0}\", message: {1}", name, ex.Message); - if (ex is System.Threading.ThreadAbortException) - throw; - } - } - - var fileErrors = 0L; - - // Restore empty files. They might not have any blocks so don't appear in any volume. - foreach (var file in database.GetFilesToRestore(true).Where(item => item.Length == 0)) - { - Logging.Log.WriteVerboseMessage(LOGTAG, "RestoreEmptyFile", "Restoring empty file \"{0}\"", file.Path); - - try - { - SystemIO.IO_OS.DirectoryCreate(SystemIO.IO_OS.PathGetDirectoryName(file.Path)); - // Just create the file and close it right away, empty statement is intentional. - using (SystemIO.IO_OS.FileCreate(file.Path)) - { - } - } - catch (Exception ex) - { - fileErrors++; - Logging.Log.WriteErrorMessage(LOGTAG, "RestoreFileFailed", ex, "Failed to restore empty file: \"{0}\". Error message was: {1}", file.Path, ex.Message); - if (ex is System.Threading.ThreadAbortException) - throw; - } - } - - // Enforcing the length of files is now already done during ScanForExistingTargetBlocks - // and thus not necessary anymore. - - // Apply metadata - if (!m_options.SkipMetadata) - ApplyStoredMetadata(m_options, metadatastorage); - - if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - return; - - m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PostRestoreVerify); - - - if (m_options.PerformRestoredFileVerification) - { - // After all blocks in the files are restored, verify the file hash - using (var filehasher = HashFactory.CreateHasher(m_options.FileHashAlgorithm)) - using (new Logging.Timer(LOGTAG, "RestoreVerification", "RestoreVerification")) - foreach (var file in database.GetFilesToRestore(true)) - { - try - { - if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - { - await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false); - return; - } - - Logging.Log.WriteVerboseMessage(LOGTAG, "TestFileIntegrity", "Testing restored file integrity: {0}", file.Path); - - string key; - long size; - using (var fs = SystemIO.IO_OS.FileOpenRead(file.Path)) - { - size = fs.Length; - key = Convert.ToBase64String(filehasher.ComputeHash(fs)); - } - - if (key != file.Hash) - throw new Exception(string.Format("Failed to restore file: \"{0}\". File hash is {1}, expected hash is {2}", file.Path, key, file.Hash)); - m_result.RestoredFiles++; - m_result.SizeOfRestoredFiles += size; - } - catch (Exception ex) - { - fileErrors++; - Logging.Log.WriteErrorMessage(LOGTAG, "RestoreFileFailed", ex, "Failed to restore file: \"{0}\". Error message was: {1}", file.Path, ex.Message); - if (ex is System.Threading.ThreadAbortException) - throw; - } - } - } - - if (fileErrors > 0 && brokenFiles.Count > 0) - Logging.Log.WriteInformationMessage(LOGTAG, "RestoreFailures", "Failed to restore {0} files, additionally the following files failed to download, which may be the cause:{1}{2}", fileErrors, Environment.NewLine, string.Join(Environment.NewLine, brokenFiles)); - else if (fileErrors > 0) - Logging.Log.WriteInformationMessage(LOGTAG, "RestoreFailures", "Failed to restore {0} files", fileErrors); - else if (m_result.RestoredFiles == 0) - Logging.Log.WriteWarningMessage(LOGTAG, "NoFilesRestored", null, "Restore completed without errors but no files were restored"); - - // Drop the temp tables - database.DropRestoreTable(); - await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false); - } - - m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_Complete); - m_result.EndTime = DateTime.UtcNow; - } - - public static bool ApplyMetadata(string path, System.IO.Stream stream, bool restorePermissions, bool restoreSymlinkMetadata, bool dryrun) - { - // TODO This has been modified to return a bool indicating if anything was written to properly report the number of files restored. The legacy restore doesn't check this, which produces an error in the CI where it reports that no files have been restored, even though one have. It's in Duplicati/UnitTests/SymLinkTests.cs the test SymLinkTests.SymLinkExists() that fails on the very last assert that there are 0 warnings. - using (var tr = new System.IO.StreamReader(stream)) - using (var jr = new Newtonsoft.Json.JsonTextReader(tr)) - { - var metadata = new Newtonsoft.Json.JsonSerializer().Deserialize>(jr); - string k; - long t; - System.IO.FileAttributes fa; - var wrote_something = false; - - // If this is dry-run, we stop after having deserialized the metadata - if (dryrun) - return wrote_something; - - var isDirTarget = path.EndsWith(DIRSEP, StringComparison.Ordinal); - var targetpath = isDirTarget ? path.Substring(0, path.Length - 1) : path; - - // Make the symlink first, otherwise we cannot apply metadata to it - if (metadata.TryGetValue("CoreSymlinkTarget", out k)) - { - // Check if the target exists, and overwrite it if it does. - if (SystemIO.IO_OS.FileExists(targetpath)) - { - SystemIO.IO_OS.FileDelete(targetpath); - } - else if (SystemIO.IO_OS.DirectoryExists(targetpath)) - { - SystemIO.IO_OS.DirectoryDelete(targetpath, false); - } - SystemIO.IO_OS.CreateSymlink(targetpath, k, isDirTarget); - wrote_something = true; - } - // If the target is a folder, make sure we create it first - else if (isDirTarget && !SystemIO.IO_OS.DirectoryExists(targetpath)) - SystemIO.IO_OS.DirectoryCreate(targetpath); - - // Avoid setting restoring symlink metadata, as that writes the symlink target, not the symlink itself - if (!restoreSymlinkMetadata && Snapshots.SnapshotUtility.IsSymlink(SystemIO.IO_OS, targetpath)) - { - Logging.Log.WriteVerboseMessage(LOGTAG, "no-symlink-metadata-restored", "Not applying metadata to symlink: {0}", targetpath); - return wrote_something; - } - - if (metadata.TryGetValue("CoreLastWritetime", out k) && long.TryParse(k, out t)) - { - if (isDirTarget) - SystemIO.IO_OS.DirectorySetLastWriteTimeUtc(targetpath, new DateTime(t, DateTimeKind.Utc)); - else - SystemIO.IO_OS.FileSetLastWriteTimeUtc(targetpath, new DateTime(t, DateTimeKind.Utc)); - } - - if (metadata.TryGetValue("CoreCreatetime", out k) && long.TryParse(k, out t)) - { - if (isDirTarget) - SystemIO.IO_OS.DirectorySetCreationTimeUtc(targetpath, new DateTime(t, DateTimeKind.Utc)); - else - SystemIO.IO_OS.FileSetCreationTimeUtc(targetpath, new DateTime(t, DateTimeKind.Utc)); - } - - if (metadata.TryGetValue("CoreAttributes", out k) && Enum.TryParse(k, true, out fa)) - SystemIO.IO_OS.SetFileAttributes(targetpath, fa); - - SystemIO.IO_OS.SetMetadata(path, metadata, restorePermissions); - return wrote_something; - } - } - - private static async Task ScanForExistingSourceBlocksFast(LocalRestoreDatabase database, Options options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, RestoreResults result) - { - // Fill BLOCKS with data from known local source files - using (var blockmarker = database.CreateBlockMarker()) - { - var updateCount = 0L; - foreach (var entry in database.GetFilesAndSourceBlocksFast(options.Blocksize)) - { - var targetpath = entry.TargetPath; - var targetfileid = entry.TargetFileID; - var sourcepath = entry.SourcePath; - var patched = false; - - try - { - if (SystemIO.IO_OS.FileExists(sourcepath)) - { - var folderpath = SystemIO.IO_OS.PathGetDirectoryName(targetpath); - if (!options.Dryrun && !SystemIO.IO_OS.DirectoryExists(folderpath)) - { - Logging.Log.WriteWarningMessage(LOGTAG, "CreateMissingFolder", null, "Creating missing folder {0} for file {1}", folderpath, targetpath); - SystemIO.IO_OS.DirectoryCreate(folderpath); - } - - using (var targetstream = options.Dryrun ? null : SystemIO.IO_OS.FileOpenWrite(targetpath)) - { - try - { - using (var sourcestream = SystemIO.IO_OS.FileOpenRead(sourcepath)) - { - foreach (var block in entry.Blocks) - { - if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - return; - - //TODO: Handle metadata - - if (sourcestream.Length > block.Offset) - { - sourcestream.Position = block.Offset; - - int size = Library.Utility.Utility.ForceStreamRead(sourcestream, blockbuffer, blockbuffer.Length); - if (size == block.Size) - { - var key = Convert.ToBase64String(hasher.ComputeHash(blockbuffer, 0, size)); - if (key == block.Hash) - { - patched = true; - if (!options.Dryrun) - { - targetstream.Position = block.Offset; - targetstream.Write(blockbuffer, 0, size); - } - - blockmarker.SetBlockRestored(targetfileid, block.Index, key, block.Size, false); - } - } - } - } - } - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "PatchingFileLocalFailed", ex, "Failed to patch file: \"{0}\" with data from local file \"{1}\", message: {2}", targetpath, sourcepath, ex.Message); - if (ex is System.Threading.ThreadAbortException) - throw; - } - } - - if ((++updateCount) % 20 == 0) - { - blockmarker.UpdateProcessed(result.OperationProgressUpdater); - if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - return; - } - - } - else - { - Logging.Log.WriteVerboseMessage(LOGTAG, "LocalSourceMissing", "Local source file not found: {0}", sourcepath); - } - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "PatchingFileLocalFailed", ex, "Failed to patch file: \"{0}\" with local data, message: {1}", targetpath, ex.Message); - if (ex is System.Threading.ThreadAbortException) - throw; - if (options.UnittestMode) - throw; - } - - if (patched) - Logging.Log.WriteVerboseMessage(LOGTAG, "FilePatchedWithLocal", "Target file is patched with some local data: {0}", targetpath); - else - Logging.Log.WriteVerboseMessage(LOGTAG, "FilePatchedWithLocal", "Target file is not patched any local data: {0}", targetpath); - - if (patched && options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPatchWithLocal", "Would patch file with local data: {0}", targetpath); - } - - blockmarker.UpdateProcessed(result.OperationProgressUpdater); - blockmarker.Commit(); - } - } - - private static async Task ScanForExistingSourceBlocks(LocalRestoreDatabase database, Options options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, RestoreResults result, RestoreHandlerMetadataStorage metadatastorage) - { - // Fill BLOCKS with data from known local source files - using (var blockmarker = database.CreateBlockMarker()) - { - var updateCount = 0L; - foreach (var restorelist in database.GetFilesAndSourceBlocks(options.SkipMetadata, options.Blocksize)) - { - var targetpath = restorelist.TargetPath; - var targetfileid = restorelist.TargetFileID; - var patched = false; - try - { - if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - return; - - var folderpath = SystemIO.IO_OS.PathGetDirectoryName(targetpath); - if (!options.Dryrun && !SystemIO.IO_OS.DirectoryExists(folderpath)) - { - Logging.Log.WriteWarningMessage(LOGTAG, "CreateMissingFolder", null, "Creating missing folder {0} for file {1}", folderpath, targetpath); - SystemIO.IO_OS.DirectoryCreate(folderpath); - } - - using (var file = options.Dryrun ? null : SystemIO.IO_OS.FileOpenWrite(targetpath)) - foreach (var targetblock in restorelist.Blocks) - { - foreach (var source in targetblock.Blocksources) - { - try - { - if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - return; - - if (SystemIO.IO_OS.FileExists(source.Path)) - { - if (source.IsMetadata) - { - // TODO: Handle this by reconstructing - // metadata from file and checking the hash - - continue; - } - else - { - using (var sourcefile = SystemIO.IO_OS.FileOpenRead(source.Path)) - { - sourcefile.Position = source.Offset; - int size = Library.Utility.Utility.ForceStreamRead(sourcefile, blockbuffer, blockbuffer.Length); - if (size == targetblock.Size) - { - var key = Convert.ToBase64String(hasher.ComputeHash(blockbuffer, 0, size)); - if (key == targetblock.Hash) - { - if (!options.Dryrun) - { - if (targetblock.IsMetadata) - metadatastorage.Add(targetpath, new System.IO.MemoryStream(blockbuffer, 0, size)); - else - { - file.Position = targetblock.Offset; - file.Write(blockbuffer, 0, size); - } - } - - blockmarker.SetBlockRestored(targetfileid, targetblock.Index, key, targetblock.Size, false); - patched = true; - break; - } - } - } - } - } - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "PatchingFileLocalFailed", ex, "Failed to patch file: \"{0}\" with data from local file \"{1}\", message: {2}", targetpath, source.Path, ex.Message); - if (ex is System.Threading.ThreadAbortException) - throw; - } - } - } - - if ((++updateCount) % 20 == 0) - blockmarker.UpdateProcessed(result.OperationProgressUpdater); - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "PatchingFileLocalFailed", ex, "Failed to patch file: \"{0}\" with local data, message: {1}", targetpath, ex.Message); - if (options.UnittestMode) - throw; - } - - if (patched) - Logging.Log.WriteVerboseMessage(LOGTAG, "FilePatchedWithLocal", "Target file is patched with some local data: {0}", targetpath); - else - Logging.Log.WriteVerboseMessage(LOGTAG, "FilePatchedWithLocal", "Target file is not patched any local data: {0}", targetpath); - - if (patched && options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPatchWithLocal", string.Format("Would patch file with local data: {0}", targetpath)); - } - - blockmarker.UpdateProcessed(result.OperationProgressUpdater); - blockmarker.Commit(); - } - } - - private static void PrepareBlockAndFileList(LocalRestoreDatabase database, Options options, Library.Utility.IFilter filter, RestoreResults result) - { - // Create a temporary table FILES by selecting the files from fileset that matches a specific operation id - // Delete all entries from the temp table that are excluded by the filter(s) - using (new Logging.Timer(LOGTAG, "PrepareRestoreFileList", "PrepareRestoreFileList")) - { - var c = database.PrepareRestoreFilelist(options.Time, options.Version, filter); - result.OperationProgressUpdater.UpdatefileCount(c.Item1, c.Item2, true); - } - - using (new Logging.Timer(LOGTAG, "SetTargetPaths", "SetTargetPaths")) - if (!string.IsNullOrEmpty(options.Restorepath)) - { - // Find the largest common prefix - var largest_prefix = options.DontCompressRestorePaths ? "" : database.GetLargestPrefix(); - - Logging.Log.WriteVerboseMessage(LOGTAG, "MappingRestorePath", "Mapping restore path prefix to \"{0}\" to \"{1}\"", largest_prefix, Util.AppendDirSeparator(options.Restorepath)); - - // Set the target paths, special care with C:\ and / - database.SetTargetPaths(largest_prefix, Util.AppendDirSeparator(options.Restorepath)); - } - else - { - database.SetTargetPaths("", ""); - } - - // Create a temporary table BLOCKS that lists all blocks that needs to be recovered - using (new Logging.Timer(LOGTAG, "FindMissingBlocks", "FindMissingBlocks")) - database.FindMissingBlocks(options.SkipMetadata); - - // Create temporary tables and triggers that automatically track progress - using (new Logging.Timer(LOGTAG, "CreateProgressTracker", "CreateProgressTracker")) - database.CreateProgressTracker(false); - - } - - private static async Task CreateDirectoryStructure(LocalRestoreDatabase database, Options options, RestoreResults result) - { - // This part is not protected by try/catch as we need the target folder to exist - if (!string.IsNullOrEmpty(options.Restorepath)) - if (!SystemIO.IO_OS.DirectoryExists(options.Restorepath)) - { - Logging.Log.WriteVerboseMessage(LOGTAG, "CreateFolder", "Creating folder: {0}", options.Restorepath); - - if (options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldCreateFolder", "Would create folder: {0}", options.Restorepath); - else - SystemIO.IO_OS.DirectoryCreate(options.Restorepath); - } - - foreach (var folder in database.GetTargetFolders()) - { - try - { - if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - return; - - if (!SystemIO.IO_OS.DirectoryExists(folder)) - { - result.RestoredFolders++; - - Logging.Log.WriteVerboseMessage(LOGTAG, "CreateFolder", "Creating folder: {0}", folder); - - if (options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldCreateFolder", "Would create folder: {0}", folder); - else - SystemIO.IO_OS.DirectoryCreate(folder); - } - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "FolderCreateFailed", ex, "Failed to create folder: \"{0}\", message: {1}", folder, ex.Message); - if (options.UnittestMode) - throw; - } - } - } - - private static async Task ScanForExistingTargetBlocks(LocalRestoreDatabase database, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm blockhasher, System.Security.Cryptography.HashAlgorithm filehasher, Options options, RestoreResults result) - { - // Scan existing files for existing BLOCKS - using (var blockmarker = database.CreateBlockMarker()) - { - var updateCount = 0L; - foreach (var restorelist in database.GetExistingFilesWithBlocks()) - { - var rename = !options.Overwrite; - var targetpath = restorelist.TargetPath; - var targetfileid = restorelist.TargetFileID; - var targetfilehash = restorelist.TargetHash; - var targetfilelength = restorelist.Length; - if (SystemIO.IO_OS.FileExists(targetpath)) - { - try - { - if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - return; - - var currentfilelength = SystemIO.IO_OS.FileLength(targetpath); - var wasTruncated = false; - - // Adjust file length in overwrite mode if necessary (smaller is ok, will be extended during restore) - // We do it before scanning for blocks. This allows full verification on files that only needs to - // be truncated (i.e. forthwritten log files). - if (!rename && currentfilelength > targetfilelength) - { - var currentAttr = SystemIO.IO_OS.GetFileAttributes(targetpath); - if ((currentAttr & System.IO.FileAttributes.ReadOnly) != 0) // clear readonly attribute - { - if (options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldResetReadOnlyAttribute", "Would reset read-only attribute on file: {0}", targetpath); - else SystemIO.IO_OS.SetFileAttributes(targetpath, currentAttr & ~System.IO.FileAttributes.ReadOnly); - } - if (options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldTruncateFile", "Would truncate file '{0}' to length of {1:N0} bytes", targetpath, targetfilelength); - else - { - using (var file = SystemIO.IO_OS.FileOpenWrite(targetpath)) - file.SetLength(targetfilelength); - currentfilelength = targetfilelength; - } - wasTruncated = true; - } - - // If file size does not match and we have to rename on conflict, - // the whole scan can be skipped here because all blocks have to be restored anyway. - // For the other cases, we will check block and and file hashes and look for blocks - // to be restored and files that can already be verified. - if (!rename || currentfilelength == targetfilelength) - { - // a file hash for verification will only be necessary if the file has exactly - // the wanted size so we have a chance to already mark the file as data-verified. - bool calcFileHash = (currentfilelength == targetfilelength); - if (calcFileHash) filehasher.Initialize(); - - using (var file = SystemIO.IO_OS.FileOpenRead(targetpath)) - using (var block = new Blockprocessor(file, blockbuffer)) - foreach (var targetblock in restorelist.Blocks) - { - var size = block.Readblock(); - if (size <= 0) - break; - - //TODO: Handle Metadata - - bool blockhashmatch = false; - if (size == targetblock.Size) - { - // Parallelize file hash calculation on rename. Running read-only on same array should not cause conflicts or races. - // Actually, in future always calculate the file hash and mark the file data as already verified. - - System.Threading.Tasks.Task calcFileHashTask = null; - if (calcFileHash) - calcFileHashTask = System.Threading.Tasks.Task.Run( - () => filehasher.TransformBlock(blockbuffer, 0, size, blockbuffer, 0)); - - var key = Convert.ToBase64String(blockhasher.ComputeHash(blockbuffer, 0, size)); - - if (calcFileHashTask != null) await calcFileHashTask.ConfigureAwait(false); // wait because blockbuffer will be overwritten. - - if (key == targetblock.Hash) - { - blockmarker.SetBlockRestored(targetfileid, targetblock.Index, key, size, false); - blockhashmatch = true; - } - } - if (calcFileHash && !blockhashmatch) // will not be necessary anymore - { - filehasher.TransformFinalBlock(blockbuffer, 0, 0); // So a new initialize will not throw - calcFileHash = false; - if (rename) // file does not match. So break. - break; - } - } - - bool fullfilehashmatch = false; - if (calcFileHash) // now check if files are identical - { - filehasher.TransformFinalBlock(blockbuffer, 0, 0); - var filekey = Convert.ToBase64String(filehasher.Hash); - fullfilehashmatch = (filekey == targetfilehash); - } - - if (!rename && !fullfilehashmatch && !wasTruncated) // Reset read-only attribute (if set) to overwrite - { - var currentAttr = SystemIO.IO_OS.GetFileAttributes(targetpath); - if ((currentAttr & System.IO.FileAttributes.ReadOnly) != 0) - { - if (options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldResetReadOnlyAttribyte", "Would reset read-only attribute on file: {0}", targetpath); - else SystemIO.IO_OS.SetFileAttributes(targetpath, currentAttr & ~System.IO.FileAttributes.ReadOnly); - } - } - - if (fullfilehashmatch) - { - //TODO: Check metadata to trigger rename? If metadata changed, it will still be restored for the file in-place. - blockmarker.SetFileDataVerified(targetfileid); - Logging.Log.WriteVerboseMessage(LOGTAG, "TargetExistsInCorrectVersion", "Target file exists{1} and is correct version: {0}", targetpath, wasTruncated ? " (but was truncated)" : ""); - rename = false; - } - else if (rename) - { - // The new file will have none of the correct blocks, - // even if the scanned file had some - blockmarker.SetAllBlocksMissing(targetfileid); - } - } - - if ((++updateCount) % 20 == 0) - { - blockmarker.UpdateProcessed(result.OperationProgressUpdater); - if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - return; - } - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "TargetFileReadError", ex, "Failed to read target file: \"{0}\", message: {1}", targetpath, ex.Message); - if (ex is System.Threading.ThreadAbortException) - throw; - if (options.UnittestMode) - throw; - } - } - else - { - Logging.Log.WriteVerboseMessage(LOGTAG, "MissingTargetFile", "Target file does not exist: {0}", targetpath); - rename = false; - } - - if (rename) - { - //Select a new filename - var ext = SystemIO.IO_OS.PathGetExtension(targetpath) ?? ""; - if (!string.IsNullOrEmpty(ext) && !ext.StartsWith(".", StringComparison.Ordinal)) - ext = "." + ext; - - // First we try with a simple date append, assuming that there are not many conflicts there - var newname = SystemIO.IO_OS.PathChangeExtension(targetpath, null) + "." + database.RestoreTime.ToLocalTime().ToString("yyyy-MM-dd", System.Globalization.CultureInfo.InvariantCulture); - var tr = newname + ext; - var c = 0; - while (SystemIO.IO_OS.FileExists(tr) && c < 1000) - { - try - { - // If we have a file with the correct name, - // it is most likely the file we want - filehasher.Initialize(); - - string key; - using (var file = SystemIO.IO_OS.FileOpenRead(tr)) - key = Convert.ToBase64String(filehasher.ComputeHash(file)); - - if (key == targetfilehash) - { - //TODO: Also needs metadata check to make correct decision. - // We stick to the policy to restore metadata in place, if data ok. So, metadata block may be restored. - blockmarker.SetAllBlocksRestored(targetfileid, false); - blockmarker.SetFileDataVerified(targetfileid); - break; - } - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "FailedToReadRestoreTarget", ex, "Failed to read candidate restore target {0}", tr); - if (options.UnittestMode) - throw; - } - tr = newname + " (" + (c++).ToString() + ")" + ext; - } - - newname = tr; - - Logging.Log.WriteVerboseMessage(LOGTAG, "TargetFileRetargeted", "Target file exists and will be restored to: {0}", newname); - database.UpdateTargetPath(targetfileid, newname); - } - - } - - blockmarker.UpdateProcessed(result.OperationProgressUpdater); - blockmarker.Commit(); - } - } - } -} +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using System; +using System.Collections.Generic; +using System.Linq; +using Duplicati.Library.Interface; +using Duplicati.Library.Common.IO; +using Duplicati.Library.Main.Database; +using Duplicati.Library.Main.Volumes; +using Duplicati.Library.Utility; +using CoCoL; +using System.Threading.Tasks; +using System.Threading; + +namespace Duplicati.Library.Main.Operation +{ + internal class RestoreHandler + { + /// + /// The tag used for logging + /// + private static readonly string LOGTAG = Logging.Log.LogTagFromType(); + + private readonly Options m_options; + private byte[] m_blockbuffer; + private readonly RestoreResults m_result; + private static readonly string DIRSEP = Util.DirectorySeparatorString; + + public RestoreHandler(Options options, RestoreResults result) + { + m_options = options; + m_result = result; + } + + /// + /// Gets the compression module by parsing the filename + /// + /// The filename to parse + /// The compression module + public static string GetCompressionModule(string filename) + { + var tmp = VolumeBase.ParseFilename(filename); + if (tmp == null) + throw new UserInformationException(string.Format("Unable to parse filename to valid entry: {0}", filename), "FailedToParseRemoteName"); + + return tmp.CompressionModule; + } + + public static RecreateDatabaseHandler.NumberedFilterFilelistDelegate FilterNumberedFilelist(DateTime time, long[] versions, bool singleTimeMatch = false) + { + if (time.Kind == DateTimeKind.Unspecified) + throw new Exception("Unspecified datetime instance, must be either local or UTC"); + + // Make sure the resolution is the same (i.e. no milliseconds) + if (time.Ticks > 0) + time = Library.Utility.Utility.DeserializeDateTime(Library.Utility.Utility.SerializeDateTime(time)).ToUniversalTime(); + + return + _lst => + { + // Unwrap, so we do not query the remote storage twice + var lst = (from n in _lst + where n.FileType == RemoteVolumeType.Files + orderby n.Time descending + select n).ToArray(); + + var numbers = lst.Zip(Enumerable.Range(0, lst.Length), (a, b) => new KeyValuePair(b, a)).ToList(); + + if (time.Ticks > 0 && versions != null && versions.Length > 0) + return from n in numbers + where (singleTimeMatch ? n.Value.Time == time : n.Value.Time <= time) && versions.Contains(n.Key) + select n; + else if (time.Ticks > 0) + return from n in numbers + where (singleTimeMatch ? n.Value.Time == time : n.Value.Time <= time) + select n; + else if (versions != null && versions.Length > 0) + return from n in numbers + where versions.Contains(n.Key) + select n; + else + return numbers; + }; + } + + public void Run(string[] paths, IBackendManager backendManager, Library.Utility.IFilter filter) + { + m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_Begin); + + // If we have both target paths and a filter, combine them into a single filter + filter = JoinedFilterExpression.Join(new FilterExpression(paths), filter); + + LocalRestoreDatabase db = null; + TempFile tmpdb = null; + try + { + if (!m_options.NoLocalDb && SystemIO.IO_OS.FileExists(m_options.Dbpath)) + { + db = new LocalRestoreDatabase(m_options.Dbpath); + db.SetResult(m_result); + } + else + { + Logging.Log.WriteInformationMessage(LOGTAG, "NoLocalDatabase", "No local database, building a temporary database"); + tmpdb = new TempFile(); + RecreateDatabaseHandler.NumberedFilterFilelistDelegate filelistfilter = FilterNumberedFilelist(m_options.Time, m_options.Version); + db = new LocalRestoreDatabase(tmpdb); + m_result.RecreateDatabaseResults = new RecreateDatabaseResults(m_result); + using (new Logging.Timer(LOGTAG, "RecreateTempDbForRestore", "Recreate temporary database for restore")) + new RecreateDatabaseHandler(m_options, (RecreateDatabaseResults)m_result.RecreateDatabaseResults) + .DoRun(backendManager, db, false, filter, filelistfilter, null).Await(); + + if (!m_options.SkipMetadata) + ApplyStoredMetadata(m_options, new RestoreHandlerMetadataStorage()); + + //If we have --version set, we need to adjust, as the db has only the required versions + //TODO: Bit of a hack to set options that way + if (m_options.Version != null && m_options.Version.Length > 0) + m_options.RawOptions["version"] = string.Join(",", Enumerable.Range(0, m_options.Version.Length).Select(x => x.ToString())); + } + + if (m_options.RestoreLegacy) + DoRun(backendManager, db, filter, m_result.TaskControl.ProgressToken).Await(); + else + DoRunNew(backendManager, db, filter, m_result.TaskControl.ProgressToken).Await(); + + db.WriteResults(); + } + finally + { + db?.Dispose(); + tmpdb?.Dispose(); + } + } + + private static void PatchWithBlocklist(LocalRestoreDatabase database, BlockVolumeReader blocks, Options options, RestoreResults result, byte[] blockbuffer, RestoreHandlerMetadataStorage metadatastorage) + { + var blocksize = options.Blocksize; + var updateCounter = 0L; + var fullblockverification = options.FullBlockVerification; + + using (var blockhasher = HashFactory.CreateHasher(options.BlockHashAlgorithm)) + using (var blockmarker = database.CreateBlockMarker()) + using (var volumekeeper = database.GetMissingBlockData(blocks, options.Blocksize)) + { + foreach (var restorelist in volumekeeper.FilesWithMissingBlocks) + { + var targetpath = restorelist.Path; + + if (options.Dryrun) + { + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPatchFile", "Would patch file with remote data: {0}", targetpath); + } + else + { + Logging.Log.WriteVerboseMessage(LOGTAG, "PatchingFile", "Patching file with remote data: {0}", targetpath); + + try + { + var folderpath = SystemIO.IO_OS.PathGetDirectoryName(targetpath); + if (!options.Dryrun && !SystemIO.IO_OS.DirectoryExists(folderpath)) + { + Logging.Log.WriteWarningMessage(LOGTAG, "CreateMissingFolder", null, "Creating missing folder {0} for file {1}", folderpath, targetpath); + SystemIO.IO_OS.DirectoryCreate(folderpath); + } + + // TODO: Much faster if we iterate the volume and checks what blocks are used, + // because the compressors usually like sequential reading + using (var file = SystemIO.IO_OS.FileOpenWrite(targetpath)) + foreach (var targetblock in restorelist.Blocks) + { + file.Position = targetblock.Offset; + var size = blocks.ReadBlock(targetblock.Key, blockbuffer); + if (targetblock.Size == size) + { + var valid = !fullblockverification; + if (!valid) + { + var key = Convert.ToBase64String(blockhasher.ComputeHash(blockbuffer, 0, size)); + if (targetblock.Key == key) + valid = true; + else + Logging.Log.WriteWarningMessage(LOGTAG, "InvalidBlock", null, "Invalid block detected for {0}, expected hash: {1}, actual hash: {2}", targetpath, targetblock.Key, key); + } + + if (valid) + { + file.Write(blockbuffer, 0, size); + blockmarker.SetBlockRestored(restorelist.FileID, targetblock.Offset / blocksize, targetblock.Key, size, false); + } + } + else + { + Logging.Log.WriteWarningMessage(LOGTAG, "WrongBlockSize", null, "Block with hash {0} should have size {1} but has size {2}", targetblock.Key, targetblock.Size, size); + } + } + + if ((++updateCounter) % 20 == 0) + blockmarker.UpdateProcessed(result.OperationProgressUpdater); + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "PatchFailed", ex, "Failed to patch file: \"{0}\", message: {1}, message: {1}", targetpath, ex.Message); + if (options.UnittestMode) + throw; + } + } + } + + if (!options.SkipMetadata) + { + foreach (var restoremetadata in volumekeeper.MetadataWithMissingBlocks) + { + var targetpath = restoremetadata.Path; + Logging.Log.WriteVerboseMessage(LOGTAG, "RecordingMetadata", "Recording metadata from remote data: {0}", targetpath); + + try + { + // TODO: When we support multi-block metadata this needs to deal with it + using (var ms = new System.IO.MemoryStream()) + { + foreach (var targetblock in restoremetadata.Blocks) + { + ms.Position = targetblock.Offset; + var size = blocks.ReadBlock(targetblock.Key, blockbuffer); + if (targetblock.Size == size) + { + ms.Write(blockbuffer, 0, size); + blockmarker.SetBlockRestored(restoremetadata.FileID, targetblock.Offset / blocksize, targetblock.Key, size, true); + } + } + + ms.Position = 0; + metadatastorage.Add(targetpath, ms); + //blockmarker.RecordMetadata(restoremetadata.FileID, ms); + } + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "MetatdataRecordFailed", ex, "Failed to record metadata for file: \"{0}\", message: {1}", targetpath, ex.Message); + if (options.UnittestMode) + throw; + } + } + } + blockmarker.UpdateProcessed(result.OperationProgressUpdater); + blockmarker.Commit(); + } + } + + private static void ApplyStoredMetadata(Options options, RestoreHandlerMetadataStorage metadatastorage) + { + foreach (var metainfo in metadatastorage.Records) + { + var targetpath = metainfo.Key; + + if (options.Dryrun) + { + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPatchMetadata", "Would patch metadata with remote data: {0}", targetpath); + } + else + { + Logging.Log.WriteVerboseMessage(LOGTAG, "PatchingMetadata", "Patching metadata with remote data: {0}", targetpath); + try + { + var folderpath = Duplicati.Library.Utility.Utility.GetParent(targetpath, false); + if (!options.Dryrun && !SystemIO.IO_OS.DirectoryExists(folderpath)) + { + Logging.Log.WriteWarningMessage(LOGTAG, "CreateMissingFolder", null, "Creating missing folder {0} for target {1}", folderpath, targetpath); + SystemIO.IO_OS.DirectoryCreate(folderpath); + } + + ApplyMetadata(targetpath, metainfo.Value, options.RestorePermissions, options.RestoreSymlinkMetadata, options.Dryrun); + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "MetadataWriteFailed", ex, "Failed to apply metadata to file: \"{0}\", message: {1}", targetpath, ex.Message); + if (options.UnittestMode) + throw; + } + } + } + } + + /// + /// Perform the restore operation. + /// This is the new implementation, which utilizes a CSP network of processes to perform the restore. + /// + /// The database containing information about the restore. + /// The filter of which files to restore. + private async Task DoRunNew(IBackendManager backendManager, LocalRestoreDatabase database, Library.Utility.IFilter filter, CancellationToken cancellationToken) + { + // Perform initial setup + Utility.UpdateOptionsFromDb(database, m_options); + Utility.VerifyOptionsAndUpdateDatabase(database, m_options); + + // Verify the backend if necessary + if (!m_options.NoBackendverification) + { + m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PreRestoreVerify); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, database, m_result.BackendWriter, false, null).ConfigureAwait(false); + } + + // Prepare the block and file list and create the directory structure + m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateFileList); + using (new Logging.Timer(LOGTAG, "PrepareBlockList", "PrepareBlockList")) + PrepareBlockAndFileList(database, m_options, filter, m_result); + m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateTargetFolders); + using (new Logging.Timer(LOGTAG, "CreateDirectory", "CreateDirectory")) + await CreateDirectoryStructure(database, m_options, m_result).ConfigureAwait(false); + database.SetResult(m_result); + + using var setup_log_timer = new Logging.Timer(LOGTAG, "RestoreNetworkSetup", "RestoreNetworkSetup"); + // Create the channels between BlockManager and FileProcessor + var fileprocessor_requests = new Channel[m_options.RestoreFileProcessors].Select(_ => ChannelManager.CreateChannel(buffersize: Restore.Channels.BufferSize)).ToArray(); + var fileprocessor_responses = new Channel[m_options.RestoreFileProcessors].Select(_ => ChannelManager.CreateChannel(buffersize: Restore.Channels.BufferSize)).ToArray(); + + // Create the process network + Restore.Channels channels = new(); + var filelister = Restore.FileLister.Run(channels, database, m_options, m_result); + var fileprocessors = Enumerable.Range(0, m_options.RestoreFileProcessors).Select(i => Restore.FileProcessor.Run(channels, database, fileprocessor_requests[i], fileprocessor_responses[i], m_options, m_result)).ToArray(); + var blockmanager = Restore.BlockManager.Run(channels, database, m_options, fileprocessor_requests, fileprocessor_responses); + var volumecache = Restore.VolumeManager.Run(channels, m_options); + var volumedownloaders = Enumerable.Range(0, m_options.RestoreVolumeDownloaders).Select(i => Restore.VolumeDownloader.Run(channels, database, backendManager.GetBackend(), m_options, m_result)).ToArray(); + var volumedecryptors = Enumerable.Range(0, m_options.RestoreVolumeDecryptors).Select(i => Restore.VolumeDecryptor.Run(channels, backendManager, m_options)).ToArray(); + var volumedecompressors = Enumerable.Range(0, m_options.RestoreVolumeDecompressors).Select(i => Restore.VolumeDecompressor.Run(channels, m_options)).ToArray(); + + setup_log_timer.Dispose(); + + // Wait for the network to complete + Task[] all = + [ + filelister, + ..fileprocessors, + blockmanager, + volumecache, + ..volumedownloaders, + ..volumedecryptors, + ..volumedecompressors + ]; + + // Start the progress updater + using (new Logging.Timer(LOGTAG, "RestoreNetworkWait", "RestoreNetworkWait")) + using (var kill_updater = new CancellationTokenSource()) + { + var updater = Task.Run(async () => + { + while (!kill_updater.Token.IsCancellationRequested) + { + m_result.OperationProgressUpdater.UpdatefilesProcessed(m_result.RestoredFiles, m_result.SizeOfRestoredFiles); + await Task.Delay(1000, kill_updater.Token).ConfigureAwait(false); + } + }, kill_updater.Token); + + await Task.WhenAll(all).ConfigureAwait(false); + kill_updater.Cancel(); + } + + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + return; + + m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PostRestoreVerify); + + // If any errors occurred, log them + if (m_result.BrokenRemoteFiles.Count > 0 || m_result.BrokenLocalFiles.Count > 0) + { + var nl = Environment.NewLine; + int maxN = 10; + long remoteFirstN = Math.Min(maxN, m_result.BrokenRemoteFiles.Count); + string remoteFirst = remoteFirstN < m_result.BrokenRemoteFiles.Count ? $"first {maxN} " : string.Empty; + long localFirstN = Math.Min(maxN, m_result.BrokenLocalFiles.Count); + string localFirst = localFirstN < m_result.BrokenLocalFiles.Count ? $"first {maxN} " : string.Empty; + + string remoteMessage = m_result.BrokenRemoteFiles.Count > 0 ? $"Failed to download {m_result.BrokenRemoteFiles.Count} remote files." : string.Empty; + string remoteList = m_result.BrokenRemoteFiles.Count > 0 ? $"The following {remoteFirst}remote files failed to download, which may be the cause:{nl}{string.Join(nl, m_result.BrokenRemoteFiles.Take(maxN))}{nl}" : string.Empty; + string localMessage = m_result.BrokenLocalFiles.Count > 0 ? $"Failed to restore {m_result.BrokenLocalFiles.Count} local files." : string.Empty; + string localList = m_result.BrokenLocalFiles.Count > 0 ? $"The following {localFirst}local files failed to restore:{nl}{string.Join(nl, m_result.BrokenLocalFiles.Take(maxN))}{nl}" : string.Empty; + + Logging.Log.WriteErrorMessage(LOGTAG, "RestoreFailures", null, $"{remoteMessage}{nl}{localMessage}{nl}{remoteList}{nl}{localList}"); + } + else if (m_result.RestoredFiles == 0) + Logging.Log.WriteWarningMessage(LOGTAG, "NoFilesRestored", null, "Restore completed without errors but no files were restored"); + + // Drop the temp tables + database.DropRestoreTable(); + await backendManager.WaitForEmptyAsync(database, null, CancellationToken.None).ConfigureAwait(false); + + // Report that the restore is complete + m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_Complete); + m_result.EndTime = DateTime.UtcNow; + } + + /// + /// Perform the restore operation. + /// This is the legacy implementation, which performs the restore in a single thread. Kept as in case the new implementation fails. + /// + /// The database containing information about the restore. + /// The filter of which files to restore. + private async Task DoRun(IBackendManager backendManager, LocalRestoreDatabase database, Library.Utility.IFilter filter, CancellationToken cancellationToken) + { + //In this case, we check that the remote storage fits with the database. + //We can then query the database and find the blocks that we need to do the restore + //using (var database = new LocalRestoreDatabase(dbparent)) + using (var metadatastorage = new RestoreHandlerMetadataStorage()) + { + database.SetResult(m_result); + Utility.UpdateOptionsFromDb(database, m_options); + Utility.VerifyOptionsAndUpdateDatabase(database, m_options); + m_blockbuffer = new byte[m_options.Blocksize]; + + if (!m_options.NoBackendverification) + { + m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PreRestoreVerify); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, database, m_result.BackendWriter, false, null).ConfigureAwait(false); + } + + //Figure out what files are to be patched, and what blocks are needed + m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateFileList); + using (new Logging.Timer(LOGTAG, "PrepareBlockList", "PrepareBlockList")) + PrepareBlockAndFileList(database, m_options, filter, m_result); + + //Make the entire output setup + m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateTargetFolders); + using (new Logging.Timer(LOGTAG, "CreateDirectory", "CreateDirectory")) + await CreateDirectoryStructure(database, m_options, m_result); + + //If we are patching an existing target folder, do not touch stuff that is already updated + m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_ScanForExistingFiles); + using (var blockhasher = HashFactory.CreateHasher(m_options.BlockHashAlgorithm)) + using (var filehasher = HashFactory.CreateHasher(m_options.FileHashAlgorithm)) + using (new Logging.Timer(LOGTAG, "ScanForExistingTargetBlocks", "ScanForExistingTargetBlocks")) + await ScanForExistingTargetBlocks(database, m_blockbuffer, blockhasher, filehasher, m_options, m_result).ConfigureAwait(false); + + //Look for existing blocks in the original source files only + if (m_options.UseLocalBlocks && !string.IsNullOrEmpty(m_options.Restorepath)) + { + using (var blockhasher = HashFactory.CreateHasher(m_options.BlockHashAlgorithm)) + using (new Logging.Timer(LOGTAG, "ScanForExistingSourceBlocksFast", "ScanForExistingSourceBlocksFast")) + { + m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_ScanForLocalBlocks); + await ScanForExistingSourceBlocksFast(database, m_options, m_blockbuffer, blockhasher, m_result).ConfigureAwait(false); + } + } + + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + { + await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false); + return; + } + + // If other local files already have the blocks we want, we use them instead of downloading + if (m_options.UseLocalBlocks) + { + m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PatchWithLocalBlocks); + using (var blockhasher = HashFactory.CreateHasher(m_options.BlockHashAlgorithm)) + using (new Logging.Timer(LOGTAG, "PatchWithLocalBlocks", "PatchWithLocalBlocks")) + await ScanForExistingSourceBlocks(database, m_options, m_blockbuffer, blockhasher, m_result, metadatastorage).ConfigureAwait(false); + } + + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + { + await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false); + return; + } + + // Fill BLOCKS with remote sources + List volumes; + using (new Logging.Timer(LOGTAG, "GetMissingVolumes", "GetMissingVolumes")) + volumes = database.GetMissingVolumes().ToList(); + + if (volumes.Count > 0) + { + Logging.Log.WriteInformationMessage(LOGTAG, "RemoteFileCount", "{0} remote files are required to restore", volumes.Count); + m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_DownloadingRemoteFiles); + } + + var brokenFiles = new List(); + + using (new Logging.Timer(LOGTAG, "PatchWithBlocklist", "PatchWithBlocklist")) + await foreach (var (tmpfile, _, _, name) in backendManager.GetFilesOverlappedAsync(volumes, cancellationToken)) + { + try + { + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + { + await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false); + return; + } + + using (tmpfile) + using (var blocks = new BlockVolumeReader(GetCompressionModule(name), tmpfile, m_options)) + PatchWithBlocklist(database, blocks, m_options, m_result, m_blockbuffer, metadatastorage); + } + catch (Exception ex) + { + brokenFiles.Add(name); + Logging.Log.WriteErrorMessage(LOGTAG, "PatchingFailed", ex, "Failed to patch with remote file: \"{0}\", message: {1}", name, ex.Message); + if (ex is System.Threading.ThreadAbortException) + throw; + } + } + + var fileErrors = 0L; + + // Restore empty files. They might not have any blocks so don't appear in any volume. + foreach (var file in database.GetFilesToRestore(true).Where(item => item.Length == 0)) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "RestoreEmptyFile", "Restoring empty file \"{0}\"", file.Path); + + try + { + SystemIO.IO_OS.DirectoryCreate(SystemIO.IO_OS.PathGetDirectoryName(file.Path)); + // Just create the file and close it right away, empty statement is intentional. + using (SystemIO.IO_OS.FileCreate(file.Path)) + { + } + } + catch (Exception ex) + { + fileErrors++; + Logging.Log.WriteErrorMessage(LOGTAG, "RestoreFileFailed", ex, "Failed to restore empty file: \"{0}\". Error message was: {1}", file.Path, ex.Message); + if (ex is System.Threading.ThreadAbortException) + throw; + } + } + + // Enforcing the length of files is now already done during ScanForExistingTargetBlocks + // and thus not necessary anymore. + + // Apply metadata + if (!m_options.SkipMetadata) + ApplyStoredMetadata(m_options, metadatastorage); + + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + return; + + m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PostRestoreVerify); + + + if (m_options.PerformRestoredFileVerification) + { + // After all blocks in the files are restored, verify the file hash + using (var filehasher = HashFactory.CreateHasher(m_options.FileHashAlgorithm)) + using (new Logging.Timer(LOGTAG, "RestoreVerification", "RestoreVerification")) + foreach (var file in database.GetFilesToRestore(true)) + { + try + { + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + { + await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false); + return; + } + + Logging.Log.WriteVerboseMessage(LOGTAG, "TestFileIntegrity", "Testing restored file integrity: {0}", file.Path); + + string key; + long size; + using (var fs = SystemIO.IO_OS.FileOpenRead(file.Path)) + { + size = fs.Length; + key = Convert.ToBase64String(filehasher.ComputeHash(fs)); + } + + if (key != file.Hash) + throw new Exception(string.Format("Failed to restore file: \"{0}\". File hash is {1}, expected hash is {2}", file.Path, key, file.Hash)); + m_result.RestoredFiles++; + m_result.SizeOfRestoredFiles += size; + } + catch (Exception ex) + { + fileErrors++; + Logging.Log.WriteErrorMessage(LOGTAG, "RestoreFileFailed", ex, "Failed to restore file: \"{0}\". Error message was: {1}", file.Path, ex.Message); + if (ex is System.Threading.ThreadAbortException) + throw; + } + } + } + + if (fileErrors > 0 && brokenFiles.Count > 0) + Logging.Log.WriteInformationMessage(LOGTAG, "RestoreFailures", "Failed to restore {0} files, additionally the following files failed to download, which may be the cause:{1}{2}", fileErrors, Environment.NewLine, string.Join(Environment.NewLine, brokenFiles)); + else if (fileErrors > 0) + Logging.Log.WriteInformationMessage(LOGTAG, "RestoreFailures", "Failed to restore {0} files", fileErrors); + else if (m_result.RestoredFiles == 0) + Logging.Log.WriteWarningMessage(LOGTAG, "NoFilesRestored", null, "Restore completed without errors but no files were restored"); + + // Drop the temp tables + database.DropRestoreTable(); + await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false); + } + + m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_Complete); + m_result.EndTime = DateTime.UtcNow; + } + + public static bool ApplyMetadata(string path, System.IO.Stream stream, bool restorePermissions, bool restoreSymlinkMetadata, bool dryrun) + { + // TODO This has been modified to return a bool indicating if anything was written to properly report the number of files restored. The legacy restore doesn't check this, which produces an error in the CI where it reports that no files have been restored, even though one have. It's in Duplicati/UnitTests/SymLinkTests.cs the test SymLinkTests.SymLinkExists() that fails on the very last assert that there are 0 warnings. + using (var tr = new System.IO.StreamReader(stream)) + using (var jr = new Newtonsoft.Json.JsonTextReader(tr)) + { + var metadata = new Newtonsoft.Json.JsonSerializer().Deserialize>(jr); + string k; + long t; + System.IO.FileAttributes fa; + var wrote_something = false; + + // If this is dry-run, we stop after having deserialized the metadata + if (dryrun) + return wrote_something; + + var isDirTarget = path.EndsWith(DIRSEP, StringComparison.Ordinal); + var targetpath = isDirTarget ? path.Substring(0, path.Length - 1) : path; + + // Make the symlink first, otherwise we cannot apply metadata to it + if (metadata.TryGetValue("CoreSymlinkTarget", out k)) + { + // Check if the target exists, and overwrite it if it does. + if (SystemIO.IO_OS.FileExists(targetpath)) + { + SystemIO.IO_OS.FileDelete(targetpath); + } + else if (SystemIO.IO_OS.DirectoryExists(targetpath)) + { + SystemIO.IO_OS.DirectoryDelete(targetpath, false); + } + SystemIO.IO_OS.CreateSymlink(targetpath, k, isDirTarget); + wrote_something = true; + } + // If the target is a folder, make sure we create it first + else if (isDirTarget && !SystemIO.IO_OS.DirectoryExists(targetpath)) + SystemIO.IO_OS.DirectoryCreate(targetpath); + + // Avoid setting restoring symlink metadata, as that writes the symlink target, not the symlink itself + if (!restoreSymlinkMetadata && Snapshots.SnapshotUtility.IsSymlink(SystemIO.IO_OS, targetpath)) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "no-symlink-metadata-restored", "Not applying metadata to symlink: {0}", targetpath); + return wrote_something; + } + + if (metadata.TryGetValue("CoreLastWritetime", out k) && long.TryParse(k, out t)) + { + if (isDirTarget) + SystemIO.IO_OS.DirectorySetLastWriteTimeUtc(targetpath, new DateTime(t, DateTimeKind.Utc)); + else + SystemIO.IO_OS.FileSetLastWriteTimeUtc(targetpath, new DateTime(t, DateTimeKind.Utc)); + } + + if (metadata.TryGetValue("CoreCreatetime", out k) && long.TryParse(k, out t)) + { + if (isDirTarget) + SystemIO.IO_OS.DirectorySetCreationTimeUtc(targetpath, new DateTime(t, DateTimeKind.Utc)); + else + SystemIO.IO_OS.FileSetCreationTimeUtc(targetpath, new DateTime(t, DateTimeKind.Utc)); + } + + if (metadata.TryGetValue("CoreAttributes", out k) && Enum.TryParse(k, true, out fa)) + SystemIO.IO_OS.SetFileAttributes(targetpath, fa); + + SystemIO.IO_OS.SetMetadata(path, metadata, restorePermissions); + return wrote_something; + } + } + + private static async Task ScanForExistingSourceBlocksFast(LocalRestoreDatabase database, Options options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, RestoreResults result) + { + // Fill BLOCKS with data from known local source files + using (var blockmarker = database.CreateBlockMarker()) + { + var updateCount = 0L; + foreach (var entry in database.GetFilesAndSourceBlocksFast(options.Blocksize)) + { + var targetpath = entry.TargetPath; + var targetfileid = entry.TargetFileID; + var sourcepath = entry.SourcePath; + var patched = false; + + try + { + if (SystemIO.IO_OS.FileExists(sourcepath)) + { + var folderpath = SystemIO.IO_OS.PathGetDirectoryName(targetpath); + if (!options.Dryrun && !SystemIO.IO_OS.DirectoryExists(folderpath)) + { + Logging.Log.WriteWarningMessage(LOGTAG, "CreateMissingFolder", null, "Creating missing folder {0} for file {1}", folderpath, targetpath); + SystemIO.IO_OS.DirectoryCreate(folderpath); + } + + using (var targetstream = options.Dryrun ? null : SystemIO.IO_OS.FileOpenWrite(targetpath)) + { + try + { + using (var sourcestream = SystemIO.IO_OS.FileOpenRead(sourcepath)) + { + foreach (var block in entry.Blocks) + { + if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + return; + + //TODO: Handle metadata + + if (sourcestream.Length > block.Offset) + { + sourcestream.Position = block.Offset; + + int size = Library.Utility.Utility.ForceStreamRead(sourcestream, blockbuffer, blockbuffer.Length); + if (size == block.Size) + { + var key = Convert.ToBase64String(hasher.ComputeHash(blockbuffer, 0, size)); + if (key == block.Hash) + { + patched = true; + if (!options.Dryrun) + { + targetstream.Position = block.Offset; + targetstream.Write(blockbuffer, 0, size); + } + + blockmarker.SetBlockRestored(targetfileid, block.Index, key, block.Size, false); + } + } + } + } + } + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "PatchingFileLocalFailed", ex, "Failed to patch file: \"{0}\" with data from local file \"{1}\", message: {2}", targetpath, sourcepath, ex.Message); + if (ex is System.Threading.ThreadAbortException) + throw; + } + } + + if ((++updateCount) % 20 == 0) + { + blockmarker.UpdateProcessed(result.OperationProgressUpdater); + if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + return; + } + + } + else + { + Logging.Log.WriteVerboseMessage(LOGTAG, "LocalSourceMissing", "Local source file not found: {0}", sourcepath); + } + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "PatchingFileLocalFailed", ex, "Failed to patch file: \"{0}\" with local data, message: {1}", targetpath, ex.Message); + if (ex is System.Threading.ThreadAbortException) + throw; + if (options.UnittestMode) + throw; + } + + if (patched) + Logging.Log.WriteVerboseMessage(LOGTAG, "FilePatchedWithLocal", "Target file is patched with some local data: {0}", targetpath); + else + Logging.Log.WriteVerboseMessage(LOGTAG, "FilePatchedWithLocal", "Target file is not patched any local data: {0}", targetpath); + + if (patched && options.Dryrun) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPatchWithLocal", "Would patch file with local data: {0}", targetpath); + } + + blockmarker.UpdateProcessed(result.OperationProgressUpdater); + blockmarker.Commit(); + } + } + + private static async Task ScanForExistingSourceBlocks(LocalRestoreDatabase database, Options options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, RestoreResults result, RestoreHandlerMetadataStorage metadatastorage) + { + // Fill BLOCKS with data from known local source files + using (var blockmarker = database.CreateBlockMarker()) + { + var updateCount = 0L; + foreach (var restorelist in database.GetFilesAndSourceBlocks(options.SkipMetadata, options.Blocksize)) + { + var targetpath = restorelist.TargetPath; + var targetfileid = restorelist.TargetFileID; + var patched = false; + try + { + if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + return; + + var folderpath = SystemIO.IO_OS.PathGetDirectoryName(targetpath); + if (!options.Dryrun && !SystemIO.IO_OS.DirectoryExists(folderpath)) + { + Logging.Log.WriteWarningMessage(LOGTAG, "CreateMissingFolder", null, "Creating missing folder {0} for file {1}", folderpath, targetpath); + SystemIO.IO_OS.DirectoryCreate(folderpath); + } + + using (var file = options.Dryrun ? null : SystemIO.IO_OS.FileOpenWrite(targetpath)) + foreach (var targetblock in restorelist.Blocks) + { + foreach (var source in targetblock.Blocksources) + { + try + { + if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + return; + + if (SystemIO.IO_OS.FileExists(source.Path)) + { + if (source.IsMetadata) + { + // TODO: Handle this by reconstructing + // metadata from file and checking the hash + + continue; + } + else + { + using (var sourcefile = SystemIO.IO_OS.FileOpenRead(source.Path)) + { + sourcefile.Position = source.Offset; + int size = Library.Utility.Utility.ForceStreamRead(sourcefile, blockbuffer, blockbuffer.Length); + if (size == targetblock.Size) + { + var key = Convert.ToBase64String(hasher.ComputeHash(blockbuffer, 0, size)); + if (key == targetblock.Hash) + { + if (!options.Dryrun) + { + if (targetblock.IsMetadata) + metadatastorage.Add(targetpath, new System.IO.MemoryStream(blockbuffer, 0, size)); + else + { + file.Position = targetblock.Offset; + file.Write(blockbuffer, 0, size); + } + } + + blockmarker.SetBlockRestored(targetfileid, targetblock.Index, key, targetblock.Size, false); + patched = true; + break; + } + } + } + } + } + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "PatchingFileLocalFailed", ex, "Failed to patch file: \"{0}\" with data from local file \"{1}\", message: {2}", targetpath, source.Path, ex.Message); + if (ex is System.Threading.ThreadAbortException) + throw; + } + } + } + + if ((++updateCount) % 20 == 0) + blockmarker.UpdateProcessed(result.OperationProgressUpdater); + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "PatchingFileLocalFailed", ex, "Failed to patch file: \"{0}\" with local data, message: {1}", targetpath, ex.Message); + if (options.UnittestMode) + throw; + } + + if (patched) + Logging.Log.WriteVerboseMessage(LOGTAG, "FilePatchedWithLocal", "Target file is patched with some local data: {0}", targetpath); + else + Logging.Log.WriteVerboseMessage(LOGTAG, "FilePatchedWithLocal", "Target file is not patched any local data: {0}", targetpath); + + if (patched && options.Dryrun) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPatchWithLocal", string.Format("Would patch file with local data: {0}", targetpath)); + } + + blockmarker.UpdateProcessed(result.OperationProgressUpdater); + blockmarker.Commit(); + } + } + + private static void PrepareBlockAndFileList(LocalRestoreDatabase database, Options options, Library.Utility.IFilter filter, RestoreResults result) + { + // Create a temporary table FILES by selecting the files from fileset that matches a specific operation id + // Delete all entries from the temp table that are excluded by the filter(s) + using (new Logging.Timer(LOGTAG, "PrepareRestoreFileList", "PrepareRestoreFileList")) + { + var c = database.PrepareRestoreFilelist(options.Time, options.Version, filter); + result.OperationProgressUpdater.UpdatefileCount(c.Item1, c.Item2, true); + } + + using (new Logging.Timer(LOGTAG, "SetTargetPaths", "SetTargetPaths")) + if (!string.IsNullOrEmpty(options.Restorepath)) + { + // Find the largest common prefix + var largest_prefix = options.DontCompressRestorePaths ? "" : database.GetLargestPrefix(); + + Logging.Log.WriteVerboseMessage(LOGTAG, "MappingRestorePath", "Mapping restore path prefix to \"{0}\" to \"{1}\"", largest_prefix, Util.AppendDirSeparator(options.Restorepath)); + + // Set the target paths, special care with C:\ and / + database.SetTargetPaths(largest_prefix, Util.AppendDirSeparator(options.Restorepath)); + } + else + { + database.SetTargetPaths("", ""); + } + + // Create a temporary table BLOCKS that lists all blocks that needs to be recovered + using (new Logging.Timer(LOGTAG, "FindMissingBlocks", "FindMissingBlocks")) + database.FindMissingBlocks(options.SkipMetadata); + + // Create temporary tables and triggers that automatically track progress + using (new Logging.Timer(LOGTAG, "CreateProgressTracker", "CreateProgressTracker")) + database.CreateProgressTracker(false); + + } + + private static async Task CreateDirectoryStructure(LocalRestoreDatabase database, Options options, RestoreResults result) + { + // This part is not protected by try/catch as we need the target folder to exist + if (!string.IsNullOrEmpty(options.Restorepath)) + if (!SystemIO.IO_OS.DirectoryExists(options.Restorepath)) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "CreateFolder", "Creating folder: {0}", options.Restorepath); + + if (options.Dryrun) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldCreateFolder", "Would create folder: {0}", options.Restorepath); + else + SystemIO.IO_OS.DirectoryCreate(options.Restorepath); + } + + foreach (var folder in database.GetTargetFolders()) + { + try + { + if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + return; + + if (!SystemIO.IO_OS.DirectoryExists(folder)) + { + result.RestoredFolders++; + + Logging.Log.WriteVerboseMessage(LOGTAG, "CreateFolder", "Creating folder: {0}", folder); + + if (options.Dryrun) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldCreateFolder", "Would create folder: {0}", folder); + else + SystemIO.IO_OS.DirectoryCreate(folder); + } + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "FolderCreateFailed", ex, "Failed to create folder: \"{0}\", message: {1}", folder, ex.Message); + if (options.UnittestMode) + throw; + } + } + } + + private static async Task ScanForExistingTargetBlocks(LocalRestoreDatabase database, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm blockhasher, System.Security.Cryptography.HashAlgorithm filehasher, Options options, RestoreResults result) + { + // Scan existing files for existing BLOCKS + using (var blockmarker = database.CreateBlockMarker()) + { + var updateCount = 0L; + foreach (var restorelist in database.GetExistingFilesWithBlocks()) + { + var rename = !options.Overwrite; + var targetpath = restorelist.TargetPath; + var targetfileid = restorelist.TargetFileID; + var targetfilehash = restorelist.TargetHash; + var targetfilelength = restorelist.Length; + if (SystemIO.IO_OS.FileExists(targetpath)) + { + try + { + if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + return; + + var currentfilelength = SystemIO.IO_OS.FileLength(targetpath); + var wasTruncated = false; + + // Adjust file length in overwrite mode if necessary (smaller is ok, will be extended during restore) + // We do it before scanning for blocks. This allows full verification on files that only needs to + // be truncated (i.e. forthwritten log files). + if (!rename && currentfilelength > targetfilelength) + { + var currentAttr = SystemIO.IO_OS.GetFileAttributes(targetpath); + if ((currentAttr & System.IO.FileAttributes.ReadOnly) != 0) // clear readonly attribute + { + if (options.Dryrun) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldResetReadOnlyAttribute", "Would reset read-only attribute on file: {0}", targetpath); + else SystemIO.IO_OS.SetFileAttributes(targetpath, currentAttr & ~System.IO.FileAttributes.ReadOnly); + } + if (options.Dryrun) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldTruncateFile", "Would truncate file '{0}' to length of {1:N0} bytes", targetpath, targetfilelength); + else + { + using (var file = SystemIO.IO_OS.FileOpenWrite(targetpath)) + file.SetLength(targetfilelength); + currentfilelength = targetfilelength; + } + wasTruncated = true; + } + + // If file size does not match and we have to rename on conflict, + // the whole scan can be skipped here because all blocks have to be restored anyway. + // For the other cases, we will check block and and file hashes and look for blocks + // to be restored and files that can already be verified. + if (!rename || currentfilelength == targetfilelength) + { + // a file hash for verification will only be necessary if the file has exactly + // the wanted size so we have a chance to already mark the file as data-verified. + bool calcFileHash = (currentfilelength == targetfilelength); + if (calcFileHash) filehasher.Initialize(); + + using (var file = SystemIO.IO_OS.FileOpenRead(targetpath)) + using (var block = new Blockprocessor(file, blockbuffer)) + foreach (var targetblock in restorelist.Blocks) + { + var size = block.Readblock(); + if (size <= 0) + break; + + //TODO: Handle Metadata + + bool blockhashmatch = false; + if (size == targetblock.Size) + { + // Parallelize file hash calculation on rename. Running read-only on same array should not cause conflicts or races. + // Actually, in future always calculate the file hash and mark the file data as already verified. + + System.Threading.Tasks.Task calcFileHashTask = null; + if (calcFileHash) + calcFileHashTask = System.Threading.Tasks.Task.Run( + () => filehasher.TransformBlock(blockbuffer, 0, size, blockbuffer, 0)); + + var key = Convert.ToBase64String(blockhasher.ComputeHash(blockbuffer, 0, size)); + + if (calcFileHashTask != null) await calcFileHashTask.ConfigureAwait(false); // wait because blockbuffer will be overwritten. + + if (key == targetblock.Hash) + { + blockmarker.SetBlockRestored(targetfileid, targetblock.Index, key, size, false); + blockhashmatch = true; + } + } + if (calcFileHash && !blockhashmatch) // will not be necessary anymore + { + filehasher.TransformFinalBlock(blockbuffer, 0, 0); // So a new initialize will not throw + calcFileHash = false; + if (rename) // file does not match. So break. + break; + } + } + + bool fullfilehashmatch = false; + if (calcFileHash) // now check if files are identical + { + filehasher.TransformFinalBlock(blockbuffer, 0, 0); + var filekey = Convert.ToBase64String(filehasher.Hash); + fullfilehashmatch = (filekey == targetfilehash); + } + + if (!rename && !fullfilehashmatch && !wasTruncated) // Reset read-only attribute (if set) to overwrite + { + var currentAttr = SystemIO.IO_OS.GetFileAttributes(targetpath); + if ((currentAttr & System.IO.FileAttributes.ReadOnly) != 0) + { + if (options.Dryrun) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldResetReadOnlyAttribyte", "Would reset read-only attribute on file: {0}", targetpath); + else SystemIO.IO_OS.SetFileAttributes(targetpath, currentAttr & ~System.IO.FileAttributes.ReadOnly); + } + } + + if (fullfilehashmatch) + { + //TODO: Check metadata to trigger rename? If metadata changed, it will still be restored for the file in-place. + blockmarker.SetFileDataVerified(targetfileid); + Logging.Log.WriteVerboseMessage(LOGTAG, "TargetExistsInCorrectVersion", "Target file exists{1} and is correct version: {0}", targetpath, wasTruncated ? " (but was truncated)" : ""); + rename = false; + } + else if (rename) + { + // The new file will have none of the correct blocks, + // even if the scanned file had some + blockmarker.SetAllBlocksMissing(targetfileid); + } + } + + if ((++updateCount) % 20 == 0) + { + blockmarker.UpdateProcessed(result.OperationProgressUpdater); + if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + return; + } + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "TargetFileReadError", ex, "Failed to read target file: \"{0}\", message: {1}", targetpath, ex.Message); + if (ex is System.Threading.ThreadAbortException) + throw; + if (options.UnittestMode) + throw; + } + } + else + { + Logging.Log.WriteVerboseMessage(LOGTAG, "MissingTargetFile", "Target file does not exist: {0}", targetpath); + rename = false; + } + + if (rename) + { + //Select a new filename + var ext = SystemIO.IO_OS.PathGetExtension(targetpath) ?? ""; + if (!string.IsNullOrEmpty(ext) && !ext.StartsWith(".", StringComparison.Ordinal)) + ext = "." + ext; + + // First we try with a simple date append, assuming that there are not many conflicts there + var newname = SystemIO.IO_OS.PathChangeExtension(targetpath, null) + "." + database.RestoreTime.ToLocalTime().ToString("yyyy-MM-dd", System.Globalization.CultureInfo.InvariantCulture); + var tr = newname + ext; + var c = 0; + while (SystemIO.IO_OS.FileExists(tr) && c < 1000) + { + try + { + // If we have a file with the correct name, + // it is most likely the file we want + filehasher.Initialize(); + + string key; + using (var file = SystemIO.IO_OS.FileOpenRead(tr)) + key = Convert.ToBase64String(filehasher.ComputeHash(file)); + + if (key == targetfilehash) + { + //TODO: Also needs metadata check to make correct decision. + // We stick to the policy to restore metadata in place, if data ok. So, metadata block may be restored. + blockmarker.SetAllBlocksRestored(targetfileid, false); + blockmarker.SetFileDataVerified(targetfileid); + break; + } + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "FailedToReadRestoreTarget", ex, "Failed to read candidate restore target {0}", tr); + if (options.UnittestMode) + throw; + } + tr = newname + " (" + (c++).ToString() + ")" + ext; + } + + newname = tr; + + Logging.Log.WriteVerboseMessage(LOGTAG, "TargetFileRetargeted", "Target file exists and will be restored to: {0}", newname); + database.UpdateTargetPath(targetfileid, newname); + } + + } + + blockmarker.UpdateProcessed(result.OperationProgressUpdater); + blockmarker.Commit(); + } + } + } +} From a9f328ae95078b24097190558b3504f3229a4370 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 19 Feb 2025 16:11:37 +0100 Subject: [PATCH 207/362] Add back missing Windows example script. This adds back the missing `run-script-example.bat`. It also makes the output conditional, so the Windows builds get the `.bat` and others get the `.sh` version. --- .../Duplicati.Library.Modules.Builtin.csproj | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Duplicati/Library/Modules/Builtin/Duplicati.Library.Modules.Builtin.csproj b/Duplicati/Library/Modules/Builtin/Duplicati.Library.Modules.Builtin.csproj index 3d2055c85..26b025594 100644 --- a/Duplicati/Library/Modules/Builtin/Duplicati.Library.Modules.Builtin.csproj +++ b/Duplicati/Library/Modules/Builtin/Duplicati.Library.Modules.Builtin.csproj @@ -24,10 +24,16 @@ - - - PreserveNewest - + + + PreserveNewest + + + + + + PreserveNewest + From 374741fc6dd492b2f085e00ac1cee7e353531c57 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 19 Feb 2025 16:11:55 +0100 Subject: [PATCH 208/362] Added BackendManager.GetDirectAsync(), which downloads a file without decrypting it --- .../Backend/BackendManager.GetOperation.cs | 5 ++++ .../Library/Main/Backend/BackendManager.cs | 26 ++++++++++++++++--- Duplicati/Library/Main/IBackendManager.cs | 13 +++++++--- 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/Duplicati/Library/Main/Backend/BackendManager.GetOperation.cs b/Duplicati/Library/Main/Backend/BackendManager.GetOperation.cs index 0381f1833..1d64b2ccd 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.GetOperation.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.GetOperation.cs @@ -31,6 +31,10 @@ partial class BackendManager /// public override long Size { get; } /// + /// Flag indicating whether the file should be decrypted + /// + public bool Decrypt { get; set; } + /// /// The hash of the remote file, or null if unknown /// public required string? Hash { get; set; } @@ -89,6 +93,7 @@ partial class BackendManager } // Perform decryption after hash validation, if needed + if (Decrypt) tmpfile = DecryptFile(tmpfile, RemoteFilename, Context.Options); return (tmpfile, fileHash, dataSizeDownloaded); diff --git a/Duplicati/Library/Main/Backend/BackendManager.cs b/Duplicati/Library/Main/Backend/BackendManager.cs index a9762e5be..dc3f0a91f 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.cs @@ -178,7 +178,8 @@ internal partial class BackendManager : IBackendManager { var op = new GetOperation(remotename, size, context, cancelToken) { - Hash = hash + Hash = hash, + Decrypt = true }; await QueueTask(op).ConfigureAwait(false); (var file, var _, var downloadSize) = await op.GetResult().ConfigureAwait(false); @@ -186,9 +187,25 @@ internal partial class BackendManager : IBackendManager return file; } - public IBackend GetBackend() + /// + /// Gets a file from the remote location without decrypting it + /// + /// The name of the remote file + /// The hash of the remote file, for verification + /// The size of the remote file, for verification + /// The cancellation token + /// A temporary file with the contents of the remote file + public async Task GetDirectAsync(string remotename, string hash, long size, CancellationToken cancelToken) { - return BackendLoader.GetBackend(m_backendurl, m_options.RawOptions); + var op = new GetOperation(remotename, size, context, cancelToken) + { + Hash = hash, + Decrypt = false + }; + await QueueTask(op).ConfigureAwait(false); + (var file, var _, var downloadSize) = await op.GetResult().ConfigureAwait(false); + LastReadSize = downloadSize; + return file; } /// @@ -215,7 +232,8 @@ internal partial class BackendManager : IBackendManager { var op = new GetOperation(remotename, size, context, cancelToken) { - Hash = hash + Hash = hash, + Decrypt = true }; await QueueTask(op).ConfigureAwait(false); (var file, var downloadHash, var downloadSize) = await op.GetResult().ConfigureAwait(false); diff --git a/Duplicati/Library/Main/IBackendManager.cs b/Duplicati/Library/Main/IBackendManager.cs index 2d01d1c8b..c9cd0372c 100644 --- a/Duplicati/Library/Main/IBackendManager.cs +++ b/Duplicati/Library/Main/IBackendManager.cs @@ -97,11 +97,18 @@ internal interface IBackendManager : IDisposable /// The hash of the file to get, or null if not known /// The size of the file to get, or -1 if not known /// The cancellation token - /// + /// The downloaded file Task GetAsync(string remotename, string hash, long size, CancellationToken cancelToken); - - public IBackend GetBackend(); + /// + /// Gets a file from the backend without decrypting it + /// + /// The name of the remote volume + /// The hash of the volume + /// The size of the volume + /// The cancellation token + /// The downloaded file + Task GetDirectAsync(string remotename, string hash, long size, CancellationToken cancelToken); /// /// Performs a download of the files specified, with pre-fetch to overlap the download and processing From 9919d86e9b7dd60705d5f95891a267efd8add666 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 19 Feb 2025 16:12:42 +0100 Subject: [PATCH 209/362] Updated VolumeDownloader to use the new BackendManager.GetDirectAsync() method --- .../Main/Operation/Restore/VolumeDownloader.cs | 11 +++-------- Duplicati/Library/Main/Operation/RestoreHandler.cs | 2 +- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs b/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs index 65c20c92a..2f2b543f9 100644 --- a/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs +++ b/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs @@ -25,7 +25,6 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; using CoCoL; -using Duplicati.Library.Interface; using Duplicati.Library.Main.Database; using Duplicati.Library.Utility; @@ -50,7 +49,7 @@ namespace Duplicati.Library.Main.Operation.Restore /// The backend to use for downloading the volumes. /// The restore options. /// The restore results. - public static Task Run(Channels channels, LocalRestoreDatabase db, IBackend backend, Options options, RestoreResults results) + public static Task Run(Channels channels, LocalRestoreDatabase db, IBackendManager backend, Options options, RestoreResults results) { return AutomationExtensions.RunTask( new @@ -75,11 +74,11 @@ namespace Duplicati.Library.Main.Operation.Restore // Trigger the download. sw_wait?.Start(); - TempFile f = new(); + TempFile f; var (volume_name, size, hash) = db.GetVolumeInfo(volume_id).First(); try { - await backend.GetAsync(volume_name, f.Name, CancellationToken.None).ConfigureAwait(false); + f = await backend.GetDirectAsync(volume_name, hash, size, CancellationToken.None).ConfigureAwait(false); } catch (Exception) { @@ -110,10 +109,6 @@ namespace Duplicati.Library.Main.Operation.Restore Logging.Log.WriteErrorMessage(LOGTAG, "DownloadError", ex, "Error during download"); throw; } - finally - { - backend.Dispose(); - } }); } } diff --git a/Duplicati/Library/Main/Operation/RestoreHandler.cs b/Duplicati/Library/Main/Operation/RestoreHandler.cs index 4b9bdc07a..829e91715 100644 --- a/Duplicati/Library/Main/Operation/RestoreHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreHandler.cs @@ -340,7 +340,7 @@ namespace Duplicati.Library.Main.Operation var fileprocessors = Enumerable.Range(0, m_options.RestoreFileProcessors).Select(i => Restore.FileProcessor.Run(channels, database, fileprocessor_requests[i], fileprocessor_responses[i], m_options, m_result)).ToArray(); var blockmanager = Restore.BlockManager.Run(channels, database, m_options, fileprocessor_requests, fileprocessor_responses); var volumecache = Restore.VolumeManager.Run(channels, m_options); - var volumedownloaders = Enumerable.Range(0, m_options.RestoreVolumeDownloaders).Select(i => Restore.VolumeDownloader.Run(channels, database, backendManager.GetBackend(), m_options, m_result)).ToArray(); + var volumedownloaders = Enumerable.Range(0, m_options.RestoreVolumeDownloaders).Select(i => Restore.VolumeDownloader.Run(channels, database, backendManager, m_options, m_result)).ToArray(); var volumedecryptors = Enumerable.Range(0, m_options.RestoreVolumeDecryptors).Select(i => Restore.VolumeDecryptor.Run(channels, backendManager, m_options)).ToArray(); var volumedecompressors = Enumerable.Range(0, m_options.RestoreVolumeDecompressors).Select(i => Restore.VolumeDecompressor.Run(channels, m_options)).ToArray(); From beefbf2a4610be3122cd7120e06a9330abc6d69e Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 19 Feb 2025 16:12:58 +0100 Subject: [PATCH 210/362] Removed some unused variables from the BackendManager --- Duplicati/Library/Main/Backend/BackendManager.cs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/Duplicati/Library/Main/Backend/BackendManager.cs b/Duplicati/Library/Main/Backend/BackendManager.cs index dc3f0a91f..2e5d6d4b7 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.cs @@ -26,13 +26,6 @@ internal partial class BackendManager : IBackendManager /// The log tag for the class /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(); - /// - /// List of backend instances that are currently in use - /// - private readonly List backendPool = []; - - private string m_backendurl; - private Options m_options; /// /// The channel for issuing and handling requests @@ -75,9 +68,6 @@ internal partial class BackendManager : IBackendManager if (string.IsNullOrWhiteSpace(backendUrl)) throw new ArgumentNullException(nameof(backendUrl)); - m_backendurl = backendUrl; - m_options = options; - // To avoid excessive parameter passing, the context is captured here context = new ExecuteContext( new ProgressHandler(options, backendWriter, taskReader).HandleProgress, From 80d60c09826ea38d944fc84d47eeb244696790a1 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 19 Feb 2025 22:23:53 +0100 Subject: [PATCH 211/362] Fix Uri ToString() Fixed a case where an extra leading slash would be removed when converting a parsed Uri back to a string. --- Duplicati/Library/Utility/Uri.cs | 40 ++++++++++++++++---------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/Duplicati/Library/Utility/Uri.cs b/Duplicati/Library/Utility/Uri.cs index 344dcc78d..bc7505a7c 100644 --- a/Duplicati/Library/Utility/Uri.cs +++ b/Duplicati/Library/Utility/Uri.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -273,7 +273,7 @@ namespace Duplicati.Library.Utility if (!string.IsNullOrEmpty(path)) { - if (!string.IsNullOrEmpty(host) && !path.StartsWith("/", StringComparison.Ordinal)) + if (!string.IsNullOrEmpty(host)) s += "/"; s += path; } From eacd46dcd8d51b9c12cb80bda5db8de575d2a332 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 20 Feb 2025 09:00:07 +0100 Subject: [PATCH 212/362] Added parallel execution to the test data generator --- Tools/TestDataGenerator/Commands/Create.cs | 76 ++++++++++++++-------- 1 file changed, 50 insertions(+), 26 deletions(-) diff --git a/Tools/TestDataGenerator/Commands/Create.cs b/Tools/TestDataGenerator/Commands/Create.cs index fb9591bd4..8eadc26a6 100644 --- a/Tools/TestDataGenerator/Commands/Create.cs +++ b/Tools/TestDataGenerator/Commands/Create.cs @@ -1,3 +1,4 @@ +using System.Collections.Concurrent; using System.CommandLine; using System.CommandLine.NamingConventionBinder; using static TestDataGenerator.Commands.Shared; @@ -173,46 +174,69 @@ public static class Create } } + object stats_lock = new(); var filesCreated = 0L; var fileSizeCreated = 0L; var generateStart = DateTime.UtcNow; - foreach (var (file, size) in filesWithSizes) + var cancelToken = new CancellationTokenSource(); + var status_updater = Task.Run(async () => { - var filename = file; - try + while (!cancelToken.Token.IsCancellationRequested) { - // Sometimes with a large number of files, the file name clashes with a folder name. In that case, generate a new file name - while (Directory.Exists(filename)) - filename = Path.Combine(Path.GetDirectoryName(filename) ?? "", GenerateFileName(rnd, input.MaxPathSegmentLength)); - - using var fs = new FileStream(filename, FileMode.Create, FileAccess.Write); - if (size > 0) + await Task.Delay((int)updateInterval.TotalMilliseconds, cancelToken.Token); + long local_filesCreated, local_fileSizeCreated; + lock (stats_lock) { - fs.SetLength(size); - fs.Position = 0; - - WriteRandomData(rnd, fs, size, input.SparseFactor); + local_filesCreated = filesCreated; + local_fileSizeCreated = fileSizeCreated; } - filesCreated++; - fileSizeCreated += size; var generateNow = DateTime.UtcNow; - if ((generateNow - lastUpdate) > updateInterval) + long throughput = (long)Math.Max(Math.Floor(local_fileSizeCreated / (generateNow - generateStart).TotalSeconds), 1); + var timeLeft = TimeSpan.FromSeconds((input.MaxTotalSize - local_fileSizeCreated) / throughput); + string timeLeftString = timeLeft.ToString(@"hh\:mm\:ss"); + Console.WriteLine($"Created {local_filesCreated} of {files.Count} files ({SizeToHumanReadable(local_fileSizeCreated)} of {SizeToHumanReadable(totalSize)}) ({SizeToHumanReadable(throughput)}/s) - ETA: {timeLeftString}"); + } + }, cancelToken.Token); + + Parallel.ForEach(Partitioner.Create(0, filesWithSizes.Count), range => + { + var local_rnd = new Random(input.RandomSeed.GetHashCode() + range.Item1); + for (int i = range.Item1; i < range.Item2; i++) + { + var (file, size) = filesWithSizes[i]; + var filename = file; + try { - long throughput = (long)Math.Floor(fileSizeCreated / (generateNow - generateStart).TotalSeconds); - var timeLeft = TimeSpan.FromSeconds((input.MaxTotalSize - fileSizeCreated) / throughput); - string timeLeftString = timeLeft.ToString(@"hh\:mm\:ss"); - Console.WriteLine($"Created {filesCreated} of {files.Count} files ({SizeToHumanReadable(fileSizeCreated)} of {SizeToHumanReadable(totalSize)}) ({SizeToHumanReadable(throughput)}/s) - ETA: {timeLeftString}"); - lastUpdate = generateNow; + // Sometimes with a large number of files, the file name clashes with a folder name. In that case, generate a new file name + while (Directory.Exists(filename)) + filename = Path.Combine(Path.GetDirectoryName(filename) ?? "", GenerateFileName(local_rnd, input.MaxPathSegmentLength)); + + using var fs = new FileStream(filename, FileMode.Create, FileAccess.Write); + if (size > 0) + { + fs.SetLength(size); + fs.Position = 0; + + WriteRandomData(local_rnd, fs, size, input.SparseFactor); + } + + lock (stats_lock) + { + filesCreated++; + fileSizeCreated += size; + } + } + catch (Exception ex) + { + Console.WriteLine($"Error creating file {filename}: {ex.Message}"); } } - catch (Exception ex) - { - Console.WriteLine($"Error creating file {filename}: {ex.Message}"); - } - } + }); + cancelToken.Cancel(); + Console.WriteLine($"Created {filesCreated} of {files.Count} files ({SizeToHumanReadable(fileSizeCreated)} of {SizeToHumanReadable(totalSize)} at {SizeToHumanReadable((long)(fileSizeCreated / (DateTime.UtcNow - generateStart).TotalSeconds))}/s)"); } /// From ef6d708d3ac880325e75d463e8f1da8ddea56263 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 20 Feb 2025 09:23:47 +0100 Subject: [PATCH 213/362] Added degree of parallelism as a commandline option to the test data generator --- Tools/TestDataGenerator/Commands/Create.cs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/Tools/TestDataGenerator/Commands/Create.cs b/Tools/TestDataGenerator/Commands/Create.cs index 8eadc26a6..967e6f7c3 100644 --- a/Tools/TestDataGenerator/Commands/Create.cs +++ b/Tools/TestDataGenerator/Commands/Create.cs @@ -24,6 +24,7 @@ public static class Create /// The maximum number of folders to create in the target folder /// The maximum length of each path segment in the target folder /// The random seed to use for generating the test data + /// The number of parallel tasks to use for creating the test data record CommandInput( DirectoryInfo TargetFolder, int FileCount, @@ -35,7 +36,8 @@ public static class Create int MaxDepth, int MaxFolderCount, int MaxPathSegmentLength, - string RandomSeed); + string RandomSeed, + int Parallel); /// /// Creates the command @@ -89,6 +91,10 @@ public static class Create randomSeedOption.SetDefaultValue("Duplicati"); command.AddOption(randomSeedOption); + var parallelOption = new Option("--parallel", "The number of parallel tasks to use for creating the test data"); + parallelOption.SetDefaultValue(Environment.ProcessorCount); + command.AddOption(parallelOption); + command.Handler = CommandHandler.Create(Execute); return command; @@ -200,7 +206,7 @@ public static class Create } }, cancelToken.Token); - Parallel.ForEach(Partitioner.Create(0, filesWithSizes.Count), range => + Parallel.ForEach(Partitioner.Create(0, filesWithSizes.Count), new ParallelOptions { MaxDegreeOfParallelism = input.Parallel }, range => { var local_rnd = new Random(input.RandomSeed.GetHashCode() + range.Item1); for (int i = range.Item1; i < range.Item2; i++) @@ -210,8 +216,8 @@ public static class Create try { // Sometimes with a large number of files, the file name clashes with a folder name. In that case, generate a new file name - while (Directory.Exists(filename)) - filename = Path.Combine(Path.GetDirectoryName(filename) ?? "", GenerateFileName(local_rnd, input.MaxPathSegmentLength)); + for (int j = 0; j < 10 && Path.Exists(filename); j++) + filename = Path.Combine(Path.GetDirectoryName(filename) ?? "", "_" + Path.GetFileName(filename)); using var fs = new FileStream(filename, FileMode.Create, FileAccess.Write); if (size > 0) @@ -236,7 +242,8 @@ public static class Create }); cancelToken.Cancel(); - Console.WriteLine($"Created {filesCreated} of {files.Count} files ({SizeToHumanReadable(fileSizeCreated)} of {SizeToHumanReadable(totalSize)} at {SizeToHumanReadable((long)(fileSizeCreated / (DateTime.UtcNow - generateStart).TotalSeconds))}/s)"); + var time_string = (DateTime.UtcNow - generateStart).ToString(@"hh\:mm\:ss"); + Console.WriteLine($"Created {filesCreated} of {files.Count} files ({SizeToHumanReadable(fileSizeCreated)} of {SizeToHumanReadable(totalSize)} at {SizeToHumanReadable((long)(fileSizeCreated / (DateTime.UtcNow - generateStart).TotalSeconds))}/s) in {time_string}"); } /// From def7327b214f729e9b445209376897e34a077281 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 20 Feb 2025 10:51:10 +0100 Subject: [PATCH 214/362] Fixed CORS preflight This fixes the CORS preflight by allowing the preflight requests to be sent without being authenticated. Without this fix all CORS requests would fail due to the access token not being included in preflight requests. --- Duplicati/WebserverCore/DuplicatiWebserver.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Duplicati/WebserverCore/DuplicatiWebserver.cs b/Duplicati/WebserverCore/DuplicatiWebserver.cs index 9f5f0b4fc..5d092691b 100644 --- a/Duplicati/WebserverCore/DuplicatiWebserver.cs +++ b/Duplicati/WebserverCore/DuplicatiWebserver.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2025, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a @@ -284,6 +284,9 @@ public class DuplicatiWebserver var app = builder.Build(); HttpClientHelper.Configure(app.Services.GetRequiredService()); + if (useCors) + app.UseCors(CorsPolicyName); + app.UseAuthentication(); app.UseAuthorization(); @@ -299,9 +302,6 @@ public class DuplicatiWebserver if (!settings.DisableStaticFiles) app.UseDefaultStaticFiles(settings.WebRoot, settings.SPAPaths); - if (useCors) - app.UseCors(CorsPolicyName); - app.UseExceptionHandler(app => { app.Run(async context => From 655f5ef63575925b5d1dcf1a9230310b2c556c69 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 20 Feb 2025 11:00:28 +0100 Subject: [PATCH 215/362] Added interuption test This adds a test with a failed upload. The failed upload is done in parallel, so (at least) one of the uploads succeeeds and one upload fails. The backup then errors, but a subsequent backup continues without errors and creates a synthetic filelist. Also refactored the DeterministicErrorBackend to not use magic strings. And made it actually deterministic. --- Duplicati/UnitTest/CompactDisruptionTests.cs | 24 ++--- .../UnitTest/DeterministicErrorBackend.cs | 44 +++++--- Duplicati/UnitTest/DisruptionTests.cs | 100 ++++++++++++++++++ Duplicati/UnitTest/IssueTests.cs | 8 +- 4 files changed, 143 insertions(+), 33 deletions(-) diff --git a/Duplicati/UnitTest/CompactDisruptionTests.cs b/Duplicati/UnitTest/CompactDisruptionTests.cs index 38da492cf..8a8f9328c 100644 --- a/Duplicati/UnitTest/CompactDisruptionTests.cs +++ b/Duplicati/UnitTest/CompactDisruptionTests.cs @@ -135,13 +135,13 @@ namespace Duplicati.UnitTest target = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER; // Fail the compact after the first dblock put is completed bool firstPutCompleted = false; - DeterministicErrorBackend.ErrorGenerator = (string action, string remotename) => + DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) => { - if (firstPutCompleted && (action == "get_0" || action == "get_1")) + if (firstPutCompleted && action.IsGetOperation) { return true; } - if (action == "put_1" || action == "put_async") + if (action == DeterministicErrorBackend.BackendAction.PutAfter) { firstPutCompleted = true; } @@ -250,13 +250,13 @@ namespace Duplicati.UnitTest target = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER; // Fail the compact after the first dblock put is completed bool firstPutCompleted = false; - DeterministicErrorBackend.ErrorGenerator = (string action, string remotename) => + DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) => { - if (firstPutCompleted && (action == "get_0" || action == "get_1")) + if (firstPutCompleted && action.IsGetOperation) { return true; } - if (action == "put_1" || action == "put_async") + if (action == DeterministicErrorBackend.BackendAction.PutAfter) { firstPutCompleted = true; } @@ -382,13 +382,13 @@ namespace Duplicati.UnitTest target = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER; // Fail the compact after the first dblock put is completed bool firstPutCompleted = false; - DeterministicErrorBackend.ErrorGenerator = (string action, string remotename) => + DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) => { - if (firstPutCompleted && (action == "get_0" || action == "get_1")) + if (firstPutCompleted && action.IsGetOperation) { return true; } - if (action == "put_1" || action == "put_async") + if (action == DeterministicErrorBackend.BackendAction.PutAfter) { firstPutCompleted = true; } @@ -523,13 +523,13 @@ namespace Duplicati.UnitTest target = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER; // Fail the compact after the first dblock put is completed bool firstPutCompleted = false; - DeterministicErrorBackend.ErrorGenerator = (string action, string remotename) => + DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) => { - if (firstPutCompleted && (action == "get_0" || action == "get_1")) + if (firstPutCompleted && action.IsGetOperation) { return true; } - if (action == "put_1" || action == "put_async") + if (action == DeterministicErrorBackend.BackendAction.PutAfter) { firstPutCompleted = true; } diff --git a/Duplicati/UnitTest/DeterministicErrorBackend.cs b/Duplicati/UnitTest/DeterministicErrorBackend.cs index 34a62ad7e..8551fc904 100644 --- a/Duplicati/UnitTest/DeterministicErrorBackend.cs +++ b/Duplicati/UnitTest/DeterministicErrorBackend.cs @@ -31,14 +31,26 @@ namespace Duplicati.UnitTest { public class DeterministicErrorBackend : IBackend, IStreamingBackend { + public sealed record BackendAction(string Code) + { + public static readonly BackendAction GetBefore = new BackendAction("get_0"); + public static readonly BackendAction GetAfter = new BackendAction("get_1"); + public static readonly BackendAction PutBefore = new BackendAction("put_0"); + public static readonly BackendAction PutAfter = new BackendAction("put_1"); + public static readonly BackendAction DeleteBefore = new BackendAction("delete_0"); + public static readonly BackendAction DeleteAfter = new BackendAction("delete_1"); + + public bool IsGetOperation => this == GetBefore || this == GetAfter; + public bool IsPutOperation => this == PutBefore || this == PutAfter; + public bool IsDeleteOperation => this == DeleteBefore || this == DeleteAfter; + } + public class DeterministicErrorBackendException(string message) : Exception(message) { }; static DeterministicErrorBackend() { WrappedBackend = "file"; } - private static readonly Random random = new Random(42); - // return true to throw exception, parameters: (action, remotename) - public static Func ErrorGenerator = null; + public static Func ErrorGenerator = null; public static string WrappedBackend { get; set; } @@ -54,7 +66,7 @@ namespace Duplicati.UnitTest m_backend = (IStreamingBackend)Library.DynamicLoader.BackendLoader.GetBackend(u, options); } - private void ThrowError(string action, string remotename) + private void ThrowError(BackendAction action, string remotename) { if (ErrorGenerator != null && ErrorGenerator(action, remotename)) { @@ -64,17 +76,15 @@ namespace Duplicati.UnitTest #region IStreamingBackend implementation public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken) { - var uploadError = random.NextDouble() > 0.9; - - using (var f = new Library.Utility.ProgressReportingStream(stream, x => { if (uploadError && stream.Position > stream.Length / 2) throw new DeterministicErrorBackendException("Random upload failure"); })) - await m_backend.PutAsync(remotename, f, cancelToken).ConfigureAwait(false); - ThrowError("put_async", remotename); + ThrowError(BackendAction.PutBefore, remotename); + await m_backend.PutAsync(remotename, stream, cancelToken).ConfigureAwait(false); + ThrowError(BackendAction.PutAfter, remotename); } public async Task GetAsync(string remotename, Stream stream, CancellationToken cancellationToken) { - ThrowError("get_0", remotename); + ThrowError(BackendAction.GetBefore, remotename); await m_backend.GetAsync(remotename, stream, cancellationToken).ConfigureAwait(false); - ThrowError("get_1", remotename); + ThrowError(BackendAction.GetAfter, remotename); } #endregion @@ -85,21 +95,21 @@ namespace Duplicati.UnitTest } public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) { - ThrowError("put_0", remotename); + ThrowError(BackendAction.PutBefore, remotename); await m_backend.PutAsync(remotename, filename, cancelToken).ConfigureAwait(false); - ThrowError("put_1", remotename); + ThrowError(BackendAction.PutAfter, remotename); } public async Task GetAsync(string remotename, string filename, CancellationToken cancelToken) { - ThrowError("get_0", remotename); + ThrowError(BackendAction.GetBefore, remotename); await m_backend.GetAsync(remotename, filename, cancelToken).ConfigureAwait(false); - ThrowError("get_1", remotename); + ThrowError(BackendAction.GetAfter, remotename); } public async Task DeleteAsync(string remotename, CancellationToken cancelToken) { - ThrowError("delete_0", remotename); + ThrowError(BackendAction.DeleteBefore, remotename); await m_backend.DeleteAsync(remotename, cancelToken).ConfigureAwait(false); - ThrowError("delete_1", remotename); + ThrowError(BackendAction.DeleteAfter, remotename); } public Task TestAsync(CancellationToken cancelToken) { diff --git a/Duplicati/UnitTest/DisruptionTests.cs b/Duplicati/UnitTest/DisruptionTests.cs index c23a15124..16fac2c5d 100644 --- a/Duplicati/UnitTest/DisruptionTests.cs +++ b/Duplicati/UnitTest/DisruptionTests.cs @@ -752,5 +752,105 @@ namespace Duplicati.UnitTest } } } + + [Test] + [Category("Disruption")] + public void TestFailedUploadWithNoRetries() + { + var testopts = TestOptions; + testopts["number-of-retries"] = "0"; + testopts["dblock-size"] = "10mb"; + + // Make a base backup + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER }); + Assert.AreEqual(0, backupResults.Errors.Count()); + Assert.AreEqual(0, backupResults.Warnings.Count()); + } + + // Make a new backup that fails uploading a dblock file + ModifySourceFiles(); + + // Deterministic error backend + Library.DynamicLoader.BackendLoader.AddBackend(new DeterministicErrorBackend()); + var failtarget = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER; + + var hasFailed = false; + var secondUploadStarted = false; + var secondUploadCompleted = false; + + // Fail the compact after the first dblock put is completed + DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) => + { + if (action.IsGetOperation) + { + return true; + } + + if (!hasFailed && action == DeterministicErrorBackend.BackendAction.PutBefore) + { + // We only fail one upload, but there are no retries + hasFailed = true; + + // Make sure we can start a second upload + Thread.Sleep(1000); + return true; + } + + if (action == DeterministicErrorBackend.BackendAction.PutBefore) + secondUploadStarted = true; + if (action == DeterministicErrorBackend.BackendAction.PutAfter) + secondUploadCompleted = true; + + return false; + }; + + using (var c = new Library.Main.Controller(failtarget, testopts, null)) + Assert.Throws(() => c.Backup(new string[] { DATAFOLDER })); + + Assert.That(secondUploadStarted, Is.True, "Second upload was not started"); + Assert.That(secondUploadCompleted, Is.True, "Second upload was not started"); + + // Create a regular backup + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER }); + Assert.AreEqual(0, backupResults.Errors.Count()); + Assert.AreEqual(0, backupResults.Warnings.Count()); + } + + // Verify that all is in order + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null)) + { + var r = c.Test(long.MaxValue); + Assert.AreEqual(0, r.Errors.Count()); + Assert.AreEqual(0, r.Warnings.Count()); + Assert.IsFalse(r.Verifications.Any(p => p.Value.Any())); + } + + // Test that we can recreate + var recreatedDatabaseFile = Path.Combine(BASEFOLDER, "recreated-database.sqlite"); + if (File.Exists(recreatedDatabaseFile)) + File.Delete(recreatedDatabaseFile); + + testopts["dbpath"] = recreatedDatabaseFile; + + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IRepairResults repairResults = c.Repair(); + Assert.AreEqual(0, repairResults.Errors.Count()); + Assert.AreEqual(0, repairResults.Warnings.Count()); + } + + // Check that we have 3 versions + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IListResults listResults = c.List(); + Assert.AreEqual(0, listResults.Errors.Count()); + Assert.AreEqual(0, listResults.Warnings.Count()); + Assert.AreEqual(3, listResults.Filesets.Count()); + } + } } } \ No newline at end of file diff --git a/Duplicati/UnitTest/IssueTests.cs b/Duplicati/UnitTest/IssueTests.cs index 62ddc787b..a0302b149 100644 --- a/Duplicati/UnitTest/IssueTests.cs +++ b/Duplicati/UnitTest/IssueTests.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2025, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a @@ -68,10 +68,10 @@ namespace Duplicati.UnitTest // Fail after index file put target = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER; string interruptedName = ""; - DeterministicErrorBackend.ErrorGenerator = (string action, string remotename) => + DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) => { // Fail dindex upload - if (action.StartsWith("put") && remotename.Contains("dindex")) + if (action == DeterministicErrorBackend.BackendAction.PutAfter && remotename.Contains("dindex")) { interruptedName = remotename; return true; @@ -460,7 +460,7 @@ namespace Duplicati.UnitTest bool failed = false; long accessCounter = 0; long errorIdx = 0; - DeterministicErrorBackend.ErrorGenerator = (string action, string remotename) => + DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) => { ++accessCounter; if (accessCounter >= errorIdx) From c164181ec2ca0ba79c3c1688aa86f9f6596c0853 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 21 Feb 2025 10:07:09 +0100 Subject: [PATCH 216/362] BackendManager.GetOperation's were unintentionally still synchronized. --- Duplicati/Library/Main/Backend/BackendManager.Handler.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Backend/BackendManager.Handler.cs b/Duplicati/Library/Main/Backend/BackendManager.Handler.cs index 1115d6de7..84619dea1 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.Handler.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.Handler.cs @@ -281,7 +281,7 @@ partial class BackendManager putOp.SetComplete(true); activeUploads.Add(ExecuteWithRetry(putOp, tcs.Token)); } - else if (op is GetOperation getOp && !getOp.WaitForComplete) + else if (op is GetOperation getOp) { await EnsureAtMostNActiveDownloads(maxParallelDownloads).ConfigureAwait(false); From 7612b5cce3d25b57240f782cef7f1a9104adc6e0 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 21 Feb 2025 10:07:41 +0100 Subject: [PATCH 217/362] Ensured proper synchronization of operations in BackendManager.Handler --- Duplicati/Library/Main/Backend/BackendManager.Handler.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Backend/BackendManager.Handler.cs b/Duplicati/Library/Main/Backend/BackendManager.Handler.cs index 84619dea1..1819057d9 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.Handler.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.Handler.cs @@ -275,6 +275,8 @@ partial class BackendManager // Allow PUT operations to be queued, if requested if (op is PutOperation putOp && !putOp.WaitForComplete) { + // Wait for any active downloads to complete before starting an upload + await EnsureAtMostNActiveDownloads(1).ConfigureAwait(false); await EnsureAtMostNActiveUploads(maxParallelUploads).ConfigureAwait(false); // Operation is accepted into queue, so we can signal completion @@ -283,6 +285,8 @@ partial class BackendManager } else if (op is GetOperation getOp) { + // Wait for any active uploads to complete before starting a download + await EnsureAtMostNActiveUploads(1).ConfigureAwait(false); await EnsureAtMostNActiveDownloads(maxParallelDownloads).ConfigureAwait(false); // Operation is accepted into queue, so we can signal completion @@ -290,8 +294,10 @@ partial class BackendManager } else { - // Wait for all of the active uploads to complete + // Wait for all of the active uploads and downloads to complete await EnsureAtMostNActiveUploads(1).ConfigureAwait(false); + await EnsureAtMostNActiveDownloads(1).ConfigureAwait(false); + // Execute the operation await ExecuteWithRetry(op, tcs.Token).ConfigureAwait(false); } From 98e572714daed2d25cf8d0ca2adb3577b55e3e29 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 21 Feb 2025 10:08:21 +0100 Subject: [PATCH 218/362] Added a comment about downloaders in the teardown of BackendManager.Handler --- Duplicati/Library/Main/Backend/BackendManager.Handler.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Backend/BackendManager.Handler.cs b/Duplicati/Library/Main/Backend/BackendManager.Handler.cs index 1819057d9..62c5fd4eb 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.Handler.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.Handler.cs @@ -313,7 +313,7 @@ partial class BackendManager } finally { - // Terminate any active uploads + // Terminate any active uploads and downloads. Exceptions thrown by the downloads should be captured by the callers. tcs.Cancel(); if (activeUploads.Count > 0) From 5a4fcaa6d77b30d29b4dc78c128d86ece686d546 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 21 Feb 2025 11:51:45 +0100 Subject: [PATCH 219/362] Added tool to work with remote sources --- Duplicati.sln | 14 +++ .../SourceTool/CommandExtensions.cs | 42 +++++++ .../SourceTool/Commands/Download.cs | 110 ++++++++++++++++++ .../CommandLine/SourceTool/Commands/List.cs | 66 +++++++++++ .../Duplicati.CommandLine.SourceTool.csproj | 25 ++++ .../CommandLine/SourceTool/GetProvider.cs | 39 +++++++ .../CommandLine/SourceTool/ListEnumerator.cs | 60 ++++++++++ Duplicati/CommandLine/SourceTool/Program.cs | 68 +++++++++++ .../Library/AutoUpdater/PackageHelper.cs | 7 +- .../DynamicLoader/SourceProviderLoader.cs | 15 ++- .../Library/Main/Operation/BackupHandler.cs | 23 +--- .../Builtin/BackendSourceProvider.cs | 13 ++- .../Duplicati.CommandLine.SourceTool.csproj | 22 ++++ .../Program.cs | 31 +++++ ReleaseBuilder/Build/Command.cs | 1 + 15 files changed, 513 insertions(+), 23 deletions(-) create mode 100644 Duplicati/CommandLine/SourceTool/CommandExtensions.cs create mode 100644 Duplicati/CommandLine/SourceTool/Commands/Download.cs create mode 100644 Duplicati/CommandLine/SourceTool/Commands/List.cs create mode 100644 Duplicati/CommandLine/SourceTool/Duplicati.CommandLine.SourceTool.csproj create mode 100644 Duplicati/CommandLine/SourceTool/GetProvider.cs create mode 100644 Duplicati/CommandLine/SourceTool/ListEnumerator.cs create mode 100644 Duplicati/CommandLine/SourceTool/Program.cs create mode 100644 Executables/net8/Duplicati.CommandLine.SourceTool/Duplicati.CommandLine.SourceTool.csproj create mode 100644 Executables/net8/Duplicati.CommandLine.SourceTool/Program.cs diff --git a/Duplicati.sln b/Duplicati.sln index 36212bbde..e9f8d2405 100644 --- a/Duplicati.sln +++ b/Duplicati.sln @@ -178,6 +178,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RemoteSynchronization", "To EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.CommandLine.SyncTool", "Executables\net8\Duplicati.CommandLine.SyncTool\Duplicati.CommandLine.SyncTool.csproj", "{48D32674-0FE9-4407-B102-4AE46D93595F}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.CommandLine.SourceTool", "Duplicati\CommandLine\SourceTool\Duplicati.CommandLine.SourceTool.csproj", "{2873BCE2-2EA6-419D-BFFA-693FDB34926E}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.CommandLine.SourceTool", "Executables\net8\Duplicati.CommandLine.SourceTool\Duplicati.CommandLine.SourceTool.csproj", "{4003BF90-6681-4155-9D06-639ED23FE3AB}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -500,6 +504,14 @@ Global {48D32674-0FE9-4407-B102-4AE46D93595F}.Debug|Any CPU.Build.0 = Debug|Any CPU {48D32674-0FE9-4407-B102-4AE46D93595F}.Release|Any CPU.ActiveCfg = Release|Any CPU {48D32674-0FE9-4407-B102-4AE46D93595F}.Release|Any CPU.Build.0 = Release|Any CPU + {2873BCE2-2EA6-419D-BFFA-693FDB34926E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2873BCE2-2EA6-419D-BFFA-693FDB34926E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2873BCE2-2EA6-419D-BFFA-693FDB34926E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2873BCE2-2EA6-419D-BFFA-693FDB34926E}.Release|Any CPU.Build.0 = Release|Any CPU + {4003BF90-6681-4155-9D06-639ED23FE3AB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4003BF90-6681-4155-9D06-639ED23FE3AB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4003BF90-6681-4155-9D06-639ED23FE3AB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4003BF90-6681-4155-9D06-639ED23FE3AB}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -563,6 +575,8 @@ Global {6D9742A7-6F32-4571-8276-79716168B525} = {424CEF73-4984-430E-9C8B-E61CC0F22074} {D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D} = {15388C37-9218-4818-972E-738EEA8F1602} {48D32674-0FE9-4407-B102-4AE46D93595F} = {6B46F6B1-1898-49B8-ADA7-5CAF68EB77E3} + {2873BCE2-2EA6-419D-BFFA-693FDB34926E} = {D19A38DD-68F1-4EF5-BF5F-8966CE0D9A5B} + {4003BF90-6681-4155-9D06-639ED23FE3AB} = {6B46F6B1-1898-49B8-ADA7-5CAF68EB77E3} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {8B40BAFE-D862-4397-9495-8F5EAF5CE80C} diff --git a/Duplicati/CommandLine/SourceTool/CommandExtensions.cs b/Duplicati/CommandLine/SourceTool/CommandExtensions.cs new file mode 100644 index 000000000..019672143 --- /dev/null +++ b/Duplicati/CommandLine/SourceTool/CommandExtensions.cs @@ -0,0 +1,42 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +using System.CommandLine; +using System.CommandLine.Invocation; + +namespace Duplicati.CommandLine.SourceTool; + +/// +/// Extensions for . +/// +public static class CommandExtensions +{ + /// + /// Adds the missing WithHandler method to . + /// + /// The command to add the handler to. + /// The handler to add. + /// The command with the handler added. + public static Command WithHandler(this Command command, ICommandHandler handler) + { + command.Handler = handler; + return command; + } +} diff --git a/Duplicati/CommandLine/SourceTool/Commands/Download.cs b/Duplicati/CommandLine/SourceTool/Commands/Download.cs new file mode 100644 index 000000000..7f2f1d046 --- /dev/null +++ b/Duplicati/CommandLine/SourceTool/Commands/Download.cs @@ -0,0 +1,110 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +using System.CommandLine; +using System.CommandLine.NamingConventionBinder; + +namespace Duplicati.CommandLine.SourceTool.Commands; + +/// +/// The download command +/// +public static class Download +{ + /// + /// Creates the download command + /// + /// The download command + public static Command Create() => + new Command("download", "Downloads all files from the remote") + { + new Argument("url", "The source URL") { + Arity = ArgumentArity.ExactlyOne + }, + new Option("--destination", description: "The destination folder", getDefaultValue: () => new DirectoryInfo(Directory.GetCurrentDirectory())), + new Option("--max-depth", description: "The maximum depth to visit", getDefaultValue: () => 0), + new Option("--max-size", description: "The maximum filesize to download", getDefaultValue: () => 0), + new Option("--overwrite", description: "Overwrite existing files", getDefaultValue: () => false), + } + .WithHandler(CommandHandler.Create(async (url, destination, maxdepth, maxsize, overwrite) => + { + var token = new CancellationTokenSource().Token; + + using var source = Common.GetProvider(url); + + string localPath(string path) + { + var relpath = path.Substring(source.PathKey.Length).TrimStart('/'); + return Path.Combine(destination.FullName, relpath); + } + + var started = DateTime.UtcNow; + var fileCount = 0L; + var folderCount = 0L; + var totalSize = 0L; + var downloadCount = 0L; + var downloadSize = 0L; + + await Common.Visit(source, maxdepth, async (entry, level) => + { + if (entry.IsMetaEntry) + return true; + + var path = localPath(entry.Path); + if (entry.IsFolder) + { + folderCount++; + if (!Directory.Exists(path)) + Directory.CreateDirectory(path); + } + else + { + fileCount++; + totalSize += entry.Size; + + if (entry.Size <= maxsize || maxsize <= 0) + { + var folder = Path.GetDirectoryName(path); + if (!string.IsNullOrWhiteSpace(folder) && !Directory.Exists(folder)) + Directory.CreateDirectory(folder); + + if (!overwrite && File.Exists(path)) + { + Console.WriteLine($"Skipping {entry.Path} as it already exists"); + } + else + { + downloadCount++; + downloadSize += entry.Size; + Console.WriteLine($"Downloading {entry.Path} to {path} ({Library.Utility.Utility.FormatSizeString(entry.Size)})"); + await using var stream = await entry.OpenRead(token); + await using var dest = File.OpenWrite(path); + await stream.CopyToAsync(dest, token); + } + } + } + + return true; + }, token); + + Console.WriteLine($"Found {fileCount} files and {folderCount} folders with a total size of {Library.Utility.Utility.FormatSizeString(totalSize)}"); + Console.WriteLine($"Downloaded {downloadCount} files with a total size of {Library.Utility.Utility.FormatSizeString(downloadSize)} in {DateTime.UtcNow - started}"); + })); +} diff --git a/Duplicati/CommandLine/SourceTool/Commands/List.cs b/Duplicati/CommandLine/SourceTool/Commands/List.cs new file mode 100644 index 000000000..a2169b99e --- /dev/null +++ b/Duplicati/CommandLine/SourceTool/Commands/List.cs @@ -0,0 +1,66 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +using System.CommandLine; +using System.CommandLine.NamingConventionBinder; + +namespace Duplicati.CommandLine.SourceTool.Commands; + +/// +/// The list command +/// +public static class List +{ + /// + /// Creates the list command + /// + /// The command + public static Command Create() => + new Command("list", "Lists all paths on the remote") + { + new Argument("url", "The source URL") { + Arity = ArgumentArity.ExactlyOne + }, + new Option("--max-depth", description: "The maximum depth to list", getDefaultValue: () => 0), + } + .WithHandler(CommandHandler.Create(async (url, maxdepth) => + { + var token = new CancellationTokenSource().Token; + var folders = 0L; + var files = 0L; + + using var source = Common.GetProvider(url); + await Common.Visit(source, maxdepth, (entry, level) => + { + if (entry.IsFolder) + folders++; + else + files++; + if (entry.IsFolder && level > 0) + level--; + Console.WriteLine($"{new string(' ', (level + 1) * 2)}{entry.Path}"); + return Task.FromResult(true); + }, token); + + Console.WriteLine($"Found {folders} folders and {files} files"); + }) + ); + +} diff --git a/Duplicati/CommandLine/SourceTool/Duplicati.CommandLine.SourceTool.csproj b/Duplicati/CommandLine/SourceTool/Duplicati.CommandLine.SourceTool.csproj new file mode 100644 index 000000000..99c103755 --- /dev/null +++ b/Duplicati/CommandLine/SourceTool/Duplicati.CommandLine.SourceTool.csproj @@ -0,0 +1,25 @@ + + + + net8.0 + enable + enable + Duplicati.CommandLine.SourceTool.Implementation + Copyright © 2025 Team Duplicati, MIT license + Duplicati.CommandLine.Sourcetool + + + + + + + + + + + + + + + + diff --git a/Duplicati/CommandLine/SourceTool/GetProvider.cs b/Duplicati/CommandLine/SourceTool/GetProvider.cs new file mode 100644 index 000000000..62888b184 --- /dev/null +++ b/Duplicati/CommandLine/SourceTool/GetProvider.cs @@ -0,0 +1,39 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +using Duplicati.Library.DynamicLoader; +using Duplicati.Library.Interface; + +namespace Duplicati.CommandLine.SourceTool; + +/// +/// Common methods for the source tool +/// +public static partial class Common +{ + /// + /// Gets the source provider for the given URL + /// + /// The URL to get the provider for + /// The source provider + public static ISourceProvider GetProvider(string url) => + SourceProviderLoader.GetSourceProvider(url, []) + ?? throw new UserInformationException(string.Format("The source \"{0}\" is not supported", url), "SourceNotSupported"); +} \ No newline at end of file diff --git a/Duplicati/CommandLine/SourceTool/ListEnumerator.cs b/Duplicati/CommandLine/SourceTool/ListEnumerator.cs new file mode 100644 index 000000000..e8899c8b6 --- /dev/null +++ b/Duplicati/CommandLine/SourceTool/ListEnumerator.cs @@ -0,0 +1,60 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +using Duplicati.Library.Interface; + +namespace Duplicati.CommandLine.SourceTool; + +/// +/// Common methods for the source tool +/// +public static partial class Common +{ + /// + /// Visits all entries in the source provider + /// + /// The source provider to visit + /// The maximum depth to visit + /// The visitor function + /// The cancellation token + /// An awaitable task + public static async Task Visit(ISourceProvider source, int maxdepth, Func> visitor, CancellationToken token) + { + var visit = new Stack<(ISourceFileEntry Entry, int Level)>(); + await foreach (var item in source.Enumerate(token)) + visit.Push((item, 0)); + + while (visit.Count() != 0) + { + var item = visit.Pop(); + var process = await visitor(item.Entry, item.Level); + if (item.Entry.IsFolder && process && (item.Level < maxdepth || maxdepth <= 0)) + { + await foreach (var subitem in item.Entry.Enumerate(token)) + { + if (subitem.IsFolder) + visit.Push((subitem, item.Level + 1)); + else + await visitor(subitem, item.Level); + } + } + } + } +} diff --git a/Duplicati/CommandLine/SourceTool/Program.cs b/Duplicati/CommandLine/SourceTool/Program.cs new file mode 100644 index 000000000..c8aea4225 --- /dev/null +++ b/Duplicati/CommandLine/SourceTool/Program.cs @@ -0,0 +1,68 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +using System.CommandLine; +using System.CommandLine.Builder; +using System.CommandLine.Parsing; +using Duplicati.Library.Interface; +using Duplicati.Library.Utility; + +namespace Duplicati.CommandLine.SourceTool; + +/// +/// The entry point of the application +/// +public static class Program +{ + /// + /// Executes the program + /// + /// The arguments + /// The return code + public static Task Main(string[] args) + { + Library.AutoUpdater.PreloadSettingsLoader.ConfigurePreloadSettings(ref args, Library.AutoUpdater.PackageHelper.NamedExecutable.ServerUtil); + + var rootCmd = new RootCommand("Source CLI tool for Duplicati") + { + Commands.List.Create(), + Commands.Download.Create(), + }; + + return new CommandLineBuilder(rootCmd) + .UseDefaults() + .UseExceptionHandler((ex, context) => + { + if (ex is UserInformationException uie) + { + Console.WriteLine(uie.Message); + context.ExitCode = 2; + } + else + { + Console.WriteLine(ex.ToString()); + context.ExitCode = 1; + } + }) + .UseAdditionalHelpAliases() + .Build() + .InvokeAsync(args); + } +} diff --git a/Duplicati/Library/AutoUpdater/PackageHelper.cs b/Duplicati/Library/AutoUpdater/PackageHelper.cs index 314facb04..4ceefc41e 100644 --- a/Duplicati/Library/AutoUpdater/PackageHelper.cs +++ b/Duplicati/Library/AutoUpdater/PackageHelper.cs @@ -92,7 +92,11 @@ namespace Duplicati.Library.AutoUpdater /// /// The sync tool /// - SyncTool + SyncTool, + /// + /// The source tool + /// + SourceTool } @@ -120,6 +124,7 @@ namespace Duplicati.Library.AutoUpdater NamedExecutable.Agent => OperatingSystem.IsWindows() ? "Duplicati.Agent.exe" : "duplicati-agent", NamedExecutable.SecretTool => OperatingSystem.IsWindows() ? "Duplicati.CommandLine.SecretTool.exe" : "duplicati-secret-tool", NamedExecutable.SyncTool => OperatingSystem.IsWindows() ? "Duplicati.CommandLine.SyncTool.exe" : "duplicati-sync-tool", + NamedExecutable.SourceTool => OperatingSystem.IsWindows() ? "Duplicati.CommandLine.Sourcetool.exe" : "duplicati-source-tool", _ => throw new ArgumentException($"Named executable not known: {exe}", nameof(exe)) }; diff --git a/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs b/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs index 23920c594..f73022611 100644 --- a/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs +++ b/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs @@ -25,6 +25,7 @@ using Duplicati.Library.Interface; using System.Linq; using System.Runtime.ExceptionServices; using Duplicati.Library.SourceProviders; +using Duplicati.Library.SourceProvider; namespace Duplicati.Library.DynamicLoader { @@ -164,7 +165,19 @@ namespace Duplicati.Library.DynamicLoader /// The instanciated SourceProvider or null if the url is not supported public static ISourceProviderModule GetSourceProvider(string url, Dictionary options) { - return _SourceProviderLoader.GetSourceProvider(url, options); + // Source providers are preferred over backends + var provider = _SourceProviderLoader.GetSourceProvider(url, options); + if (provider != null) + return provider; + + // See if there is a backend that can also be a source + var backend = BackendLoader.GetBackend(url, options); + if (backend is IFolderEnabledBackend folderBackend) + return new BackendSourceProvider(folderBackend); + + backend?.Dispose(); + return null; + } #endregion diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 059bbc8df..8d09a6f27 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -138,27 +138,10 @@ namespace Duplicati.Library.Main.Operation results.Add(new LocalFileSource(SnapshotUtility.CreateSnapshot(entry, options.RawOptions, options.SymlinkPolicy == Options.SymlinkStrategy.Follow))); else { + // Assume it is a remote source foreach (var url in entry) - { - // Source providers are preferred over backends - var provider = SourceProviderLoader.GetSourceProvider(url, options.RawOptions); - if (provider != null) - { - results.Add(provider); - continue; - } - - // See if there is a backend that can also be a source - var backend = BackendLoader.GetBackend(url, options.RawOptions); - if (backend is IFolderEnabledBackend folderBackend) - { - results.Add(new BackendSourceProvider(folderBackend)); - continue; - } - - backend?.Dispose(); - throw new UserInformationException(string.Format("The source \"{0}\" is not supported", url), "SourceNotSupported"); - } + results.Add(SourceProviderLoader.GetSourceProvider(url, options.RawOptions) + ?? throw new UserInformationException(string.Format("The source \"{0}\" is not supported", url), "SourceNotSupported")); } } } diff --git a/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs index 6411cc0a7..618de97f7 100644 --- a/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs +++ b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs @@ -19,6 +19,8 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +using System.Runtime.CompilerServices; +using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; namespace Duplicati.Library.SourceProvider; @@ -27,7 +29,7 @@ namespace Duplicati.Library.SourceProvider; /// A source provider that wraps a backend /// /// The backend to wrap -public class BackendSourceProvider(IFolderEnabledBackend backend) : ISourceProvider +public class BackendSourceProvider(IFolderEnabledBackend backend) : ISourceProvider, ISourceProviderModule { /// /// The wrapped backend @@ -39,6 +41,15 @@ public class BackendSourceProvider(IFolderEnabledBackend backend) : ISourceProvi /// public string PathKey => backend.PathKey; + /// + public string Key => backend.ProtocolKey; + + /// + public string DisplayName => backend.DisplayName; + + /// + public IList SupportedCommands => backend.SupportedCommands; + /// public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) => new[] { new BackendSourceFileEntry(this, PathKey, true, true, new DateTime(0), new DateTime(0), 0) }.ToAsyncEnumerable(); diff --git a/Executables/net8/Duplicati.CommandLine.SourceTool/Duplicati.CommandLine.SourceTool.csproj b/Executables/net8/Duplicati.CommandLine.SourceTool/Duplicati.CommandLine.SourceTool.csproj new file mode 100644 index 000000000..038a3b98d --- /dev/null +++ b/Executables/net8/Duplicati.CommandLine.SourceTool/Duplicati.CommandLine.SourceTool.csproj @@ -0,0 +1,22 @@ + + + + Exe + net8.0 + SyncTool for Duplicati + Copyright © 2025 Team Duplicati, MIT license + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + diff --git a/Executables/net8/Duplicati.CommandLine.SourceTool/Program.cs b/Executables/net8/Duplicati.CommandLine.SourceTool/Program.cs new file mode 100644 index 000000000..490864a25 --- /dev/null +++ b/Executables/net8/Duplicati.CommandLine.SourceTool/Program.cs @@ -0,0 +1,31 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +using Duplicati.Library.Crashlog; + +namespace Duplicati.CommandLine.SourceTool.Net8 +{ + // Wrapper class to keep code independent + public static class Program + { + public static int Main(string[] args) + => CrashlogHelper.WrapWithCrashLog(() => SourceTool.Program.Main(args).ConfigureAwait(false).GetAwaiter().GetResult()); + } +} \ No newline at end of file diff --git a/ReleaseBuilder/Build/Command.cs b/ReleaseBuilder/Build/Command.cs index 46ea9f935..94cd6a8c3 100644 --- a/ReleaseBuilder/Build/Command.cs +++ b/ReleaseBuilder/Build/Command.cs @@ -87,6 +87,7 @@ public static partial class Command { "Duplicati.CommandLine.ServerUtil", "duplicati-server-util" }, { "Duplicati.CommandLine.SecretTool", "duplicati-secret-tool" }, { "Duplicati.CommandLine.SyncTool", "duplicati-sync-tool" }, + { "Duplicati.CommandLine.SourceTool", "duplicati-source-tool" }, { "Duplicati.Service", "duplicati-service" }, { "Duplicati.Agent", "duplicati-agent" }, { "Duplicati.CommandLine", "duplicati-cli" }, From 1716d09b5175728d478cc2687867bf2c4dfd8e37 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 21 Feb 2025 13:20:52 +0100 Subject: [PATCH 220/362] Fixed source providers for S3, CIFS, and SSH. Made the SMBShareConnection slightly more robust. Introduced v2 List API to S3 backend, currently it retains v1 as the default. Changed S3 backends to not recurse, potentially making smaller listings. This option can be disabled if it breaks things. Added support for relative paths in the SSH backend. --- Duplicati/Library/Backend/CIFS/CIFS.cs | 20 +- .../Backend/CIFS/SMBShareConnection.cs | 7 +- .../Backend/Idrivee2/Idrivee2Backend.cs | 31 +-- Duplicati/Library/Backend/S3/IS3Client.cs | 2 +- Duplicati/Library/Backend/S3/S3AwsClient.cs | 85 +++++++-- Duplicati/Library/Backend/S3/S3Backend.cs | 39 ++-- Duplicati/Library/Backend/S3/S3MinioClient.cs | 7 +- Duplicati/Library/Backend/S3/Strings.cs | 42 +++-- .../Library/Backend/SSHv2/SSHv2Backend.cs | 65 +++++-- Duplicati/Library/Backend/SSHv2/Strings.cs | 40 ++-- Duplicati/Library/Common/IO/Util.cs | 7 + Duplicati/Library/Main/Controller.cs | 14 ++ .../Main/Database/LocalBackupDatabase.cs | 5 +- .../Library/Main/Operation/BackupHandler.cs | 2 +- .../Builtin/BackendSourceFileEntry.cs | 109 +++++++++-- .../Builtin/BackendSourceProvider.cs | 38 +++- ...cati.Library.SourceProvider.Builtin.csproj | 4 + Duplicati/Library/Utility/MapStream.cs | 176 ------------------ .../Library/Utility/OverrideableStream.cs | 136 +++++++------- Duplicati/Library/Utility/TempFileStream.cs | 10 + 20 files changed, 440 insertions(+), 399 deletions(-) delete mode 100644 Duplicati/Library/Utility/MapStream.cs diff --git a/Duplicati/Library/Backend/CIFS/CIFS.cs b/Duplicati/Library/Backend/CIFS/CIFS.cs index 549721edf..34b05ec0a 100644 --- a/Duplicati/Library/Backend/CIFS/CIFS.cs +++ b/Duplicati/Library/Backend/CIFS/CIFS.cs @@ -32,6 +32,7 @@ using Duplicati.Library.Utility; using SMBLibrary; using Duplicati.Library.SourceProvider; using System.Runtime.CompilerServices; +using Duplicati.Library.Common.IO; namespace Duplicati.Library.Backend; @@ -190,7 +191,11 @@ public class CIFSBackend : IStreamingBackend, IFolderEnabledBackend writeBufferSize ); - _sourcePathKey = $"@{ProtocolKey}://{_connectionParameters.ServerName}/~{_connectionParameters.AuthUser}/{_connectionParameters.Path}/"; + _sourcePathKey = $"{Util.RemotePathPrefix}{ProtocolKey}://{_connectionParameters.ServerName}/{_connectionParameters.ShareName}/"; + if (!string.IsNullOrEmpty(_connectionParameters.AuthUser)) + _sourcePathKey += $"~{_connectionParameters.AuthUser}/"; + if (!string.IsNullOrEmpty(_connectionParameters.Path)) + _sourcePathKey += $"{_connectionParameters.Path}/"; _sourceProvider = new BackendSourceProvider(this); } @@ -360,15 +365,16 @@ public class CIFSBackend : IStreamingBackend, IFolderEnabledBackend public async IAsyncEnumerable ListAsync(string path, [EnumeratorCancellation] CancellationToken cancellationToken) { var sourcePath = _connectionParameters.Path; - if (!sourcePath.EndsWith('\\')) - sourcePath += '\\'; + if (!string.IsNullOrWhiteSpace(sourcePath) && !sourcePath.EndsWith('\\')) + sourcePath += '/'; - var list = (await GetConnection().ListAsync(sourcePath + path, cancellationToken).ConfigureAwait(false)) + foreach (var v in (await GetConnection().ListAsync(sourcePath + path, cancellationToken).ConfigureAwait(false)) + .Where(x => x.Name != "." && x.Name != "..") .Select(f => BackendSourceFileEntry.FromFileEntry(_sourceProvider, path, f)) - .Cast(); - - foreach (var v in list) + .Cast()) + { yield return v; + } } /// diff --git a/Duplicati/Library/Backend/CIFS/SMBShareConnection.cs b/Duplicati/Library/Backend/CIFS/SMBShareConnection.cs index 710e36ea2..47e23b742 100644 --- a/Duplicati/Library/Backend/CIFS/SMBShareConnection.cs +++ b/Duplicati/Library/Backend/CIFS/SMBShareConnection.cs @@ -79,7 +79,7 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable if (!_smb2Client.Connect(connectionParameters.ServerName, connectionParameters.TransportType)) throw new UserInformationException($"{LC.L("Failed to connect to server")} {connectionParameters.ServerName}", "ConnectionError"); - var status = _smb2Client.Login(connectionParameters.AuthDomain, connectionParameters.AuthUser, connectionParameters.AuthPassword); + var status = _smb2Client.Login(connectionParameters.AuthDomain ?? "", connectionParameters.AuthUser ?? "", connectionParameters.AuthPassword ?? ""); if (status != NTStatus.STATUS_SUCCESS) throw new UserInformationException($"{LC.L("Failed to authenticate to server")} {connectionParameters.ServerName} with status {status}", "ConnectionError"); @@ -207,7 +207,6 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable { try { - var status = _smbFileStore.CreateFile( out directoryHandle, out fileStatus, @@ -223,7 +222,7 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable if (status == NTStatus.STATUS_OBJECT_PATH_NOT_FOUND || status == NTStatus.STATUS_OBJECT_NAME_NOT_FOUND) throw new FolderMissingException(); else - throw new UserInformationException($"{LC.L("Failed to open directory")} {NormalizeSlashes(_connectionParameters.Path)} with status {status.ToString()}", "DirectoryOpenError"); + throw new UserInformationException($"{LC.L("Failed to open directory")} {NormalizeSlashes(path)} with status {status}", "DirectoryOpenError"); List fileList; status = _smbFileStore.QueryDirectory( @@ -233,7 +232,7 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable FileInformationClass.FileDirectoryInformation); if (status != NTStatus.STATUS_NO_MORE_FILES) - throw new UserInformationException($"{LC.L("Failed to query directory contents")} with status {status.ToString()}", "DirectoryQueryError"); + throw new UserInformationException($"{LC.L("Failed to query directory contents")} with status {status}", "DirectoryQueryError"); return [ diff --git a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs index a3c5d47be..2d5afdbab 100644 --- a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs +++ b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs @@ -90,7 +90,9 @@ namespace Duplicati.Library.Backend m_s3Client = new S3AwsClient(accessKeyId, accessKeySecret, null, host, null, true, false, options); - m_sourcePathKey = $"@{ProtocolKey}://{host}/{m_bucket}/{m_prefix}/"; + m_sourcePathKey = $"{Util.RemotePathPrefix}{ProtocolKey}://{host}/{m_bucket}/"; + if (!string.IsNullOrWhiteSpace(m_prefix)) + m_sourcePathKey += m_prefix; m_sourceProvider = new BackendSourceProvider(this); } @@ -142,7 +144,7 @@ namespace Duplicati.Library.Backend public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { - await foreach (IFileEntry file in Connection.ListBucketAsync(m_bucket, m_prefix, cancelToken).ConfigureAwait(false)) + await foreach (IFileEntry file in Connection.ListBucketAsync(m_bucket, m_prefix, false, cancelToken).ConfigureAwait(false)) { ((FileEntry)file).Name = file.Name.Substring(m_prefix.Length); if (file.Name.StartsWith("/", StringComparison.Ordinal) && !m_prefix.StartsWith("/", StringComparison.Ordinal)) @@ -249,30 +251,9 @@ namespace Duplicati.Library.Backend public string PathKey => m_sourcePathKey; /// - public async IAsyncEnumerable ListAsync(string path, [EnumeratorCancellation] CancellationToken cancellationToken) - { - var prefixPath = m_prefix; - if (!prefixPath.EndsWith("/", StringComparison.Ordinal)) - prefixPath += "/"; + public IAsyncEnumerable ListAsync(string path, CancellationToken cancellationToken) + => m_sourceProvider.ListFromFileEntryAsync(m_prefix, path, (filter, token) => m_s3Client.ListBucketAsync(m_bucket, filter, false, token), cancellationToken); - var filterPath = prefixPath + path; - if (!filterPath.EndsWith("/", StringComparison.Ordinal)) - filterPath += "/"; - - await foreach (var f in m_s3Client.ListBucketAsync(m_bucket, filterPath, cancellationToken).ConfigureAwait(false)) - { - if (!f.Name.StartsWith(prefixPath, StringComparison.Ordinal)) - continue; - - ((FileEntry)f).Name = f.Name.Substring(prefixPath.Length); - - // Skip sub-folder items - if (f.Name.Contains("/")) - continue; - - yield return BackendSourceFileEntry.FromFileEntry(m_sourceProvider, path, f); - } - } /// public Task GetEntryAsync(string path, CancellationToken cancellationToken) diff --git a/Duplicati/Library/Backend/S3/IS3Client.cs b/Duplicati/Library/Backend/S3/IS3Client.cs index 6d32833ee..3a0cb65b9 100644 --- a/Duplicati/Library/Backend/S3/IS3Client.cs +++ b/Duplicati/Library/Backend/S3/IS3Client.cs @@ -28,7 +28,7 @@ namespace Duplicati.Library.Backend { public interface IS3Client : IDisposable { - IAsyncEnumerable ListBucketAsync(string bucketName, string prefix, CancellationToken cancellationToken); + IAsyncEnumerable ListBucketAsync(string bucketName, string prefix, bool recursive, CancellationToken cancellationToken); Task AddBucketAsync(string bucketName, CancellationToken cancelToken); diff --git a/Duplicati/Library/Backend/S3/S3AwsClient.cs b/Duplicati/Library/Backend/S3/S3AwsClient.cs index 94e578182..5cc424666 100644 --- a/Duplicati/Library/Backend/S3/S3AwsClient.cs +++ b/Duplicati/Library/Backend/S3/S3AwsClient.cs @@ -49,6 +49,7 @@ namespace Duplicati.Library.Backend private readonly bool m_useChunkEncoding; private readonly string m_dnsHost; + private readonly bool m_useV2ListApi; public S3AwsClient(string awsID, string awsKey, string locationConstraint, string servername, string storageClass, bool useSSL, bool disableChunkEncoding, Dictionary options) @@ -61,6 +62,7 @@ namespace Duplicati.Library.Backend m_client = new AmazonS3Client(awsID, awsKey, cfg); + m_useV2ListApi = string.Equals(options.GetValueOrDefault("list-api-version", "v1"), "v2", StringComparison.OrdinalIgnoreCase); m_locationConstraint = locationConstraint; m_storageClass = storageClass; m_dnsHost = string.IsNullOrWhiteSpace(cfg.ServiceURL) ? null : new System.Uri(cfg.ServiceURL).Host; @@ -192,32 +194,65 @@ namespace Duplicati.Library.Backend return m_client.DeleteObjectAsync(objectDeleteRequest, cancellationToken); } - public async IAsyncEnumerable ListBucketAsync(string bucketName, string prefix, [EnumeratorCancellation] CancellationToken cancellationToken) + /// + /// Lists the contents of a bucket, using a plugable API call + /// + /// The bucket to list + /// The prefix to list + /// If true, the list is recursive + /// The cancellation token + /// The list of files + public async IAsyncEnumerable ListBucketAsync(string bucketName, string prefix, bool recursive, [EnumeratorCancellation] CancellationToken cancellationToken) { - bool isTruncated = true; + var isTruncated = true; string filename = null; + var delimiter = recursive ? "" : "/"; //TODO: Figure out if this is the case with AWSSDK too //Unfortunately S3 sometimes reports duplicate values when requesting more than one page of results //So, track the files that have already been returned and skip any duplicates. - HashSet alreadyReturned = new HashSet(); + var alreadyReturned = new HashSet(); //We truncate after ITEM_LIST_LIMIT elements, and then repeat while (isTruncated) { - var listRequest = new ListObjectsRequest { BucketName = bucketName }; - - if (!string.IsNullOrEmpty(filename)) - listRequest.Marker = filename; - - listRequest.MaxKeys = ITEM_LIST_LIMIT; - if (!string.IsNullOrEmpty(prefix)) - listRequest.Prefix = prefix; - - ListObjectsResponse listResponse; + ListObjectsV2Response listResponse; try { - listResponse = await m_client.ListObjectsAsync(listRequest, cancellationToken).ConfigureAwait(false); + if (m_useV2ListApi) + { + var listRequest = new ListObjectsV2Request + { + BucketName = bucketName, + Prefix = prefix, + ContinuationToken = filename, + MaxKeys = ITEM_LIST_LIMIT, + Delimiter = delimiter + }; + listResponse = await m_client.ListObjectsV2Async(listRequest, cancellationToken).ConfigureAwait(false); + } + else + { + // Use the V1 API + var listRequest = new ListObjectsRequest + { + BucketName = bucketName, + Prefix = prefix, + Marker = filename, + MaxKeys = ITEM_LIST_LIMIT, + Delimiter = delimiter + }; + var listResponsev1 = await m_client.ListObjectsAsync(listRequest, cancellationToken).ConfigureAwait(false); + + // Map the V1 response to the V2 response + listResponse = new ListObjectsV2Response + { + CommonPrefixes = listResponsev1.CommonPrefixes, + IsTruncated = listResponsev1.IsTruncated, + NextContinuationToken = listResponsev1.NextMarker, + S3Objects = listResponsev1.S3Objects + }; + } } catch (AmazonS3Exception e) { @@ -231,16 +266,34 @@ namespace Duplicati.Library.Backend } isTruncated = listResponse.IsTruncated; - filename = listResponse.NextMarker; + filename = listResponse.NextContinuationToken; + + foreach (var obj in listResponse.CommonPrefixes) + { + // Because the prefixes are returned, and not the folder objects + // we do not get the folder modification date + yield return new Common.IO.FileEntry( + obj, + -1, + new DateTime(0), + new DateTime(0) + ) + { IsFolder = true }; + } foreach (var obj in listResponse.S3Objects.Where(obj => alreadyReturned.Add(obj.Key))) { + // Skip self-prefix, this discards the folder modification date :/ + if (obj.Key == prefix) + continue; + yield return new Common.IO.FileEntry( obj.Key, obj.Size, obj.LastModified, obj.LastModified - ); + ) + { IsFolder = obj.Key.EndsWith("/") }; } } } diff --git a/Duplicati/Library/Backend/S3/S3Backend.cs b/Duplicati/Library/Backend/S3/S3Backend.cs index 92d4029d1..dffb5035d 100644 --- a/Duplicati/Library/Backend/S3/S3Backend.cs +++ b/Duplicati/Library/Backend/S3/S3Backend.cs @@ -44,6 +44,8 @@ namespace Duplicati.Library.Backend private const string SSL_OPTION = "use-ssl"; private const string S3_CLIENT_OPTION = "s3-client"; private const string S3_DISABLE_CHUNK_ENCODING_OPTION = "s3-disable-chunk-encoding"; + private const string S3_LIST_API_VERSION_OPTION = "s3-list-api-version"; + private const string S3_RECURSIVE_LIST = "s3-recursive-list"; public static readonly Dictionary KNOWN_S3_PROVIDERS = new Dictionary(StringComparer.OrdinalIgnoreCase) { { "Amazon S3", "s3.amazonaws.com" }, @@ -174,6 +176,7 @@ namespace Duplicati.Library.Backend private readonly string m_bucket; private readonly string m_prefix; + private readonly bool m_recurseLists; /// /// Source provider to be used if the backend is a source @@ -235,6 +238,8 @@ namespace Duplicati.Library.Backend if (m_prefix.Length != 0) m_prefix = Util.AppendDirSeparator(m_prefix, "/"); + m_recurseLists = Utility.Utility.ParseBoolOption(options, S3_RECURSIVE_LIST); + // Auto-disable DNS lookup for non-AWS configurations if (!options.ContainsKey("s3-ext-forcepathstyle") && !hostname.EndsWith(".amazonaws.com", StringComparison.OrdinalIgnoreCase)) options["s3-ext-forcepathstyle"] = "true"; @@ -256,7 +261,9 @@ namespace Duplicati.Library.Backend throw new UserInformationException(Strings.S3Backend.UnknownS3ClientError(s3ClientOptionValue), "UnknownS3Client"); } - m_sourcePathKey = $"@{ProtocolKey}://{hostname}/{m_bucket}/{m_prefix}/"; + m_sourcePathKey = $"{Util.RemotePathPrefix}{ProtocolKey}://{hostname}/{m_bucket}/"; + if (!string.IsNullOrWhiteSpace(m_prefix)) + m_sourcePathKey += m_prefix; m_sourceProvider = new BackendSourceProvider(this); } @@ -287,7 +294,7 @@ namespace Duplicati.Library.Backend public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { - await foreach (IFileEntry file in Connection.ListBucketAsync(m_bucket, m_prefix, cancelToken)) + await foreach (IFileEntry file in Connection.ListBucketAsync(m_bucket, m_prefix, m_recurseLists, cancelToken)) { ((FileEntry)file).Name = file.Name.Substring(m_prefix.Length); @@ -349,6 +356,8 @@ namespace Duplicati.Library.Backend new CommandLineArgument(SSL_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.S3Backend.DescriptionUseSSLShort, Strings.S3Backend.DescriptionUseSSLLong), new CommandLineArgument(S3_CLIENT_OPTION, CommandLineArgument.ArgumentType.Enumeration, Strings.S3Backend.S3ClientDescriptionShort, Strings.S3Backend.S3ClientDescriptionLong, "aws", null, new string[] { "aws", "minio" }), new CommandLineArgument(S3_DISABLE_CHUNK_ENCODING_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.S3Backend.DescriptionDisableChunkEncodingShort, Strings.S3Backend.DescriptionDisableChunkEncodingLong, "false"), + new CommandLineArgument(S3_LIST_API_VERSION_OPTION, CommandLineArgument.ArgumentType.Enumeration, Strings.S3Backend.DescriptionListApiVersionShort, Strings.S3Backend.DescriptionListApiVersionLong, "v1", null, ["v1", "v2"]), + new CommandLineArgument(S3_RECURSIVE_LIST, CommandLineArgument.ArgumentType.Boolean, Strings.S3Backend.DescriptionRecursiveListShort, Strings.S3Backend.DescriptionRecursiveListLong, "false"), new CommandLineArgument("auth-password", CommandLineArgument.ArgumentType.Password, Strings.S3Backend.AuthPasswordDescriptionShort, Strings.S3Backend.AuthPasswordDescriptionLong), new CommandLineArgument("auth-username", CommandLineArgument.ArgumentType.String, Strings.S3Backend.AuthUsernameDescriptionShort, Strings.S3Backend.AuthUsernameDescriptionLong), }; @@ -416,30 +425,8 @@ namespace Duplicati.Library.Backend public string PathKey => m_sourcePathKey; /// - public async IAsyncEnumerable ListAsync(string path, [EnumeratorCancellation] CancellationToken cancellationToken) - { - var prefixPath = m_prefix; - if (!prefixPath.EndsWith("/", StringComparison.Ordinal)) - prefixPath += "/"; - - var filterPath = prefixPath + path; - if (!filterPath.EndsWith("/", StringComparison.Ordinal)) - filterPath += "/"; - - await foreach (var f in s3Client.ListBucketAsync(m_bucket, filterPath, cancellationToken).ConfigureAwait(false)) - { - if (!f.Name.StartsWith(prefixPath, StringComparison.Ordinal)) - continue; - - ((FileEntry)f).Name = f.Name.Substring(prefixPath.Length); - - // Skip sub-folder items - if (f.Name.Contains("/")) - continue; - - yield return BackendSourceFileEntry.FromFileEntry(m_sourceProvider, path, f); - } - } + public IAsyncEnumerable ListAsync(string path, CancellationToken cancellationToken) + => m_sourceProvider.ListFromFileEntryAsync(m_prefix, path, (filter, token) => s3Client.ListBucketAsync(m_bucket, filter, m_recurseLists, token), cancellationToken); /// public Task GetEntryAsync(string path, CancellationToken cancellationToken) diff --git a/Duplicati/Library/Backend/S3/S3MinioClient.cs b/Duplicati/Library/Backend/S3/S3MinioClient.cs index cee966473..e48c7b301 100644 --- a/Duplicati/Library/Backend/S3/S3MinioClient.cs +++ b/Duplicati/Library/Backend/S3/S3MinioClient.cs @@ -81,11 +81,11 @@ namespace Duplicati.Library.Backend } } - public async IAsyncEnumerable ListBucketAsync(string bucketName, string prefix, [EnumeratorCancellation] CancellationToken cancellationToken) + public async IAsyncEnumerable ListBucketAsync(string bucketName, string prefix, bool recursive, [EnumeratorCancellation] CancellationToken cancellationToken) { ThrowExceptionIfBucketDoesNotExist(bucketName); - var observable = m_client.ListObjectsAsync(bucketName, prefix, true, cancellationToken); + var observable = m_client.ListObjectsAsync(bucketName, prefix, recursive, cancellationToken); await foreach (var obj in ToAsyncEnumerable(observable, cancellationToken).ConfigureAwait(false)) { yield return new Common.IO.FileEntry( @@ -93,7 +93,8 @@ namespace Duplicati.Library.Backend (long)obj.Size, Convert.ToDateTime(obj.LastModified), Convert.ToDateTime(obj.LastModified) - ); + ) + { IsFolder = obj.Key.EndsWith("/") }; } } diff --git a/Duplicati/Library/Backend/S3/Strings.cs b/Duplicati/Library/Backend/S3/Strings.cs index 49062dc97..c7721bcdd 100644 --- a/Duplicati/Library/Backend/S3/Strings.cs +++ b/Duplicati/Library/Backend/S3/Strings.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Localization.Short; namespace Duplicati.Library.Backend.Strings @@ -49,6 +49,10 @@ namespace Duplicati.Library.Backend.Strings public static string DescriptionDisableChunkEncodingShort { get { return LC.L(@"Disable chunk encoding (aws client only)"); } } public static string S3StorageclassDescriptionLong { get { return LC.L(@"Use this option to specify a storage class. If this option is not used, the server will choose a default storage class."); } } public static string S3StorageclassDescriptionShort { get { return LC.L(@"Specify storage class"); } } + public static string DescriptionListApiVersionShort { get { return LC.L(@"Specify the S3 list API version to use"); } } + public static string DescriptionListApiVersionLong { get { return LC.L(@"Use this option to specify the S3 list API version to use. This can be used to work around issues with some S3 providers."); } } + public static string DescriptionRecursiveListShort { get { return LC.L(@"Use this option to list all files in the bucket"); } } + public static string DescriptionRecursiveListLong { get { return LC.L(@"To reduce the number of objects listed, the default is to only list the first level of objects. Use this option to list all objects in the bucket."); } } public static string UnknownS3ClientError(string client) { return LC.L(@"Unknown S3 client: {0}", client); } } } diff --git a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs index e9abbe27e..8f3c1e846 100644 --- a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs +++ b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs @@ -39,6 +39,7 @@ namespace Duplicati.Library.Backend private const string SSH_KEYFILE_OPTION = "ssh-keyfile"; private const string SSH_KEYFILE_INLINE = "ssh-key"; private const string SSH_FINGERPRINT_OPTION = "ssh-fingerprint"; + private const string SSH_RELATIVE_PATH_OPTION = "ssh-relative-path"; private const string SSH_FINGERPRINT_ACCEPT_ANY_OPTION = "ssh-accept-any-fingerprints"; public const string KEYFILE_URI = "sshkey://"; private const string SSH_TIMEOUT_OPTION = "ssh-operation-timeout"; @@ -56,6 +57,8 @@ namespace Duplicati.Library.Backend private readonly TimeSpan m_keepaliveinterval; private readonly int m_port = 22; + private readonly bool m_relativePath; + private string m_initialDirectory; /// /// The source provider for this backend @@ -91,6 +94,7 @@ namespace Duplicati.Library.Backend m_password = uri.Password; m_fingerprintallowall = Utility.Utility.ParseBoolOption(options, SSH_FINGERPRINT_ACCEPT_ANY_OPTION); + m_relativePath = Utility.Utility.ParseBoolOption(options, SSH_RELATIVE_PATH_OPTION); m_path = uri.Path; @@ -99,8 +103,15 @@ namespace Duplicati.Library.Backend m_path = Util.AppendDirSeparator(m_path, "/"); } - if (!m_path.StartsWith("/", StringComparison.Ordinal)) - m_path = "/" + m_path; + if (m_relativePath) + { + m_path = m_path.TrimStart('/'); + } + else + { + if (!m_path.StartsWith("/", StringComparison.Ordinal)) + m_path = "/" + m_path; + } m_server = uri.Host; @@ -118,7 +129,9 @@ namespace Duplicati.Library.Backend if (!string.IsNullOrWhiteSpace(timeoutstr)) m_keepaliveinterval = Library.Utility.Timeparser.ParseTimeSpan(timeoutstr); - m_sourcePathKey = $"@{ProtocolKey}://{m_server}:{m_port}/~{m_username}/{m_path}/"; + m_sourcePathKey = $"{Util.RemotePathPrefix}{ProtocolKey}://{m_server}:{m_port}/~{m_username}/"; + if (string.IsNullOrWhiteSpace(m_path)) + m_sourcePathKey += $"{m_path}/"; m_sourceProvider = new BackendSourceProvider(this); } @@ -179,7 +192,7 @@ namespace Duplicati.Library.Backend try { CreateConnection(); - ChangeDirectory(m_path); + SetWorkingDirectory(); await m_con.DeleteFileAsync(remotename, cancelToken).ConfigureAwait(false); } catch (SftpPathNotFoundException ex) @@ -218,6 +231,9 @@ namespace Duplicati.Library.Backend new CommandLineArgument(SSH_KEEPALIVE_OPTION, CommandLineArgument.ArgumentType.Timespan, Strings.SSHv2Backend.DescriptionSshkeepaliveShort, Strings.SSHv2Backend.DescriptionSshkeepaliveLong, "0"), + new CommandLineArgument(SSH_RELATIVE_PATH_OPTION, CommandLineArgument.ArgumentType.Boolean, + Strings.SSHv2Backend.DescriptionRelativePathShort, + Strings.SSHv2Backend.DescriptionRelativePathLong) }); } } @@ -255,7 +271,7 @@ namespace Duplicati.Library.Backend public Task PutAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) { CreateConnection(); - ChangeDirectory(m_path); + SetWorkingDirectory(); return Task.Factory.FromAsync( (cb, state) => m_con.BeginUploadFile(stream, remotename, cb, state, _ => cancelToken.ThrowIfCancellationRequested()), m_con.EndUploadFile, @@ -265,7 +281,7 @@ namespace Duplicati.Library.Backend public async Task GetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) { CreateConnection(); - ChangeDirectory(m_path); + SetWorkingDirectory(); try { @@ -287,7 +303,7 @@ namespace Duplicati.Library.Backend public Task RenameAsync(string source, string target, CancellationToken cancelToken) { CreateConnection(); - ChangeDirectory(m_path); + SetWorkingDirectory(); return m_con.RenameFileAsync(source, target, cancelToken); } @@ -358,6 +374,9 @@ namespace Duplicati.Library.Backend con.KeepAliveInterval = m_keepaliveinterval; this.TryConnect(con); + if (m_relativePath && (string.IsNullOrWhiteSpace(con.WorkingDirectory) || !con.WorkingDirectory.StartsWith("/"))) + throw new UserInformationException("Server does not report absolute initial directory, please switch to absolute paths", "RelativePathNotSupported"); + m_initialDirectory = con.WorkingDirectory; m_con = con; } @@ -367,23 +386,27 @@ namespace Duplicati.Library.Backend client.Connect(); } - private void ChangeDirectory(string path) + private void SetWorkingDirectory() { - if (string.IsNullOrEmpty(path)) + if (string.IsNullOrEmpty(m_path)) return; + var targetPath = m_relativePath + ? Util.AppendDirSeparator(m_initialDirectory, "/") + m_path + : m_path; + var workingDir = Util.AppendDirSeparator(m_con.WorkingDirectory, "/"); - if (workingDir == path) + if (workingDir == targetPath) return; try { - m_con.ChangeDirectory(path); + m_con.ChangeDirectory(targetPath); } catch (Exception ex) { throw new Interface.FolderMissingException( - Strings.SSHv2Backend.FolderNotFoundManagedError(path, ex.Message), ex); + Strings.SSHv2Backend.FolderNotFoundManagedError(m_path, ex.Message), ex); } } @@ -392,7 +415,7 @@ namespace Duplicati.Library.Backend string path = "."; CreateConnection(); - ChangeDirectory(m_path); + SetWorkingDirectory(); foreach (var ls in m_con.ListDirectory(path)) { @@ -449,13 +472,19 @@ namespace Duplicati.Library.Backend /// public IAsyncEnumerable ListAsync(string path, CancellationToken cancellationToken) { - var prefixPath = m_path; - if (!prefixPath.EndsWith("/")) - prefixPath += "/"; + CreateConnection(); + SetWorkingDirectory(); + + var prefixPath = m_relativePath ? "" : m_path; + if (!string.IsNullOrWhiteSpace(prefixPath)) + prefixPath = Util.AppendDirSeparator(prefixPath, "/"); var filterPath = prefixPath + path; - if (!filterPath.EndsWith("/")) - filterPath += "/"; + if (!string.IsNullOrWhiteSpace(filterPath)) + filterPath = Util.AppendDirSeparator(filterPath, "/"); + + if (string.IsNullOrWhiteSpace(filterPath)) + filterPath = "."; return Client.ListDirectoryAsync(filterPath, cancellationToken) .Select(x => new BackendSourceFileEntry( diff --git a/Duplicati/Library/Backend/SSHv2/Strings.cs b/Duplicati/Library/Backend/SSHv2/Strings.cs index 95360cdd8..a182b771a 100644 --- a/Duplicati/Library/Backend/SSHv2/Strings.cs +++ b/Duplicati/Library/Backend/SSHv2/Strings.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Localization.Short; namespace Duplicati.Library.Backend.Strings @@ -61,6 +61,8 @@ namespace Duplicati.Library.Backend.Strings public static string DescriptionSshtimeoutShort { get { return LC.L(@"Set the operation timeout value"); } } public static string DescriptionSshkeepaliveLong { get { return LC.L(@"Use this option to enable the keep-alive interval for the SSH connection. If the connection is idle, aggressive firewalls might close the connection. Using keep-alive will keep the connection open in this scenario. If this value is set to zero, the keep-alive is disabled."); } } public static string DescriptionSshkeepaliveShort { get { return LC.L(@"Set a keepalive value"); } } + public static string DescriptionRelativePathShort { get { return LC.L(@"Treat source path as relative to the initial path"); } } + public static string DescriptionRelativePathLong { get { return LC.L(@"Use this option to treat the source path as relative to the initial path. This is useful when the full path of the system is not known."); } } public static string FolderNotFoundManagedError(string foldername, string message) { return LC.L(@"Unable to set folder to {0}, error message: {1}", foldername, message); } public static string FingerprintNotMatchManagedError(string fingerprint) { return LC.L(@"Validation of server fingerprint failed. Server returned fingerprint ""{0}"". Cause of this message is either not correct configuration or Man-in-the-middle attack!", fingerprint); } public static string FingerprintNotSpecifiedManagedError(string fingerprint, string hostkeyoption, string allkeysoptions) { return LC.L(@"Please add --{1}=""{0}"" to trust this host. Optionally you can use --{2} (NOT SECURE) for testing!", fingerprint, hostkeyoption, allkeysoptions); } diff --git a/Duplicati/Library/Common/IO/Util.cs b/Duplicati/Library/Common/IO/Util.cs index 80b247cf6..460c6fb2c 100644 --- a/Duplicati/Library/Common/IO/Util.cs +++ b/Duplicati/Library/Common/IO/Util.cs @@ -36,6 +36,11 @@ namespace Duplicati.Library.Common.IO /// public static readonly string AltDirectorySeparatorString = Path.AltDirectorySeparatorChar.ToString(); + /// + /// The prefix marker for a remote path + /// + public const string RemotePathPrefix = "@"; + /// /// Filename of a marker file that can be put inside the data folder to prevent Duplicati from fixing lax permissions /// @@ -72,6 +77,8 @@ namespace Duplicati.Library.Common.IO /// The guessed directory separator public static string GuessDirSeparator(string path) { + if (path != null && path.StartsWith(RemotePathPrefix, StringComparison.Ordinal)) + throw new ArgumentException("Path cannot start with @, most likely this is a remote path", nameof(path)); return string.IsNullOrWhiteSpace(path) || path.StartsWith("/", StringComparison.Ordinal) ? "/" : "\\"; } diff --git a/Duplicati/Library/Main/Controller.cs b/Duplicati/Library/Main/Controller.cs index d391b7fb3..312df9f8f 100644 --- a/Duplicati/Library/Main/Controller.cs +++ b/Duplicati/Library/Main/Controller.cs @@ -948,6 +948,20 @@ namespace Duplicati.Library.Main string source; try { + // Check if this is not a file source + var scheme = Library.Utility.Utility.GuessScheme(expandedSource) ?? "file"; + if (scheme != "file") + { + // TODO: If the remote source fails to load, + // this will be an enumeration warning, but will result + // in the backup being recorded without files from the source + // Eventually, this could lead to retention causing the last backup + // with the data from the source to be deleted + foundAnyPaths = true; + sources.Add(expandedSource); + continue; + } + // TODO: This expands "C:" to CWD, but not C:\ source = System.IO.Path.GetFullPath(expandedSource); } diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index 888b4fea5..f558c6e8a 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -22,6 +22,7 @@ using System; using System.Collections.Generic; using System.Data; using System.Linq; +using Duplicati.Library.Common.IO; namespace Duplicati.Library.Main.Database { @@ -877,11 +878,11 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist } } - public string GetFirstPath() + public string GetFirstLocalPath() { using (var cmd = m_connection.CreateCommand()) { - cmd.CommandText = @"SELECT ""Path"" FROM ""File"" ORDER BY LENGTH(""Path"") DESC LIMIT 1"; + cmd.CommandText = @$"SELECT ""Path"" FROM ""File"" WHERE ""Path"" NOT LIKE '{Util.RemotePathPrefix}%' ORDER BY LENGTH(""Path"") DESC LIMIT 1"; var v0 = cmd.ExecuteScalar(); if (v0 == null || v0 == DBNull.Value) return null; diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 8d09a6f27..de97b7449 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -269,7 +269,7 @@ namespace Duplicati.Library.Main.Operation Utility.UpdateOptionsFromDb(database, options); Utility.VerifyOptionsAndUpdateDatabase(database, options); - var probe_path = database.GetFirstPath(); + var probe_path = database.GetFirstLocalPath(); if (probe_path != null && Util.GuessDirSeparator(probe_path) != Util.DirectorySeparatorString) throw new UserInformationException(string.Format("The backup contains files that belong to another operating system. Proceeding with a backup would cause the database to contain paths from two different operation systems, which is not supported. To proceed without losing remote data, delete all filesets and make sure the --{0} option is set, then run the backup again to re-use the existing data on the remote store.", "no-auto-compact"), "CrossOsDatabaseReuseNotSupported"); diff --git a/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs b/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs index 7b6fa496c..535ec7d5d 100644 --- a/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs +++ b/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs @@ -19,6 +19,7 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +using System.IO.Pipelines; using Duplicati.Library.Interface; using Duplicati.Library.Utility; @@ -37,6 +38,11 @@ namespace Duplicati.Library.SourceProvider; public class BackendSourceFileEntry(BackendSourceProvider parent, string path, bool isFolder, bool isMetaEntry, DateTime createdUtc, DateTime lastModificationUtc, long size) : ISourceFileEntry { + /// + /// The log tag for this instance + /// + private static string LOGTAG = Logging.Log.LogTagFromType(); + /// public bool IsFolder => isFolder; @@ -55,11 +61,6 @@ public class BackendSourceFileEntry(BackendSourceProvider parent, string path, b /// public string Path => ConcatPaths(parent.PathKey, NormalizePath(path)); - /// - /// The local path of the entry - /// - public string LocalPath => path; - /// public long Size => size; /// @@ -95,7 +96,13 @@ public class BackendSourceFileEntry(BackendSourceProvider parent, string path, b throw new InvalidOperationException("Enumerate can only be called on folders"); return parent.WrappedBackend.ListAsync(path, cancellationToken) - .Select(x => new BackendSourceFileEntry(parent, x.Path, x.IsFolder, false, x.CreatedUtc, x.LastModificationUtc, x.Size)); + .Select(x => + { + if (x is BackendSourceFileEntry) + return x; + + return new BackendSourceFileEntry(parent, x.Path, x.IsFolder, false, x.CreatedUtc, x.LastModificationUtc, x.Size); + }); } /// @@ -119,6 +126,59 @@ public class BackendSourceFileEntry(BackendSourceProvider parent, string path, b public Task OpenMetadataRead(CancellationToken cancellationToken) => Task.FromResult(null); + /// + /// Helper class for reporting th length of a stream + /// + /// The stream to wrap + /// The length to report + private class LengthReportingStream(Stream stream, long reportedLength) : OverrideableStream(stream) + { + /// + /// Track the position + /// + private long position = 0; + + /// + public override long Length => reportedLength; + + /// + public override long Position + { + get => position; + set => position = base.Position = value; + } + + /// + public override void Write(byte[] buffer, int offset, int count) + { + base.Write(buffer, offset, count); + position += count; + } + + /// + public override int Read(byte[] buffer, int offset, int count) + { + var read = base.Read(buffer, offset, count); + position += read; + return read; + } + + /// + public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + var read = await base.ReadAsync(buffer, offset, count, cancellationToken); + position += read; + return read; + } + + /// + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + await base.WriteAsync(buffer, offset, count, cancellationToken); + position += count; + } + } + /// public async Task OpenRead(CancellationToken cancellationToken) { @@ -127,23 +187,48 @@ public class BackendSourceFileEntry(BackendSourceProvider parent, string path, b if (parent.WrappedBackend is IStreamingBackend streamingBackend) { - var mapStream = new MapStream(); - mapStream.CopyTask = streamingBackend.GetAsync(this.Path, mapStream, cancellationToken); - return mapStream; + var pipe = new Pipe(); + + // Start writing data to the pipe asynchronously + _ = Task.Run(async () => + { + try + { + await streamingBackend.GetAsync(path, pipe.Writer.AsStream(), cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + await pipe.Writer.CompleteAsync(ex).ConfigureAwait(false); + } + finally + { + await pipe.Writer.CompleteAsync().ConfigureAwait(false); + } + }) + .ContinueWith(t => + { + if (t.IsFaulted) + Logging.Log.WriteWarningMessage(LOGTAG, "ErrorPipingStream", t.Exception, "Error piping stream for {0}", this.Path); + }); + + // Return the readable stream so the caller can read data as it's produced + return new LengthReportingStream(pipe.Reader.AsStream(), size); } else { + TempFile? tempFile = null; TempFileStream? file = null; try { - file = TempFileStream.Create(); - await parent.WrappedBackend.GetAsync(this.Path, file.Path, cancellationToken); - file.Position = 0; + tempFile = new TempFile(); + await parent.WrappedBackend.GetAsync(path, tempFile, cancellationToken); + file = TempFileStream.Create(tempFile); return file; } catch { file?.Dispose(); + tempFile?.Dispose(); throw; } } diff --git a/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs index 618de97f7..3943fb1b0 100644 --- a/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs +++ b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs @@ -52,7 +52,7 @@ public class BackendSourceProvider(IFolderEnabledBackend backend) : ISourceProvi /// public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) - => new[] { new BackendSourceFileEntry(this, PathKey, true, true, new DateTime(0), new DateTime(0), 0) }.ToAsyncEnumerable(); + => new[] { new BackendSourceFileEntry(this, "", true, true, new DateTime(0), new DateTime(0), 0) }.ToAsyncEnumerable(); /// public Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) @@ -64,4 +64,40 @@ public class BackendSourceProvider(IFolderEnabledBackend backend) : ISourceProvi WrappedBackend.Dispose(); GC.SuppressFinalize(this); } + + /// + /// Helper function to use a List function that returns IFileEntry and convert it to ISourceFileEntry + /// + /// The prefix to hide from the caller + /// The logical path to list + /// The function that returns the IFileEntry items + /// The cancellation token + /// The list of source file entries + public async IAsyncEnumerable ListFromFileEntryAsync(string prefix, string path, Func> listFunction, [EnumeratorCancellation] CancellationToken cancellationToken) + { + if (!string.IsNullOrWhiteSpace(prefix) && !prefix.EndsWith("/", StringComparison.Ordinal)) + prefix += "/"; + + // Logic here is that the prefix is invisible to the caller, so we add it to the prefix to filter the results + // and remove it from the name to make the prefix invisible. + // The input path is preserved so the output looks like a "full path" to the caller. + + var filterPath = prefix + path; + if (filterPath != "" && !filterPath.EndsWith("/", StringComparison.Ordinal)) + filterPath += "/"; + + await foreach (var f in listFunction(filterPath, cancellationToken).ConfigureAwait(false)) + { + if (!f.Name.StartsWith(filterPath, StringComparison.Ordinal)) + continue; + + ((FileEntry)f).Name = f.Name.Substring(filterPath.Length); + + // Skip self and sub-folder items + if (f.Name.Length == 0 || f.Name[0..^1].Contains("/")) + continue; + + yield return BackendSourceFileEntry.FromFileEntry(this, path, f); + } + } } diff --git a/Duplicati/Library/SourceProvider/Builtin/Duplicati.Library.SourceProvider.Builtin.csproj b/Duplicati/Library/SourceProvider/Builtin/Duplicati.Library.SourceProvider.Builtin.csproj index 4ba5377b7..e48c06ee4 100644 --- a/Duplicati/Library/SourceProvider/Builtin/Duplicati.Library.SourceProvider.Builtin.csproj +++ b/Duplicati/Library/SourceProvider/Builtin/Duplicati.Library.SourceProvider.Builtin.csproj @@ -24,4 +24,8 @@ + + + + diff --git a/Duplicati/Library/Utility/MapStream.cs b/Duplicati/Library/Utility/MapStream.cs deleted file mode 100644 index b2c717739..000000000 --- a/Duplicati/Library/Utility/MapStream.cs +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - -#nullable enable - -using System; -using System.IO; -using System.Threading; -using System.Threading.Channels; -using System.Threading.Tasks; - -namespace Duplicati.Library.Utility; - -/// -/// A stream that maps data with an internal buffer. -/// This allows a reader and a writer to share a stream without blocking. -/// -public class MapStream : Stream -{ - /// - /// The channel used to communicate between the reader and writer. - /// - private readonly Channel channel = Channel.CreateBounded(0); - - /// - /// The buffer used to store data that has been written but not consumed by the reader. - /// - private Memory readBuffer = Array.Empty(); - - /// - /// The task that is using the map stream. - /// - public Task? CopyTask { get; set; } - - /// - public override bool CanRead => true; - - /// - public override bool CanSeek => false; - - /// - public override bool CanWrite => true; - - /// - public override long Length => throw new NotImplementedException(); - - /// - public override long Position { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } - - /// - public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - int read = 0; - // If we have left-over data from the previous read, return it - if (readBuffer.Length > 0) - { - // Return as much as possible - var requested = Math.Min(readBuffer.Length, count); - readBuffer.Span.Slice(0, requested).CopyTo(buffer.AsSpan(offset, count)); - // Adjust the remaining buffer - if (readBuffer.Length > requested) - readBuffer = readBuffer.Slice(requested); - else - readBuffer = Array.Empty(); - - // Adjust the counters - offset += requested; - count -= requested; - read += requested; - - // Return if we have read all that was requested - if (count == 0) - return read; - } - - // Handle external task - var ct = CopyTask; - - // If the task is completed, clear it - if (ct != null && ct.IsCompletedSuccessfully) - CopyTask = ct = null; - // If the task failed, stop reading - else if (ct != null && (ct.IsCanceled || ct.IsFaulted)) - { - // If we have any data in the buffer, return it - if (read > 0) - return read; - - // Otherwise, throw the exception - await ct; - } - - if (!await channel.Reader.WaitToReadAsync(cancellationToken)) - { - // If we have any data in the buffer, return it - if (read > 0) - return read; - - // If the copy crashed, throw the exception - ct = CopyTask; - if (ct != null && (ct.IsCanceled || ct.IsFaulted)) - await ct; - - // Otherwise, return 0 - return read; - } - - // Read the data from the channel - var res = await channel.Reader.ReadAsync(cancellationToken); - - // Consume as much as was requested - var consumed = Math.Min(res.Length, count); - - // Keep the rest in the buffer - if (res.Length > count) - readBuffer = res.AsMemory(count); - - res.AsSpan(0, consumed).CopyTo(buffer.AsSpan(offset, count)); - read += consumed; - return read; - } - - /// - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - var ct = CopyTask; - // Throw if the task failed - if (ct != null && (ct.IsCanceled || ct.IsFaulted)) - await ct; - await channel.Writer.WriteAsync(buffer.AsSpan(offset, count).ToArray(), cancellationToken); - } - - public override void Close() - { - base.Close(); - channel.Writer.Complete(); - } - - protected override void Dispose(bool disposing) - { - channel.Writer.Complete(); - base.Dispose(disposing); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) - => ReadAsync(buffer, offset, count, CancellationToken.None).Await(); - - public override long Seek(long offset, SeekOrigin origin) - => throw new NotImplementedException(); - - public override void SetLength(long value) - => throw new NotImplementedException(); - - public override void Write(byte[] buffer, int offset, int count) - => WriteAsync(buffer, offset, count, CancellationToken.None).Await(); -} diff --git a/Duplicati/Library/Utility/OverrideableStream.cs b/Duplicati/Library/Utility/OverrideableStream.cs index dd65cbbd0..35e8a5273 100644 --- a/Duplicati/Library/Utility/OverrideableStream.cs +++ b/Duplicati/Library/Utility/OverrideableStream.cs @@ -1,28 +1,27 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. using System; -using System.Collections.Generic; -using System.Text; using System.IO; +using System.Threading.Tasks; namespace Duplicati.Library.Utility { @@ -33,79 +32,78 @@ namespace Duplicati.Library.Utility /// public class OverrideableStream : Stream { - protected System.IO.Stream m_basestream; + /// + /// The base stream that is wrapped + /// + protected Stream m_basestream; + /// + /// Creates a new instance + /// + /// The stream to wrap + /// public OverrideableStream(Stream basestream) { - if (basestream == null) - throw new ArgumentNullException(nameof(basestream)); - m_basestream = basestream; + m_basestream = basestream ?? throw new ArgumentNullException(nameof(basestream)); } - public override bool CanRead - { - get { return m_basestream.CanRead; } - } + /// + public override bool CanRead => m_basestream.CanRead; - public override bool CanSeek - { - get { return m_basestream.CanSeek; } - } + /// + public override bool CanSeek => m_basestream.CanSeek; - public override bool CanWrite - { - get { return m_basestream.CanWrite; } - } + /// + public override bool CanWrite => m_basestream.CanWrite; - public override void Flush() - { - m_basestream.Flush(); - } + /// + public override void Flush() => m_basestream.Flush(); - public override long Length - { - get { return m_basestream.Length; } - } + /// + public override long Length => m_basestream.Length; + /// public override long Position { - get - { - return m_basestream.Position; - } - set - { - m_basestream.Position = value; - } + get => m_basestream.Position; + set => m_basestream.Position = value; } + /// public override int Read(byte[] buffer, int offset, int count) - { - return m_basestream.Read(buffer, offset, count); - } + => m_basestream.Read(buffer, offset, count); + /// public override long Seek(long offset, System.IO.SeekOrigin origin) - { - return m_basestream.Seek(offset, origin); - } + => m_basestream.Seek(offset, origin); + /// public override void SetLength(long value) - { - m_basestream.SetLength(value); - } + => m_basestream.SetLength(value); + /// public override void Write(byte[] buffer, int offset, int count) - { - m_basestream.Write(buffer, offset, count); - } + => m_basestream.Write(buffer, offset, count); + /// + public override Task ReadAsync(byte[] buffer, int offset, int count, System.Threading.CancellationToken cancellationToken) + => m_basestream.ReadAsync(buffer, offset, count, cancellationToken); + + /// + public override Task WriteAsync(byte[] buffer, int offset, int count, System.Threading.CancellationToken cancellationToken) + => m_basestream.WriteAsync(buffer, offset, count, cancellationToken); + + /// + public override Task FlushAsync(System.Threading.CancellationToken cancellationToken) + => m_basestream.FlushAsync(cancellationToken); + + /// protected override void Dispose(bool disposing) { - if (m_basestream != null) - m_basestream.Dispose(); + m_basestream?.Dispose(); m_basestream = null; base.Dispose(disposing); } - + } } diff --git a/Duplicati/Library/Utility/TempFileStream.cs b/Duplicati/Library/Utility/TempFileStream.cs index 104eab261..85b8efcd2 100644 --- a/Duplicati/Library/Utility/TempFileStream.cs +++ b/Duplicati/Library/Utility/TempFileStream.cs @@ -59,6 +59,16 @@ public class TempFileStream : StreamUtil.WrappingAsyncStream this.stream = stream; } + /// + /// Creates a new TempFileStream instance. + /// + /// The temporary file instance. + /// The new TempFileStream instance. + public static TempFileStream Create(TempFile tempFile) + { + return new TempFileStream(tempFile, File.Open(tempFile.Name, FileMode.Open, FileAccess.ReadWrite, FileShare.None)); + } + /// /// Creates a new TempFileStream instance. /// From cd512ea7dd06e19d87d283235861d2a3ea921038 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 21 Feb 2025 13:25:00 +0100 Subject: [PATCH 221/362] Changed remote path prefix to `>` to avoid clash with literal string prefix `@` --- Duplicati/Library/Common/IO/Util.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Common/IO/Util.cs b/Duplicati/Library/Common/IO/Util.cs index 460c6fb2c..f982b7568 100644 --- a/Duplicati/Library/Common/IO/Util.cs +++ b/Duplicati/Library/Common/IO/Util.cs @@ -39,7 +39,7 @@ namespace Duplicati.Library.Common.IO /// /// The prefix marker for a remote path /// - public const string RemotePathPrefix = "@"; + public const string RemotePathPrefix = ">"; /// /// Filename of a marker file that can be put inside the data folder to prevent Duplicati from fixing lax permissions From 9dfe977d3f67b04f07303aca41d9e93a821a010d Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 21 Feb 2025 16:06:55 +0100 Subject: [PATCH 222/362] Changed logic so the source providers are now logically mounted into the source directory structure. This fixes the restore issues as all sources will now appear as if they were originally part of the source file system. The backend implementation has now been simplified so the backends only return names of entries with no path information. Most backends already do this, but the S3 backends did not, so this was fixed as well. This also clears up the interfaces, so the backend does not know about the `ISourceProviderEntry`. All mapping is done by the `BackendSourceProvider`, making it simpler to support other backends as sources. --- Duplicati/CommandLine/CLI/Commands.cs | 2 +- .../SourceTool/Commands/Download.cs | 2 +- .../CommandLine/SourceTool/Commands/List.cs | 3 +- .../CommandLine/SourceTool/GetProvider.cs | 2 +- .../CommandLine/SourceTool/ListEnumerator.cs | 4 +- Duplicati/Library/Backend/CIFS/CIFS.cs | 40 +++-------- .../Backend/Idrivee2/Idrivee2Backend.cs | 24 ++----- Duplicati/Library/Backend/S3/S3AwsClient.cs | 12 +++- Duplicati/Library/Backend/S3/S3Backend.cs | 35 ++++----- Duplicati/Library/Backend/S3/S3MinioClient.cs | 11 ++- .../Library/Backend/SSHv2/SSHv2Backend.cs | 42 ++++------- Duplicati/Library/Common/IO/Util.cs | 7 -- .../DynamicLoader/SourceProviderLoader.cs | 11 +-- .../Library/Interface/IDestinationProvider.cs | 43 +++++++++++ .../Interface/IFolderEnabledBackend.cs | 18 +++-- .../Library/Interface/ISourceProvider.cs | 8 +-- ...ceFileEntry.cs => ISourceProviderEntry.cs} | 6 +- Duplicati/Library/Main/Controller.cs | 9 ++- .../Main/Database/LocalBackupDatabase.cs | 4 +- .../Library/Main/Operation/Backup/Channels.cs | 2 +- .../Backup/FileEnumerationProcess.cs | 22 +++--- .../Operation/Backup/MetadataGenerator.cs | 2 +- .../Operation/Backup/MetadataPreProcess.cs | 4 +- .../Library/Main/Operation/BackupHandler.cs | 31 ++++++-- .../Library/Snapshots/ISnapshotService.cs | 6 +- Duplicati/Library/Snapshots/LinuxSnapshot.cs | 2 +- .../Library/Snapshots/NoSnapshotLinux.cs | 2 +- .../Library/Snapshots/NoSnapshotWindows.cs | 2 +- Duplicati/Library/Snapshots/SnapshotBase.cs | 6 +- .../Snapshots/SnapshotSourceFileEntry.cs | 4 +- .../Snapshots/USN/UsnJournalService.cs | 22 +++--- .../Library/Snapshots/WindowsSnapshot.cs | 2 +- .../Builtin/BackendSourceFileEntry.cs | 72 ++++++++++--------- .../Builtin/BackendSourceProvider.cs | 54 +++----------- .../SourceProvider/Builtin/Combiner.cs | 12 ++-- .../SourceProvider/Builtin/LocalFileSource.cs | 6 +- Duplicati/Library/Utility/Utility.cs | 3 + 37 files changed, 256 insertions(+), 281 deletions(-) create mode 100644 Duplicati/Library/Interface/IDestinationProvider.cs rename Duplicati/Library/Interface/{ISourceFileEntry.cs => ISourceProviderEntry.cs} (96%) diff --git a/Duplicati/CommandLine/CLI/Commands.cs b/Duplicati/CommandLine/CLI/Commands.cs index b5834a95d..76425af0c 100644 --- a/Duplicati/CommandLine/CLI/Commands.cs +++ b/Duplicati/CommandLine/CLI/Commands.cs @@ -622,7 +622,7 @@ namespace Duplicati.CommandLine switch (phase) { case Duplicati.Library.Main.OperationPhase.Backup_ProcessingFiles: - output.MessageEvent("Scanning local files ..."); + output.MessageEvent("Scanning source files ..."); periodicOutput.SetReady(); break; case Duplicati.Library.Main.OperationPhase.Backup_Finalize: diff --git a/Duplicati/CommandLine/SourceTool/Commands/Download.cs b/Duplicati/CommandLine/SourceTool/Commands/Download.cs index 7f2f1d046..295545ccd 100644 --- a/Duplicati/CommandLine/SourceTool/Commands/Download.cs +++ b/Duplicati/CommandLine/SourceTool/Commands/Download.cs @@ -51,7 +51,7 @@ public static class Download string localPath(string path) { - var relpath = path.Substring(source.PathKey.Length).TrimStart('/'); + var relpath = path.Substring(source.MountedPath.Length).TrimStart(Path.DirectorySeparatorChar); return Path.Combine(destination.FullName, relpath); } diff --git a/Duplicati/CommandLine/SourceTool/Commands/List.cs b/Duplicati/CommandLine/SourceTool/Commands/List.cs index a2169b99e..fd8cb6c25 100644 --- a/Duplicati/CommandLine/SourceTool/Commands/List.cs +++ b/Duplicati/CommandLine/SourceTool/Commands/List.cs @@ -55,7 +55,8 @@ public static class List files++; if (entry.IsFolder && level > 0) level--; - Console.WriteLine($"{new string(' ', (level + 1) * 2)}{entry.Path}"); + if (!entry.IsRootEntry) + Console.WriteLine($"{new string(' ', (level + 1) * 2)}{entry.Path}"); return Task.FromResult(true); }, token); diff --git a/Duplicati/CommandLine/SourceTool/GetProvider.cs b/Duplicati/CommandLine/SourceTool/GetProvider.cs index 62888b184..1f8b30e01 100644 --- a/Duplicati/CommandLine/SourceTool/GetProvider.cs +++ b/Duplicati/CommandLine/SourceTool/GetProvider.cs @@ -34,6 +34,6 @@ public static partial class Common /// The URL to get the provider for /// The source provider public static ISourceProvider GetProvider(string url) => - SourceProviderLoader.GetSourceProvider(url, []) + SourceProviderLoader.GetSourceProvider(url, "", []) ?? throw new UserInformationException(string.Format("The source \"{0}\" is not supported", url), "SourceNotSupported"); } \ No newline at end of file diff --git a/Duplicati/CommandLine/SourceTool/ListEnumerator.cs b/Duplicati/CommandLine/SourceTool/ListEnumerator.cs index e8899c8b6..5c6ca8845 100644 --- a/Duplicati/CommandLine/SourceTool/ListEnumerator.cs +++ b/Duplicati/CommandLine/SourceTool/ListEnumerator.cs @@ -35,9 +35,9 @@ public static partial class Common /// The visitor function /// The cancellation token /// An awaitable task - public static async Task Visit(ISourceProvider source, int maxdepth, Func> visitor, CancellationToken token) + public static async Task Visit(ISourceProvider source, int maxdepth, Func> visitor, CancellationToken token) { - var visit = new Stack<(ISourceFileEntry Entry, int Level)>(); + var visit = new Stack<(ISourceProviderEntry Entry, int Level)>(); await foreach (var item in source.Enumerate(token)) visit.Push((item, 0)); diff --git a/Duplicati/Library/Backend/CIFS/CIFS.cs b/Duplicati/Library/Backend/CIFS/CIFS.cs index 34b05ec0a..8af6a8494 100644 --- a/Duplicati/Library/Backend/CIFS/CIFS.cs +++ b/Duplicati/Library/Backend/CIFS/CIFS.cs @@ -61,16 +61,6 @@ public class CIFSBackend : IStreamingBackend, IFolderEnabledBackend /// private string _DnsName; - /// - /// Source provider to be used if the backend is a source - /// - private readonly BackendSourceProvider _sourceProvider; - - /// - /// Key to be used in the path for the source provider - /// - private readonly string _sourcePathKey; - /// /// Path separators (both Windows \ and unix /) to be used in path manipulation /// @@ -190,13 +180,6 @@ public class CIFSBackend : IStreamingBackend, IFolderEnabledBackend readBufferSize, writeBufferSize ); - - _sourcePathKey = $"{Util.RemotePathPrefix}{ProtocolKey}://{_connectionParameters.ServerName}/{_connectionParameters.ShareName}/"; - if (!string.IsNullOrEmpty(_connectionParameters.AuthUser)) - _sourcePathKey += $"~{_connectionParameters.AuthUser}/"; - if (!string.IsNullOrEmpty(_connectionParameters.Path)) - _sourcePathKey += $"{_connectionParameters.Path}/"; - _sourceProvider = new BackendSourceProvider(this); } /// @@ -359,25 +342,18 @@ public class CIFSBackend : IStreamingBackend, IFolderEnabledBackend } /// - public string PathKey => _sourcePathKey; - - /// - public async IAsyncEnumerable ListAsync(string path, [EnumeratorCancellation] CancellationToken cancellationToken) + public async IAsyncEnumerable ListAsync(string path, [EnumeratorCancellation] CancellationToken cancellationToken) { var sourcePath = _connectionParameters.Path; - if (!string.IsNullOrWhiteSpace(sourcePath) && !sourcePath.EndsWith('\\')) - sourcePath += '/'; + if (!string.IsNullOrWhiteSpace(sourcePath)) + sourcePath = Util.AppendDirSeparator(sourcePath, "/"); - foreach (var v in (await GetConnection().ListAsync(sourcePath + path, cancellationToken).ConfigureAwait(false)) - .Where(x => x.Name != "." && x.Name != "..") - .Select(f => BackendSourceFileEntry.FromFileEntry(_sourceProvider, path, f)) - .Cast()) - { - yield return v; - } + foreach (var v in await GetConnection().ListAsync(sourcePath + BackendSourceFileEntry.NormalizePathTo(path, '/'), cancellationToken).ConfigureAwait(false)) + if (v.Name != "." && v.Name != "..") + yield return v; } /// - public Task GetEntryAsync(string path, CancellationToken cancellationToken) - => Task.FromResult(null); + public Task GetEntryAsync(string path, CancellationToken cancellationToken) + => Task.FromResult(null); } \ No newline at end of file diff --git a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs index 2d5afdbab..db797a927 100644 --- a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs +++ b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs @@ -44,15 +44,6 @@ namespace Duplicati.Library.Backend private readonly string m_prefix; private readonly string m_bucket; - /// - /// The source provider for this backend - /// - private readonly BackendSourceProvider m_sourceProvider; - /// - /// The key the backend is identified by if it is a source - /// - private readonly string m_sourcePathKey; - private IS3Client m_s3Client; public Idrivee2Backend() @@ -90,10 +81,6 @@ namespace Duplicati.Library.Backend m_s3Client = new S3AwsClient(accessKeyId, accessKeySecret, null, host, null, true, false, options); - m_sourcePathKey = $"{Util.RemotePathPrefix}{ProtocolKey}://{host}/{m_bucket}/"; - if (!string.IsNullOrWhiteSpace(m_prefix)) - m_sourcePathKey += m_prefix; - m_sourceProvider = new BackendSourceProvider(this); } public string GetRegionEndpoint(string url) @@ -248,15 +235,12 @@ namespace Duplicati.Library.Backend } /// - public string PathKey => m_sourcePathKey; - - /// - public IAsyncEnumerable ListAsync(string path, CancellationToken cancellationToken) - => m_sourceProvider.ListFromFileEntryAsync(m_prefix, path, (filter, token) => m_s3Client.ListBucketAsync(m_bucket, filter, false, token), cancellationToken); + public IAsyncEnumerable ListAsync(string path, CancellationToken cancellationToken) + => m_s3Client.ListBucketAsync(m_bucket, path, false, cancellationToken); /// - public Task GetEntryAsync(string path, CancellationToken cancellationToken) - => Task.FromResult(null); + public Task GetEntryAsync(string path, CancellationToken cancellationToken) + => Task.FromResult(null); } } diff --git a/Duplicati/Library/Backend/S3/S3AwsClient.cs b/Duplicati/Library/Backend/S3/S3AwsClient.cs index 5cc424666..4527826dd 100644 --- a/Duplicati/Library/Backend/S3/S3AwsClient.cs +++ b/Duplicati/Library/Backend/S3/S3AwsClient.cs @@ -22,6 +22,7 @@ using Amazon.S3; using Amazon.S3.Model; +using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using Duplicati.Library.Utility; using System; @@ -207,6 +208,8 @@ namespace Duplicati.Library.Backend var isTruncated = true; string filename = null; var delimiter = recursive ? "" : "/"; + if (!string.IsNullOrWhiteSpace(prefix)) + prefix = Util.AppendDirSeparator(prefix, "/"); //TODO: Figure out if this is the case with AWSSDK too //Unfortunately S3 sometimes reports duplicate values when requesting more than one page of results @@ -270,10 +273,13 @@ namespace Duplicati.Library.Backend foreach (var obj in listResponse.CommonPrefixes) { + if (obj == prefix || !obj.StartsWith(prefix)) + continue; + // Because the prefixes are returned, and not the folder objects // we do not get the folder modification date yield return new Common.IO.FileEntry( - obj, + obj.Substring(prefix.Length), -1, new DateTime(0), new DateTime(0) @@ -284,11 +290,11 @@ namespace Duplicati.Library.Backend foreach (var obj in listResponse.S3Objects.Where(obj => alreadyReturned.Add(obj.Key))) { // Skip self-prefix, this discards the folder modification date :/ - if (obj.Key == prefix) + if (obj.Key == prefix || !obj.Key.StartsWith(prefix)) continue; yield return new Common.IO.FileEntry( - obj.Key, + obj.Key.Substring(prefix.Length), obj.Size, obj.LastModified, obj.LastModified diff --git a/Duplicati/Library/Backend/S3/S3Backend.cs b/Duplicati/Library/Backend/S3/S3Backend.cs index dffb5035d..2136eedbc 100644 --- a/Duplicati/Library/Backend/S3/S3Backend.cs +++ b/Duplicati/Library/Backend/S3/S3Backend.cs @@ -178,15 +178,6 @@ namespace Duplicati.Library.Backend private readonly string m_prefix; private readonly bool m_recurseLists; - /// - /// Source provider to be used if the backend is a source - /// - private readonly BackendSourceProvider m_sourceProvider; - /// - /// The key path for the backend as a source - /// - private readonly string m_sourcePathKey; - private const string DEFAULT_S3_HOST = "s3.amazonaws.com"; private IS3Client s3Client; @@ -260,11 +251,6 @@ namespace Duplicati.Library.Backend { throw new UserInformationException(Strings.S3Backend.UnknownS3ClientError(s3ClientOptionValue), "UnknownS3Client"); } - - m_sourcePathKey = $"{Util.RemotePathPrefix}{ProtocolKey}://{hostname}/{m_bucket}/"; - if (!string.IsNullOrWhiteSpace(m_prefix)) - m_sourcePathKey += m_prefix; - m_sourceProvider = new BackendSourceProvider(this); } public static bool IsValidHostname(string bucketname) @@ -422,14 +408,21 @@ namespace Duplicati.Library.Backend } /// - public string PathKey => m_sourcePathKey; + public IAsyncEnumerable ListAsync(string path, CancellationToken cancellationToken) + { + var prefix = m_prefix; + if (!string.IsNullOrWhiteSpace(prefix)) + prefix = Util.AppendDirSeparator(prefix); + + var filterPath = prefix + path; + if (!string.IsNullOrWhiteSpace(filterPath)) + filterPath = Util.AppendDirSeparator(filterPath); + + return s3Client.ListBucketAsync(m_bucket, filterPath, m_recurseLists, cancellationToken); + } /// - public IAsyncEnumerable ListAsync(string path, CancellationToken cancellationToken) - => m_sourceProvider.ListFromFileEntryAsync(m_prefix, path, (filter, token) => s3Client.ListBucketAsync(m_bucket, filter, m_recurseLists, token), cancellationToken); - - /// - public Task GetEntryAsync(string path, CancellationToken cancellationToken) - => Task.FromResult(null); + public Task GetEntryAsync(string path, CancellationToken cancellationToken) + => Task.FromResult(null); } } diff --git a/Duplicati/Library/Backend/S3/S3MinioClient.cs b/Duplicati/Library/Backend/S3/S3MinioClient.cs index e48c7b301..3f7d48f1c 100644 --- a/Duplicati/Library/Backend/S3/S3MinioClient.cs +++ b/Duplicati/Library/Backend/S3/S3MinioClient.cs @@ -26,6 +26,7 @@ using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Channels; using System.Threading.Tasks; +using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using Duplicati.Library.Utility; using Minio; @@ -85,11 +86,17 @@ namespace Duplicati.Library.Backend { ThrowExceptionIfBucketDoesNotExist(bucketName); + if (!string.IsNullOrWhiteSpace(prefix)) + prefix = Util.AppendDirSeparator(prefix, "/"); + var observable = m_client.ListObjectsAsync(bucketName, prefix, recursive, cancellationToken); await foreach (var obj in ToAsyncEnumerable(observable, cancellationToken).ConfigureAwait(false)) { - yield return new Common.IO.FileEntry( - obj.Key, + if (obj.Key == prefix || !obj.Key.StartsWith(prefix)) + continue; + + yield return new FileEntry( + obj.Key.Substring(prefix.Length), (long)obj.Size, Convert.ToDateTime(obj.LastModified), Convert.ToDateTime(obj.LastModified) diff --git a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs index 8f3c1e846..fa31e6c6f 100644 --- a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs +++ b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs @@ -60,15 +60,6 @@ namespace Duplicati.Library.Backend private readonly bool m_relativePath; private string m_initialDirectory; - /// - /// The source provider for this backend - /// - private readonly BackendSourceProvider m_sourceProvider; - /// - /// The path key for this backend if used as a source - /// - private readonly string m_sourcePathKey; - private SftpClient m_con; public SSHv2() @@ -128,11 +119,6 @@ namespace Duplicati.Library.Backend if (!string.IsNullOrWhiteSpace(timeoutstr)) m_keepaliveinterval = Library.Utility.Timeparser.ParseTimeSpan(timeoutstr); - - m_sourcePathKey = $"{Util.RemotePathPrefix}{ProtocolKey}://{m_server}:{m_port}/~{m_username}/"; - if (string.IsNullOrWhiteSpace(m_path)) - m_sourcePathKey += $"{m_path}/"; - m_sourceProvider = new BackendSourceProvider(this); } #region IBackend Members @@ -467,10 +453,7 @@ namespace Duplicati.Library.Backend public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new[] { m_server }); /// - public string PathKey => m_sourcePathKey; - - /// - public IAsyncEnumerable ListAsync(string path, CancellationToken cancellationToken) + public IAsyncEnumerable ListAsync(string path, CancellationToken cancellationToken) { CreateConnection(); SetWorkingDirectory(); @@ -479,7 +462,7 @@ namespace Duplicati.Library.Backend if (!string.IsNullOrWhiteSpace(prefixPath)) prefixPath = Util.AppendDirSeparator(prefixPath, "/"); - var filterPath = prefixPath + path; + var filterPath = prefixPath + BackendSourceFileEntry.NormalizePathTo(path, '/'); if (!string.IsNullOrWhiteSpace(filterPath)) filterPath = Util.AppendDirSeparator(filterPath, "/"); @@ -487,19 +470,18 @@ namespace Duplicati.Library.Backend filterPath = "."; return Client.ListDirectoryAsync(filterPath, cancellationToken) - .Select(x => new BackendSourceFileEntry( - m_sourceProvider, - BackendSourceFileEntry.ConcatPaths(path, x.Name), - x.Attributes.IsDirectory, - false, - new DateTime(0), - x.Attributes.LastWriteTimeUtc, - x.Attributes.Size - )); + .Select(x => new FileEntry( + x.Name, + x.Attributes.Size, + x.Attributes.LastAccessTimeUtc, + x.Attributes.LastWriteTimeUtc) + { + IsFolder = x.Attributes.IsDirectory + }); } /// - public Task GetEntryAsync(string path, CancellationToken cancellationToken) - => Task.FromResult(null); + public Task GetEntryAsync(string path, CancellationToken cancellationToken) + => Task.FromResult(null); } } \ No newline at end of file diff --git a/Duplicati/Library/Common/IO/Util.cs b/Duplicati/Library/Common/IO/Util.cs index f982b7568..80b247cf6 100644 --- a/Duplicati/Library/Common/IO/Util.cs +++ b/Duplicati/Library/Common/IO/Util.cs @@ -36,11 +36,6 @@ namespace Duplicati.Library.Common.IO /// public static readonly string AltDirectorySeparatorString = Path.AltDirectorySeparatorChar.ToString(); - /// - /// The prefix marker for a remote path - /// - public const string RemotePathPrefix = ">"; - /// /// Filename of a marker file that can be put inside the data folder to prevent Duplicati from fixing lax permissions /// @@ -77,8 +72,6 @@ namespace Duplicati.Library.Common.IO /// The guessed directory separator public static string GuessDirSeparator(string path) { - if (path != null && path.StartsWith(RemotePathPrefix, StringComparison.Ordinal)) - throw new ArgumentException("Path cannot start with @, most likely this is a remote path", nameof(path)); return string.IsNullOrWhiteSpace(path) || path.StartsWith("/", StringComparison.Ordinal) ? "/" : "\\"; } diff --git a/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs b/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs index f73022611..b5e7819a6 100644 --- a/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs +++ b/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs @@ -65,7 +65,7 @@ namespace Duplicati.Library.DynamicLoader /// The url to create the instance for /// The options to pass to the instance constructor /// The instanciated SourceProvider or null if the url is not supported - public ISourceProviderModule GetSourceProvider(string url, Dictionary options) + public ISourceProviderModule GetSourceProvider(string url, string mountPoint, Dictionary options) { var uri = new Utility.Uri(url); @@ -80,7 +80,7 @@ namespace Duplicati.Library.DynamicLoader try { if (m_interfaces.ContainsKey(uri.Scheme)) - return (ISourceProviderModule)Activator.CreateInstance(m_interfaces[uri.Scheme].GetType(), url, newOpts); + return (ISourceProviderModule)Activator.CreateInstance(m_interfaces[uri.Scheme].GetType(), url, mountPoint, newOpts); } catch (System.Reflection.TargetInvocationException tex) { @@ -161,19 +161,20 @@ namespace Duplicati.Library.DynamicLoader /// Instanciates a specific SourceProvider, given the url and options /// /// The url to create the instance for + /// The mount point to use /// The options to pass to the instance constructor /// The instanciated SourceProvider or null if the url is not supported - public static ISourceProviderModule GetSourceProvider(string url, Dictionary options) + public static ISourceProviderModule GetSourceProvider(string url, string mountPoint, Dictionary options) { // Source providers are preferred over backends - var provider = _SourceProviderLoader.GetSourceProvider(url, options); + var provider = _SourceProviderLoader.GetSourceProvider(url, mountPoint, options); if (provider != null) return provider; // See if there is a backend that can also be a source var backend = BackendLoader.GetBackend(url, options); if (backend is IFolderEnabledBackend folderBackend) - return new BackendSourceProvider(folderBackend); + return new BackendSourceProvider(folderBackend, mountPoint); backend?.Dispose(); return null; diff --git a/Duplicati/Library/Interface/IDestinationProvider.cs b/Duplicati/Library/Interface/IDestinationProvider.cs new file mode 100644 index 000000000..20467a0b5 --- /dev/null +++ b/Duplicati/Library/Interface/IDestinationProvider.cs @@ -0,0 +1,43 @@ +#nullable enable + +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace Duplicati.Library.Interface +{ + /// + /// Implements a destination where data can be restored to + /// + public interface IDestinationProvider : IDynamicModule + { + /// + /// Gets the module key + /// + string Key { get; } + /// + /// Gets the display name of the module + /// + string DisplayName { get; } + + /// + /// Checks if a restore is needed + /// + /// The path to restore to + /// The hash of the data to restore + /// The hash of the metadata to restore + /// The cancellation token + /// True if a restore is needed, false otherwise + Task IsRestoreNeededAsync(string path, string dataHash, string? metadataHash, CancellationToken cancel); + + /// + /// Restores a file to the given path + /// + /// The path to restore to + /// The data stream to restore + /// The metadata stream to restore + /// The cancellation token + /// A task that completes when the restore is done + Task WriteAsync(string path, Stream data, Stream? metadata, CancellationToken cancel); + } +} \ No newline at end of file diff --git a/Duplicati/Library/Interface/IFolderEnabledBackend.cs b/Duplicati/Library/Interface/IFolderEnabledBackend.cs index 4b9c6047a..6f9374aed 100644 --- a/Duplicati/Library/Interface/IFolderEnabledBackend.cs +++ b/Duplicati/Library/Interface/IFolderEnabledBackend.cs @@ -34,11 +34,6 @@ namespace Duplicati.Library.Interface; /// public interface IFolderEnabledBackend : IBackend { - /// - /// Returns the key for this backend instance's path, should be prefixed with a "@" - /// - string PathKey { get; } - /// /// Enumerates a list of files and folders found on the remote location /// @@ -46,11 +41,10 @@ public interface IFolderEnabledBackend : IBackend /// The cancellation token /// The list of files, where paths are scoped to the backend. /// - /// The paths returned should be relative to the backend root, and should not include the root path. - /// For example, if the backend is defined as proto://host/rootpath, then listing files should - /// return paths like "/subfolder/file.txt" and not "/rootpath/subfolder/file.txt". + /// The paths returned should be only the filenames or folder names, not the full path. + /// Folders should end with the operating system's directory separator. /// - IAsyncEnumerable ListAsync(string? path, CancellationToken cancellationToken); + IAsyncEnumerable ListAsync(string? path, CancellationToken cancellationToken); /// /// Gets a single file or folder entry @@ -58,6 +52,10 @@ public interface IFolderEnabledBackend : IBackend /// The path to get /// The cancellation token /// The file or folder entry - Task GetEntryAsync(string path, CancellationToken cancellationToken); + /// + /// The paths returned should be only the filenames or folder names, not the full path. + /// Folders should end with the operating system's directory separator. + /// + Task GetEntryAsync(string path, CancellationToken cancellationToken); } diff --git a/Duplicati/Library/Interface/ISourceProvider.cs b/Duplicati/Library/Interface/ISourceProvider.cs index 11c61d4a4..386680f00 100644 --- a/Duplicati/Library/Interface/ISourceProvider.cs +++ b/Duplicati/Library/Interface/ISourceProvider.cs @@ -34,16 +34,16 @@ namespace Duplicati.Library.Interface; public interface ISourceProvider : IDisposable { /// - /// Returns the key for this provider instance's path + /// Returns the path where this provider is logically mounted /// - string PathKey { get; } + string MountedPath { get; } /// /// Gets the root entries /// /// The cancellation token /// An enumerable of file entries - IAsyncEnumerable Enumerate(CancellationToken cancellationToken); + IAsyncEnumerable Enumerate(CancellationToken cancellationToken); /// /// Gets a specific entry @@ -52,5 +52,5 @@ public interface ISourceProvider : IDisposable /// True if the path is a folder /// The cancellation token /// The file entry - Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken); + Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken); } diff --git a/Duplicati/Library/Interface/ISourceFileEntry.cs b/Duplicati/Library/Interface/ISourceProviderEntry.cs similarity index 96% rename from Duplicati/Library/Interface/ISourceFileEntry.cs rename to Duplicati/Library/Interface/ISourceProviderEntry.cs index 0f41fb690..c93d945e2 100644 --- a/Duplicati/Library/Interface/ISourceFileEntry.cs +++ b/Duplicati/Library/Interface/ISourceProviderEntry.cs @@ -30,9 +30,9 @@ using System.Threading.Tasks; namespace Duplicati.Library.Interface; /// -/// Interface for an instance of a file +/// Interface for an instance of a file or folder /// -public interface ISourceFileEntry +public interface ISourceProviderEntry { /// /// True if the entry represents a folder, false otherwise @@ -122,5 +122,5 @@ public interface ISourceFileEntry /// /// A token that can be used to cancel the operation /// The contents of the folder - IAsyncEnumerable Enumerate(CancellationToken cancellationToken); + IAsyncEnumerable Enumerate(CancellationToken cancellationToken); } diff --git a/Duplicati/Library/Main/Controller.cs b/Duplicati/Library/Main/Controller.cs index 312df9f8f..abd3b311e 100644 --- a/Duplicati/Library/Main/Controller.cs +++ b/Duplicati/Library/Main/Controller.cs @@ -948,15 +948,14 @@ namespace Duplicati.Library.Main string source; try { - // Check if this is not a file source - var scheme = Library.Utility.Utility.GuessScheme(expandedSource) ?? "file"; - if (scheme != "file") + // Check if this is a mounted path + if (expandedSource.StartsWith("@")) { // TODO: If the remote source fails to load, // this will be an enumeration warning, but will result // in the backup being recorded without files from the source - // Eventually, this could lead to retention causing the last backup - // with the data from the source to be deleted + // Eventually, this could lead to retention deletion, + // causing the last backup with the data from the source to be deleted foundAnyPaths = true; sources.Add(expandedSource); continue; diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index f558c6e8a..6417344d0 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -878,11 +878,11 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist } } - public string GetFirstLocalPath() + public string GetFirstPath() { using (var cmd = m_connection.CreateCommand()) { - cmd.CommandText = @$"SELECT ""Path"" FROM ""File"" WHERE ""Path"" NOT LIKE '{Util.RemotePathPrefix}%' ORDER BY LENGTH(""Path"") DESC LIMIT 1"; + cmd.CommandText = @$"SELECT ""Path"" FROM ""File"" ORDER BY LENGTH(""Path"") DESC LIMIT 1"; var v0 = cmd.ExecuteScalar(); if (v0 == null || v0 == DBNull.Value) return null; diff --git a/Duplicati/Library/Main/Operation/Backup/Channels.cs b/Duplicati/Library/Main/Operation/Backup/Channels.cs index b5948e13e..f65c545d7 100644 --- a/Duplicati/Library/Main/Operation/Backup/Channels.cs +++ b/Duplicati/Library/Main/Operation/Backup/Channels.cs @@ -52,7 +52,7 @@ namespace Duplicati.Library.Main.Operation.Backup /// /// When enumerating the source folders, all discovered paths are sent by the to the /// - public readonly IChannel SourcePaths = ChannelManager.CreateChannel(); + public readonly IChannel SourcePaths = ChannelManager.CreateChannel(); /// /// All progress events are communicated over this channel, to ensure that parallel progress is reported as a if it was sequential /// diff --git a/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs b/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs index a181907ff..848610186 100644 --- a/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs @@ -80,7 +80,7 @@ namespace Duplicati.Library.Main.Operation.Backup // The mixin queue is used to store symlinks that should be processed // The symlinks are emitted during the enumeration process when they are found - var mixinqueue = new Queue(); + var mixinqueue = new Queue(); // The enumeration filter is used to determine what paths to // recurse into. If the emit filter only has includes, @@ -98,16 +98,16 @@ namespace Duplicati.Library.Main.Operation.Backup // Shared filter function with bound variables - ValueTask FilterEntry(ISourceFileEntry entry) + ValueTask FilterEntry(ISourceProviderEntry entry) => SourceFileEntryFilter(entry, blacklistPaths, hardlinkPolicy, symlinkPolicy, hardlinkmap, fileAttributeFilter, enumeratefilter, ignorenames, mixinqueue, token); // Prepare the work list - IAsyncEnumerable worklist; + IAsyncEnumerable worklist; // If we have a specific list, use that instead of enumerating the filesystem if (changedfilelist != null && changedfilelist.Any()) { - async IAsyncEnumerable ExpandSources(IEnumerable list) + async IAsyncEnumerable ExpandSources(IEnumerable list) { foreach (var s in list) { @@ -203,7 +203,7 @@ namespace Duplicati.Library.Main.Operation.Backup /// /// The item being tracked /// - public required ISourceFileEntry Item; + public required ISourceProviderEntry Item; /// /// A flag indicating if any items are found in this folder /// @@ -216,7 +216,7 @@ namespace Duplicati.Library.Main.Operation.Backup /// /// The list without empty folders. /// The list with potential empty folders. - private static async IAsyncEnumerable ExcludeEmptyFolders(IAsyncEnumerable source, [EnumeratorCancellation] CancellationToken cancellationToken) + private static async IAsyncEnumerable ExcludeEmptyFolders(IAsyncEnumerable source, [EnumeratorCancellation] CancellationToken cancellationToken) { var pathstack = new Stack(); @@ -276,9 +276,9 @@ namespace Duplicati.Library.Main.Operation.Backup /// The entries to recurse /// The filter to apply /// - private static async IAsyncEnumerable RecurseEntries(IAsyncEnumerable entries, Func> filter, [EnumeratorCancellation] CancellationToken cancellationToken) + private static async IAsyncEnumerable RecurseEntries(IAsyncEnumerable entries, Func> filter, [EnumeratorCancellation] CancellationToken cancellationToken) { - var work = new Stack(); + var work = new Stack(); await foreach (var e in entries.WithCancellation(cancellationToken).ConfigureAwait(false)) if (await filter(e).ConfigureAwait(false)) @@ -317,7 +317,7 @@ namespace Duplicati.Library.Main.Operation.Backup /// The mix in queue. /// The emitfilter. /// The enumeratefilter. - private static async IAsyncEnumerable ExpandWorkList(IAsyncEnumerable worklist, Queue mixinqueue, Library.Utility.IFilter emitfilter, Library.Utility.IFilter enumeratefilter, [EnumeratorCancellation] CancellationToken cancellationToken) + private static async IAsyncEnumerable ExpandWorkList(IAsyncEnumerable worklist, Queue mixinqueue, Library.Utility.IFilter emitfilter, Library.Utility.IFilter enumeratefilter, [EnumeratorCancellation] CancellationToken cancellationToken) { // Process each path, and dequeue the mixins with symlinks as we go await foreach (var s in worklist.WithCancellation(cancellationToken).ConfigureAwait(false)) @@ -345,7 +345,7 @@ namespace Duplicati.Library.Main.Operation.Backup /// The entry to evaluate. /// The blacklist paths. /// True if the path should be returned, false otherwise. - private static bool PreFilterSourceEntry(ISourceFileEntry entry, HashSet blacklistPaths) + private static bool PreFilterSourceEntry(ISourceProviderEntry entry, HashSet blacklistPaths) { // Don't filter meta stuff if (entry.IsMetaEntry) @@ -405,7 +405,7 @@ namespace Duplicati.Library.Main.Operation.Backup /// The ignore names. /// The mixin queue. /// True if the path should be returned, false otherwise. - private static async ValueTask SourceFileEntryFilter(ISourceFileEntry entry, HashSet blacklistPaths, Options.HardlinkStrategy hardlinkPolicy, Options.SymlinkStrategy symlinkPolicy, Dictionary hardlinkmap, FileAttributes fileAttributeFilter, Duplicati.Library.Utility.IFilter enumeratefilter, string[]? ignorenames, Queue mixinqueue, CancellationToken cancellationToken) + private static async ValueTask SourceFileEntryFilter(ISourceProviderEntry entry, HashSet blacklistPaths, Options.HardlinkStrategy hardlinkPolicy, Options.SymlinkStrategy symlinkPolicy, Dictionary hardlinkmap, FileAttributes fileAttributeFilter, Duplicati.Library.Utility.IFilter enumeratefilter, string[]? ignorenames, Queue mixinqueue, CancellationToken cancellationToken) { // Do the course pre-filtering first if (!PreFilterSourceEntry(entry, blacklistPaths)) diff --git a/Duplicati/Library/Main/Operation/Backup/MetadataGenerator.cs b/Duplicati/Library/Main/Operation/Backup/MetadataGenerator.cs index 02c6f75c2..6be52d47c 100644 --- a/Duplicati/Library/Main/Operation/Backup/MetadataGenerator.cs +++ b/Duplicati/Library/Main/Operation/Backup/MetadataGenerator.cs @@ -32,7 +32,7 @@ namespace Duplicati.Library.Main.Operation.Backup { private static readonly string METALOGTAG = Logging.Log.LogTagFromType(typeof(MetadataGenerator)) + ".Metadata"; - public static Dictionary GenerateMetadata(ISourceFileEntry entry, System.IO.FileAttributes attributes, Options options) + public static Dictionary GenerateMetadata(ISourceProviderEntry entry, System.IO.FileAttributes attributes, Options options) { try { diff --git a/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs b/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs index f8761ed56..2ae10faa1 100644 --- a/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs @@ -40,7 +40,7 @@ namespace Duplicati.Library.Main.Operation.Backup public class FileEntry { // From input - public ISourceFileEntry Entry; + public ISourceProviderEntry Entry; // Split public long PathPrefixID; @@ -180,7 +180,7 @@ namespace Duplicati.Library.Main.Operation.Backup /// Processes the metadata for the given path. /// /// True if the path should be submitted to more analysis, false if there is nothing else to do - private static async Task ProcessMetadata(ISourceFileEntry entry, FileAttributes attributes, DateTime lastwrite, Options options, IMetahash emptymetadata, BackupDatabase database, IWriteChannel streamblockchannel) + private static async Task ProcessMetadata(ISourceProviderEntry entry, FileAttributes attributes, DateTime lastwrite, Options options, IMetahash emptymetadata, BackupDatabase database, IWriteChannel streamblockchannel) { if (entry.IsSymlink) { diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index de97b7449..a9786b2d8 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -37,6 +37,8 @@ using System.Data; using Duplicati.Library.DynamicLoader; using Duplicati.Library.SourceProvider; using Duplicati.Library.Snapshots.USN; +using System.Text.RegularExpressions; +using System.IO; namespace Duplicati.Library.Main.Operation { @@ -123,7 +125,7 @@ namespace Duplicati.Library.Main.Operation private static List GetSourceProviders(IEnumerable sources, Options options) { // Group the sources by their type, so we can combine all snapshot paths into a single snapshot - var sourceTypes = sources.GroupBy(x => Library.Utility.Utility.GuessScheme(x) ?? "file", StringComparer.OrdinalIgnoreCase); + var sourceTypes = sources.GroupBy(x => x.StartsWith("@") ? "@" : Library.Utility.Utility.GuessScheme(x) ?? "file", StringComparer.OrdinalIgnoreCase); // To avoid leaking snapshot instances, we create all instances first and then dispose them if an exception occurs // The number of instances is expected to be low, so the memory overhead is acceptable @@ -136,13 +138,30 @@ namespace Duplicati.Library.Main.Operation results.Add(new LocalFileSource(GetFileSnapshotService(entry, options))); else if ("vss".Equals(entry.Key, StringComparison.OrdinalIgnoreCase) || "lvm".Equals(entry.Key, StringComparison.OrdinalIgnoreCase)) results.Add(new LocalFileSource(SnapshotUtility.CreateSnapshot(entry, options.RawOptions, options.SymlinkPolicy == Options.SymlinkStrategy.Follow))); - else + else if ("@".Equals(entry.Key, StringComparison.OrdinalIgnoreCase)) { - // Assume it is a remote source foreach (var url in entry) - results.Add(SourceProviderLoader.GetSourceProvider(url, options.RawOptions) - ?? throw new UserInformationException(string.Format("The source \"{0}\" is not supported", url), "SourceNotSupported")); + { + var m = Regex.Match(url, @"^@(?[^|]+)\|(?.+)$", RegexOptions.IgnoreCase); + if (m.Success) + { + var mountpoint = m.Groups["mountpoint"].Value; + + if (mountpoint.Any(x => Path.GetInvalidPathChars().Contains(x))) + throw new UserInformationException(string.Format("The mountpoint \"{0}\" contains invalid characters", mountpoint), "InvalidMountpoint"); + if (!Path.IsPathRooted(mountpoint)) + throw new UserInformationException(string.Format("The mountpoint \"{0}\" is not a valid rooted mountpoint", mountpoint), "InvalidMountpoint"); + + var backendurl = m.Groups["url"].Value; + results.Add(SourceProviderLoader.GetSourceProvider(backendurl, Path.GetFullPath(mountpoint), options.RawOptions) + ?? throw new UserInformationException(string.Format("The source \"{0}\" is not supported", backendurl), "SourceNotSupported")); + } + else + throw new UserInformationException(string.Format("The source \"{0}\" is not a supported format", url), "SourceFormatNotSupported"); + } } + else + throw new UserInformationException(string.Format("The source type \"{0}\" is not supported", entry.Key), "SourceTypeNotSupported"); } } catch @@ -269,7 +288,7 @@ namespace Duplicati.Library.Main.Operation Utility.UpdateOptionsFromDb(database, options); Utility.VerifyOptionsAndUpdateDatabase(database, options); - var probe_path = database.GetFirstLocalPath(); + var probe_path = database.GetFirstPath(); if (probe_path != null && Util.GuessDirSeparator(probe_path) != Util.DirectorySeparatorString) throw new UserInformationException(string.Format("The backup contains files that belong to another operating system. Proceeding with a backup would cause the database to contain paths from two different operation systems, which is not supported. To proceed without losing remote data, delete all filesets and make sure the --{0} option is set, then run the backup again to re-use the existing data on the remote store.", "no-auto-compact"), "CrossOsDatabaseReuseNotSupported"); diff --git a/Duplicati/Library/Snapshots/ISnapshotService.cs b/Duplicati/Library/Snapshots/ISnapshotService.cs index 72542e465..bebf5c92a 100644 --- a/Duplicati/Library/Snapshots/ISnapshotService.cs +++ b/Duplicati/Library/Snapshots/ISnapshotService.cs @@ -44,20 +44,20 @@ namespace Duplicati.Library.Snapshots /// Enumerates the root source files and folders /// /// The source files and folders - IEnumerable EnumerateFilesystemEntries(); + IEnumerable EnumerateFilesystemEntries(); /// /// Enumerates the files and folders in a given folder /// /// The parent folder to enumerate /// The files and folders in the parent folder - IEnumerable EnumerateFilesystemEntries(ISourceFileEntry parent); + IEnumerable EnumerateFilesystemEntries(ISourceProviderEntry parent); /// /// Gets a filesystem entry for a given path /// /// The path to get /// True if the path is a folder /// The filesystem entry - ISourceFileEntry? GetFilesystemEntry(string path, bool isFolder); + ISourceProviderEntry? GetFilesystemEntry(string path, bool isFolder); /// /// Gets the last write time of a given file in UTC diff --git a/Duplicati/Library/Snapshots/LinuxSnapshot.cs b/Duplicati/Library/Snapshots/LinuxSnapshot.cs index 05cf884d7..83b786ddc 100644 --- a/Duplicati/Library/Snapshots/LinuxSnapshot.cs +++ b/Duplicati/Library/Snapshots/LinuxSnapshot.cs @@ -116,7 +116,7 @@ namespace Duplicati.Library.Snapshots /// Enumerates the root source files and folders /// /// The source files and folders - public override IEnumerable EnumerateFilesystemEntries() + public override IEnumerable EnumerateFilesystemEntries() { foreach (var sourceEntry in SourceEntries) { diff --git a/Duplicati/Library/Snapshots/NoSnapshotLinux.cs b/Duplicati/Library/Snapshots/NoSnapshotLinux.cs index 6aaa1ecd6..de21cec98 100644 --- a/Duplicati/Library/Snapshots/NoSnapshotLinux.cs +++ b/Duplicati/Library/Snapshots/NoSnapshotLinux.cs @@ -93,7 +93,7 @@ namespace Duplicati.Library.Snapshots /// Enumerates the root source files and folders /// /// The source files and folders - public override IEnumerable EnumerateFilesystemEntries() + public override IEnumerable EnumerateFilesystemEntries() { foreach (var folder in m_sources) { diff --git a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs index 029ae3791..aa8c9a5ac 100644 --- a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs +++ b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs @@ -84,7 +84,7 @@ namespace Duplicati.Library.Snapshots /// Enumerates the root source files and folders /// /// The source files and folders - public override IEnumerable EnumerateFilesystemEntries() + public override IEnumerable EnumerateFilesystemEntries() { foreach (var folder in m_sources.Select(SystemIOWindows.RemoveExtendedDevicePathPrefix)) { diff --git a/Duplicati/Library/Snapshots/SnapshotBase.cs b/Duplicati/Library/Snapshots/SnapshotBase.cs index 1d8f5a253..0bacef4af 100644 --- a/Duplicati/Library/Snapshots/SnapshotBase.cs +++ b/Duplicati/Library/Snapshots/SnapshotBase.cs @@ -64,7 +64,7 @@ namespace Duplicati.Library.Snapshots /// Enuemrates the root source files and folders /// /// The source files and folders - public abstract IEnumerable EnumerateFilesystemEntries(); + public abstract IEnumerable EnumerateFilesystemEntries(); /// /// Gets a filesystem entry for a given path @@ -72,7 +72,7 @@ namespace Duplicati.Library.Snapshots /// The path to get the entry for /// A flag indicating if the path is a folder /// The filesystem entry - public virtual ISourceFileEntry GetFilesystemEntry(string path, bool isFolder) + public virtual ISourceProviderEntry GetFilesystemEntry(string path, bool isFolder) { if (isFolder && !DirectoryExists(path)) return null; @@ -87,7 +87,7 @@ namespace Duplicati.Library.Snapshots /// /// Source to enumerate /// The files and folders in the given folder - public virtual IEnumerable EnumerateFilesystemEntries(ISourceFileEntry source) + public virtual IEnumerable EnumerateFilesystemEntries(ISourceProviderEntry source) { if (source.IsFolder) { diff --git a/Duplicati/Library/Snapshots/SnapshotSourceFileEntry.cs b/Duplicati/Library/Snapshots/SnapshotSourceFileEntry.cs index 932963611..7fc9cae22 100644 --- a/Duplicati/Library/Snapshots/SnapshotSourceFileEntry.cs +++ b/Duplicati/Library/Snapshots/SnapshotSourceFileEntry.cs @@ -39,7 +39,7 @@ namespace Duplicati.Library.Snapshots; /// The path of the file /// True if the entry is a folder, false otherwise /// True if the entry is the root entry, false otherwise -public class SnapshotSourceFileEntry(ISnapshotService service, string path, bool isFolder, bool isRoot) : ISourceFileEntry +public class SnapshotSourceFileEntry(ISnapshotService service, string path, bool isFolder, bool isRoot) : ISourceProviderEntry { /// /// The symlink target @@ -137,7 +137,7 @@ public class SnapshotSourceFileEntry(ISnapshotService service, string path, bool /// public Task OpenMetadataRead(CancellationToken cancellationToken) => Task.FromResult(new MemoryStream(System.Text.Json.JsonSerializer.SerializeToUtf8Bytes(MinorMetadata))); /// - public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) => service.EnumerateFilesystemEntries(this).ToAsyncEnumerable(); + public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) => service.EnumerateFilesystemEntries(this).ToAsyncEnumerable(); /// public Task FileExists(string filename, CancellationToken cancellationToken) => Task.FromResult(service.FileExists(System.IO.Path.Combine(path, filename))); } \ No newline at end of file diff --git a/Duplicati/Library/Snapshots/USN/UsnJournalService.cs b/Duplicati/Library/Snapshots/USN/UsnJournalService.cs index d44bad907..cc5963199 100644 --- a/Duplicati/Library/Snapshots/USN/UsnJournalService.cs +++ b/Duplicati/Library/Snapshots/USN/UsnJournalService.cs @@ -229,7 +229,7 @@ namespace Duplicati.Library.Snapshots.USN /// /// Cancellation token /// Filtered sources - public async IAsyncEnumerable GetFullScanSources([EnumeratorCancellation] CancellationToken token) + public async IAsyncEnumerable GetFullScanSources([EnumeratorCancellation] CancellationToken token) { // Make the method async enumerable await Task.CompletedTask; @@ -276,7 +276,7 @@ namespace Duplicati.Library.Snapshots.USN /// Filter callback to exclude filtered items /// Cancellation token /// Filtered sources - public async IAsyncEnumerable GetModifiedSources(Func> filter, [EnumeratorCancellation] CancellationToken token) + public async IAsyncEnumerable GetModifiedSources(Func> filter, [EnumeratorCancellation] CancellationToken token) { foreach (var volumeData in m_volumeDataDict.Values.Where(x => !x.IsFullScan)) { @@ -338,10 +338,10 @@ namespace Duplicati.Library.Snapshots.USN /// Cache of included and excluded files / folders /// Cancellation token /// Filtered files - private static async IAsyncEnumerable FilterExcludedFiles( + private static async IAsyncEnumerable FilterExcludedFiles( IEnumerable? files, ISnapshotService snapshot, - Func> filter, + Func> filter, IDictionary cache, [EnumeratorCancellation] CancellationToken token) { @@ -350,7 +350,7 @@ namespace Duplicati.Library.Snapshots.USN foreach (var filePath in files) { - ISourceFileEntry? file; + ISourceProviderEntry? file; try { file = snapshot.GetFilesystemEntry(filePath, false); @@ -392,10 +392,10 @@ namespace Duplicati.Library.Snapshots.USN /// Cache of excluded folders /// Cancellation token /// Filtered folders - private static async IAsyncEnumerable FilterExcludedFolders( + private static async IAsyncEnumerable FilterExcludedFolders( IEnumerable? folders, ISnapshotService snapshot, - Func> filter, + Func> filter, IDictionary cache, [EnumeratorCancellation] CancellationToken token) { @@ -404,7 +404,7 @@ namespace Duplicati.Library.Snapshots.USN foreach (var folderPath in folders) { - ISourceFileEntry? folder; + ISourceProviderEntry? folder; try { folder = snapshot.GetFilesystemEntry(folderPath, true); @@ -431,14 +431,14 @@ namespace Duplicati.Library.Snapshots.USN /// Cache of excluded folders (optional) /// True if excluded, false otherwise private static async ValueTask IsFolderOrAncestorsExcluded( - ISourceFileEntry inputFolder, + ISourceProviderEntry inputFolder, ISnapshotService snapshot, - Func> filter, + Func> filter, IDictionary cache, CancellationToken token) { List? parents = null; - ISourceFileEntry? folder = inputFolder; + ISourceProviderEntry? folder = inputFolder; while (folder != null) { diff --git a/Duplicati/Library/Snapshots/WindowsSnapshot.cs b/Duplicati/Library/Snapshots/WindowsSnapshot.cs index 1c1e70416..bbe4154c3 100644 --- a/Duplicati/Library/Snapshots/WindowsSnapshot.cs +++ b/Duplicati/Library/Snapshots/WindowsSnapshot.cs @@ -126,7 +126,7 @@ namespace Duplicati.Library.Snapshots /// Enumerates the root source files and folders /// /// The source files and folders - public override IEnumerable EnumerateFilesystemEntries() + public override IEnumerable EnumerateFilesystemEntries() { foreach (var folder in _sourceEntries) { diff --git a/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs b/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs index 535ec7d5d..95fe59f91 100644 --- a/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs +++ b/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs @@ -20,6 +20,7 @@ // DEALINGS IN THE SOFTWARE. using System.IO.Pipelines; +using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using Duplicati.Library.Utility; @@ -36,7 +37,7 @@ namespace Duplicati.Library.SourceProvider; /// The last modification time of the entry /// The size of the entry public class BackendSourceFileEntry(BackendSourceProvider parent, string path, bool isFolder, bool isMetaEntry, DateTime createdUtc, DateTime lastModificationUtc, long size) - : ISourceFileEntry + : ISourceProviderEntry { /// /// The log tag for this instance @@ -59,7 +60,7 @@ public class BackendSourceFileEntry(BackendSourceProvider parent, string path, b public DateTime LastModificationUtc => lastModificationUtc; /// - public string Path => ConcatPaths(parent.PathKey, NormalizePath(path)); + public string Path => SystemIO.IO_OS.PathCombine(parent?.MountedPath, NormalizePathToLocalSystem(path)); /// public long Size => size; @@ -90,18 +91,32 @@ public class BackendSourceFileEntry(BackendSourceProvider parent, string path, b public string? HardlinkTargetId => null; /// - public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) + public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) { if (!isFolder) throw new InvalidOperationException("Enumerate can only be called on folders"); return parent.WrappedBackend.ListAsync(path, cancellationToken) + // Remove the current and parent folder entries + .Where(x => !string.IsNullOrWhiteSpace(x.Name) && x.Name != "." && x.Name != "..") + // Remove sub-folder entries + .Where(x => !x.Name[0..^1].Contains('\\') && !x.Name[0..^1].Contains('/')) + // Convert to source file entries .Select(x => { - if (x is BackendSourceFileEntry) - return x; + var localPath = SystemIO.IO_OS.PathCombine(path, NormalizePathToLocalSystem(x.Name)); + if (x.IsFolder) + localPath = Util.AppendDirSeparator(localPath); - return new BackendSourceFileEntry(parent, x.Path, x.IsFolder, false, x.CreatedUtc, x.LastModificationUtc, x.Size); + return new BackendSourceFileEntry( + parent, + localPath, + x.IsFolder, + false, + x.Created, + x.LastModification, + x.Size + ); }); } @@ -109,7 +124,7 @@ public class BackendSourceFileEntry(BackendSourceProvider parent, string path, b public async Task FileExists(string path, CancellationToken cancellationToken) { if (!isFolder) - throw new InvalidOperationException("FileExists can only be called on folders"); + throw new InvalidOperationException("FileExists cannot be called on folders"); try { @@ -235,39 +250,28 @@ public class BackendSourceFileEntry(BackendSourceProvider parent, string path, b } /// - /// Normalizes the path, turning backslashes into forward slashes + /// Normalizes the path, turning backslashes into forward slashes, + /// or vice versa, depending on the platform /// /// The path to normalize /// The normalized path - public static string NormalizePath(string path) - { - if (System.IO.Path.DirectorySeparatorChar == '/') - return path; + public static string NormalizePathToLocalSystem(string path) + => NormalizePathTo(path, System.IO.Path.DirectorySeparatorChar); + /// + /// Normalizes the path, turning backslashes into forward slashes, + /// or vice versa, depending on the platform + /// + /// The path to normalize + /// The normalized path + public static string NormalizePathTo(string path, char separator) + { if (string.IsNullOrEmpty(path)) return path; - return path.Replace('\\', '/'); - } - - /// - /// Concatenates two paths - /// - /// The first path - /// The second path - /// The concatenated path - public static string ConcatPaths(string path1, string path2) - { - if (string.IsNullOrEmpty(path1)) - return path2; - - if (string.IsNullOrEmpty(path2)) - return path1; - - if (path1.EndsWith('/') || path1.EndsWith('\\')) - return path1 + path2; - - return path1 + "/" + path2; + return path + .Replace('/', separator) + .Replace('\\', separator); } /// @@ -278,6 +282,6 @@ public class BackendSourceFileEntry(BackendSourceProvider parent, string path, b /// The prefix to add to the path /// The new backend source entry public static BackendSourceFileEntry FromFileEntry(BackendSourceProvider parent, string prefix, IFileEntry entry) - => new BackendSourceFileEntry(parent, ConcatPaths(prefix, entry.Name), entry.IsFolder, false, entry.Created, entry.LastModification, entry.Size); + => new BackendSourceFileEntry(parent, SystemIO.IO_OS.PathCombine(prefix, entry.Name), entry.IsFolder, false, entry.Created, entry.LastModification, entry.Size); } diff --git a/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs index 3943fb1b0..2ae3d9a71 100644 --- a/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs +++ b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs @@ -19,8 +19,6 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System.Runtime.CompilerServices; -using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; namespace Duplicati.Library.SourceProvider; @@ -29,7 +27,8 @@ namespace Duplicati.Library.SourceProvider; /// A source provider that wraps a backend /// /// The backend to wrap -public class BackendSourceProvider(IFolderEnabledBackend backend) : ISourceProvider, ISourceProviderModule +/// The path to mount the backend +public class BackendSourceProvider(IFolderEnabledBackend backend, string mountedPath) : ISourceProvider, ISourceProviderModule { /// /// The wrapped backend @@ -37,9 +36,9 @@ public class BackendSourceProvider(IFolderEnabledBackend backend) : ISourceProvi public IFolderEnabledBackend WrappedBackend => backend; /// - /// The path key unique for the destination + /// The path where the provider is logically mounted /// - public string PathKey => backend.PathKey; + public string MountedPath => mountedPath; /// public string Key => backend.ProtocolKey; @@ -51,12 +50,15 @@ public class BackendSourceProvider(IFolderEnabledBackend backend) : ISourceProvi public IList SupportedCommands => backend.SupportedCommands; /// - public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) + public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) => new[] { new BackendSourceFileEntry(this, "", true, true, new DateTime(0), new DateTime(0), 0) }.ToAsyncEnumerable(); /// - public Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) - => backend.GetEntryAsync(path, cancellationToken); + public async Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) + { + var entry = await backend.GetEntryAsync(path, cancellationToken).ConfigureAwait(false); + return entry == null ? null : BackendSourceFileEntry.FromFileEntry(this, path, entry); + } /// public void Dispose() @@ -64,40 +66,4 @@ public class BackendSourceProvider(IFolderEnabledBackend backend) : ISourceProvi WrappedBackend.Dispose(); GC.SuppressFinalize(this); } - - /// - /// Helper function to use a List function that returns IFileEntry and convert it to ISourceFileEntry - /// - /// The prefix to hide from the caller - /// The logical path to list - /// The function that returns the IFileEntry items - /// The cancellation token - /// The list of source file entries - public async IAsyncEnumerable ListFromFileEntryAsync(string prefix, string path, Func> listFunction, [EnumeratorCancellation] CancellationToken cancellationToken) - { - if (!string.IsNullOrWhiteSpace(prefix) && !prefix.EndsWith("/", StringComparison.Ordinal)) - prefix += "/"; - - // Logic here is that the prefix is invisible to the caller, so we add it to the prefix to filter the results - // and remove it from the name to make the prefix invisible. - // The input path is preserved so the output looks like a "full path" to the caller. - - var filterPath = prefix + path; - if (filterPath != "" && !filterPath.EndsWith("/", StringComparison.Ordinal)) - filterPath += "/"; - - await foreach (var f in listFunction(filterPath, cancellationToken).ConfigureAwait(false)) - { - if (!f.Name.StartsWith(filterPath, StringComparison.Ordinal)) - continue; - - ((FileEntry)f).Name = f.Name.Substring(filterPath.Length); - - // Skip self and sub-folder items - if (f.Name.Length == 0 || f.Name[0..^1].Contains("/")) - continue; - - yield return BackendSourceFileEntry.FromFileEntry(this, path, f); - } - } } diff --git a/Duplicati/Library/SourceProvider/Builtin/Combiner.cs b/Duplicati/Library/SourceProvider/Builtin/Combiner.cs index 7f027847f..ed886a028 100644 --- a/Duplicati/Library/SourceProvider/Builtin/Combiner.cs +++ b/Duplicati/Library/SourceProvider/Builtin/Combiner.cs @@ -30,7 +30,7 @@ namespace Duplicati.Library.SourceProvider; public class Combiner(IEnumerable providers) : ISourceProvider { /// - public string PathKey => string.Empty; + public string MountedPath => string.Empty; /// /// The providers to combine @@ -38,7 +38,7 @@ public class Combiner(IEnumerable providers) : ISourceProvider private readonly List providers = providers.SelectMany(x => x is Combiner c ? c.providers.AsEnumerable() : [x]).ToList(); /// - public async IAsyncEnumerable Enumerate([EnumeratorCancellation] CancellationToken cancellationToken) + public async IAsyncEnumerable Enumerate([EnumeratorCancellation] CancellationToken cancellationToken) { foreach (var provider in providers) { @@ -59,16 +59,16 @@ public class Combiner(IEnumerable providers) : ISourceProvider /// True if the path is a folder /// The cancellation token /// The filesystem entry - public async Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) + public async Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) { foreach (var provider in providers) - if (!string.IsNullOrWhiteSpace(PathKey) && provider.PathKey.StartsWith(path)) + if (!string.IsNullOrWhiteSpace(MountedPath) && provider.MountedPath.StartsWith(path)) { - var subpath = provider.PathKey.Substring(path.Length); + var subpath = provider.MountedPath.Substring(path.Length); return await provider.GetEntry(subpath, isFolder, cancellationToken).ConfigureAwait(false); } - foreach (var provider in providers.Where(x => string.IsNullOrWhiteSpace(x.PathKey))) + foreach (var provider in providers.Where(x => string.IsNullOrWhiteSpace(x.MountedPath))) { var res = await provider.GetEntry(path, isFolder, cancellationToken).ConfigureAwait(false); if (res != null) diff --git a/Duplicati/Library/SourceProvider/Builtin/LocalFileSource.cs b/Duplicati/Library/SourceProvider/Builtin/LocalFileSource.cs index 9c75ca3bb..d4ae9a336 100644 --- a/Duplicati/Library/SourceProvider/Builtin/LocalFileSource.cs +++ b/Duplicati/Library/SourceProvider/Builtin/LocalFileSource.cs @@ -31,10 +31,10 @@ namespace Duplicati.Library.SourceProvider; public class LocalFileSource(ISnapshotService snapshotService) : ISourceProvider { /// - public string PathKey => string.Empty; + public string MountedPath => string.Empty; /// - public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) + public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) => snapshotService.EnumerateFilesystemEntries().ToAsyncEnumerable(); /// @@ -43,7 +43,7 @@ public class LocalFileSource(ISnapshotService snapshotService) : ISourceProvider public ISnapshotService SnapshotService => snapshotService; /// - public Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) + public Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) => Task.FromResult(snapshotService.GetFilesystemEntry(path, isFolder)); /// diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index 2bffe665c..d3ce83d50 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -1557,6 +1557,9 @@ namespace Duplicati.Library.Utility /// The guessed scheme, or null if no scheme was found public static string? GuessScheme(string url) { + if (string.IsNullOrWhiteSpace(url)) + return null; + var idx = url.IndexOf("://"); if (idx < 0 && idx < 15 && idx + "://".Length < url.Length) return null; From de622b29d07270819911184262d498569f07fe9a Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 21 Feb 2025 16:12:40 +0100 Subject: [PATCH 223/362] Added source tool to launch config --- .vscode/launch.json | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index ba69c97a3..e114a9d04 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -114,6 +114,13 @@ "stopAtEntry": false, "console": "internalConsole" }, - + { + "name": "Launch SourceTool executable", + "type": "coreclr", + "request": "launch", + "preLaunchTask": "build", + "program": "${workspaceFolder}/Executables/net8/Duplicati.CommandLine.SourceTool/bin/Debug/net8.0/Duplicati.CommandLine.SourceTool", + "args": [] + } ] } \ No newline at end of file From 4e73be18061a652e2b87f02722667fcfc548db8b Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 21 Feb 2025 16:25:18 +0100 Subject: [PATCH 224/362] Fixed path conversion on Windows --- Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs | 8 +++++++- Duplicati/Library/Backend/S3/S3Backend.cs | 8 ++------ .../SourceProvider/Builtin/BackendSourceFileEntry.cs | 2 +- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs index db797a927..66d27e56b 100644 --- a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs +++ b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs @@ -236,7 +236,13 @@ namespace Duplicati.Library.Backend /// public IAsyncEnumerable ListAsync(string path, CancellationToken cancellationToken) - => m_s3Client.ListBucketAsync(m_bucket, path, false, cancellationToken); + { + var filterPath = GetFullKey(path); + if (!string.IsNullOrWhiteSpace(filterPath)) + filterPath = Util.AppendDirSeparator(filterPath, "/"); + + return m_s3Client.ListBucketAsync(m_bucket, filterPath, true, cancellationToken); + } /// diff --git a/Duplicati/Library/Backend/S3/S3Backend.cs b/Duplicati/Library/Backend/S3/S3Backend.cs index 2136eedbc..91877df04 100644 --- a/Duplicati/Library/Backend/S3/S3Backend.cs +++ b/Duplicati/Library/Backend/S3/S3Backend.cs @@ -410,13 +410,9 @@ namespace Duplicati.Library.Backend /// public IAsyncEnumerable ListAsync(string path, CancellationToken cancellationToken) { - var prefix = m_prefix; - if (!string.IsNullOrWhiteSpace(prefix)) - prefix = Util.AppendDirSeparator(prefix); - - var filterPath = prefix + path; + var filterPath = GetFullKey(path); if (!string.IsNullOrWhiteSpace(filterPath)) - filterPath = Util.AppendDirSeparator(filterPath); + filterPath = Util.AppendDirSeparator(filterPath, "/"); return s3Client.ListBucketAsync(m_bucket, filterPath, m_recurseLists, cancellationToken); } diff --git a/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs b/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs index 95fe59f91..93bea7d8c 100644 --- a/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs +++ b/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs @@ -96,7 +96,7 @@ public class BackendSourceFileEntry(BackendSourceProvider parent, string path, b if (!isFolder) throw new InvalidOperationException("Enumerate can only be called on folders"); - return parent.WrappedBackend.ListAsync(path, cancellationToken) + return parent.WrappedBackend.ListAsync(NormalizePathTo(path, '/'), cancellationToken) // Remove the current and parent folder entries .Where(x => !string.IsNullOrWhiteSpace(x.Name) && x.Name != "." && x.Name != "..") // Remove sub-folder entries From 26bbe6a5778561bbc37311316b836a7c8d9fbd8d Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 24 Feb 2025 12:50:58 +0100 Subject: [PATCH 225/362] Added setup step for source providers. This enables a fail-early approach, where the remote source loads the initial results. By storing the `IAsyncEnumerable` it is possible to not perform any additional remote calls, even with the initial probing call. This makes it possible to detect the absence of sources, so the backup can be aborted if sources are missing. --- .../SourceTool/Commands/Download.cs | 2 +- .../CommandLine/SourceTool/Commands/List.cs | 2 +- .../CommandLine/SourceTool/GetProvider.cs | 4 +- .../DynamicLoader/SourceProviderLoader.cs | 47 ++++++-- .../Library/Interface/ISourceProvider.cs | 7 ++ .../Library/Main/Operation/BackupHandler.cs | 37 ++++-- .../Main/Operation/TestFilterHandler.cs | 2 +- Duplicati/Library/Main/Options.cs | 4 +- .../Builtin/BackendSourceFileEntry.cs | 108 ++++++++++++++---- .../Builtin/BackendSourceProvider.cs | 32 +++++- .../SourceProvider/Builtin/Combiner.cs | 4 + .../SourceProvider/Builtin/LocalFileSource.cs | 3 + Duplicati/Library/Utility/Utility.cs | 31 +++++ 13 files changed, 231 insertions(+), 52 deletions(-) diff --git a/Duplicati/CommandLine/SourceTool/Commands/Download.cs b/Duplicati/CommandLine/SourceTool/Commands/Download.cs index 295545ccd..0297a2ae3 100644 --- a/Duplicati/CommandLine/SourceTool/Commands/Download.cs +++ b/Duplicati/CommandLine/SourceTool/Commands/Download.cs @@ -47,7 +47,7 @@ public static class Download { var token = new CancellationTokenSource().Token; - using var source = Common.GetProvider(url); + using var source = await Common.GetProvider(url); string localPath(string path) { diff --git a/Duplicati/CommandLine/SourceTool/Commands/List.cs b/Duplicati/CommandLine/SourceTool/Commands/List.cs index fd8cb6c25..f5ceca3fb 100644 --- a/Duplicati/CommandLine/SourceTool/Commands/List.cs +++ b/Duplicati/CommandLine/SourceTool/Commands/List.cs @@ -46,7 +46,7 @@ public static class List var folders = 0L; var files = 0L; - using var source = Common.GetProvider(url); + using var source = await Common.GetProvider(url); await Common.Visit(source, maxdepth, (entry, level) => { if (entry.IsFolder) diff --git a/Duplicati/CommandLine/SourceTool/GetProvider.cs b/Duplicati/CommandLine/SourceTool/GetProvider.cs index 1f8b30e01..f4637e398 100644 --- a/Duplicati/CommandLine/SourceTool/GetProvider.cs +++ b/Duplicati/CommandLine/SourceTool/GetProvider.cs @@ -33,7 +33,7 @@ public static partial class Common /// /// The URL to get the provider for /// The source provider - public static ISourceProvider GetProvider(string url) => - SourceProviderLoader.GetSourceProvider(url, "", []) + public static async Task GetProvider(string url) => + (await SourceProviderLoader.GetSourceProvider(url, "", [], CancellationToken.None)) ?? throw new UserInformationException(string.Format("The source \"{0}\" is not supported", url), "SourceNotSupported"); } \ No newline at end of file diff --git a/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs b/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs index b5e7819a6..979721398 100644 --- a/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs +++ b/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs @@ -26,6 +26,8 @@ using System.Linq; using System.Runtime.ExceptionServices; using Duplicati.Library.SourceProviders; using Duplicati.Library.SourceProvider; +using System.Threading.Tasks; +using System.Threading; namespace Duplicati.Library.DynamicLoader { @@ -154,7 +156,16 @@ namespace Duplicati.Library.DynamicLoader if (string.IsNullOrEmpty(url)) throw new ArgumentNullException(nameof(url)); - return _SourceProviderLoader.GetSupportedCommands(url); + var commands = _SourceProviderLoader.GetSupportedCommands(url); + if (commands != null) + return commands; + + var backend = BackendLoader.GetBackend(url, []); + if (backend is IFolderEnabledBackend folderBackend) + commands = folderBackend.SupportedCommands.AsReadOnly(); + backend?.Dispose(); + + return commands; } /// @@ -163,21 +174,35 @@ namespace Duplicati.Library.DynamicLoader /// The url to create the instance for /// The mount point to use /// The options to pass to the instance constructor + /// The cancellation token /// The instanciated SourceProvider or null if the url is not supported - public static ISourceProviderModule GetSourceProvider(string url, string mountPoint, Dictionary options) + public static async Task GetSourceProvider(string url, string mountPoint, Dictionary options, CancellationToken cancellationToken) { // Source providers are preferred over backends var provider = _SourceProviderLoader.GetSourceProvider(url, mountPoint, options); - if (provider != null) + if (provider == null) + { + // See if there is a backend that can also be a source + var backend = BackendLoader.GetBackend(url, options); + if (backend is IFolderEnabledBackend folderBackend) + provider = new BackendSourceProvider(folderBackend, mountPoint); + else + backend?.Dispose(); + } + + if (provider == null) + return null; + + try + { + await provider.Initialize(cancellationToken).ConfigureAwait(false); return provider; - - // See if there is a backend that can also be a source - var backend = BackendLoader.GetBackend(url, options); - if (backend is IFolderEnabledBackend folderBackend) - return new BackendSourceProvider(folderBackend, mountPoint); - - backend?.Dispose(); - return null; + } + catch + { + provider.Dispose(); + throw; + } } #endregion diff --git a/Duplicati/Library/Interface/ISourceProvider.cs b/Duplicati/Library/Interface/ISourceProvider.cs index 386680f00..6ac0b9ea0 100644 --- a/Duplicati/Library/Interface/ISourceProvider.cs +++ b/Duplicati/Library/Interface/ISourceProvider.cs @@ -38,6 +38,13 @@ public interface ISourceProvider : IDisposable /// string MountedPath { get; } + /// + /// Initializes the provider, if needed + /// + /// The cancellation token + /// An awaitable task + Task Initialize(CancellationToken cancellationToken); + /// /// Gets the root entries /// diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index a9786b2d8..93a8e3222 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -85,8 +85,8 @@ namespace Duplicati.Library.Main.Operation /// The sources to get providers for /// The options to use /// The source providers - public static ISourceProvider GetSourceProvider(IEnumerable sources, Options options) - => Combiner.Combine(GetSourceProviders(sources, options)); + public static async Task GetSourceProvider(IEnumerable sources, Options options, CancellationToken cancellationToken) + => Combiner.Combine(await GetSourceProviders(sources, options, cancellationToken)); /// /// Gets a snapshot service for the given sources @@ -122,7 +122,7 @@ namespace Duplicati.Library.Main.Operation /// The filter to use /// The last fileset id /// The source providers - private static List GetSourceProviders(IEnumerable sources, Options options) + private static async Task> GetSourceProviders(IEnumerable sources, Options options, CancellationToken cancellationToken) { // Group the sources by their type, so we can combine all snapshot paths into a single snapshot var sourceTypes = sources.GroupBy(x => x.StartsWith("@") ? "@" : Library.Utility.Utility.GuessScheme(x) ?? "file", StringComparer.OrdinalIgnoreCase); @@ -142,6 +142,7 @@ namespace Duplicati.Library.Main.Operation { foreach (var url in entry) { + var sanitizedUrl = Library.Utility.Utility.GetUrlWithoutCredentials(url); var m = Regex.Match(url, @"^@(?[^|]+)\|(?.+)$", RegexOptions.IgnoreCase); if (m.Success) { @@ -153,15 +154,32 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException(string.Format("The mountpoint \"{0}\" is not a valid rooted mountpoint", mountpoint), "InvalidMountpoint"); var backendurl = m.Groups["url"].Value; - results.Add(SourceProviderLoader.GetSourceProvider(backendurl, Path.GetFullPath(mountpoint), options.RawOptions) - ?? throw new UserInformationException(string.Format("The source \"{0}\" is not supported", backendurl), "SourceNotSupported")); + + ISourceProvider provider; + try + { + provider = await SourceProviderLoader.GetSourceProvider(backendurl, Path.GetFullPath(mountpoint), options.RawOptions, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + if (options.AllowMissingSource) + { + Log.WriteWarningMessage(LOGTAG, "SourceProviderFailed", ex, "Failed to load source provider for \"{0}\"", sanitizedUrl); + continue; + } + + throw new UserInformationException($"Failed to load source provider for \"{sanitizedUrl}\": {ex.Message}", "SourceProviderFailed", ex); + } + + // Don't accept missing providers + results.Add(provider ?? throw new UserInformationException($"The source \"{sanitizedUrl}\" is not supported", "SourceNotSupported")); } else - throw new UserInformationException(string.Format("The source \"{0}\" is not a supported format", url), "SourceFormatNotSupported"); + throw new UserInformationException($"The source \"{sanitizedUrl}\" is not a supported format", "SourceFormatNotSupported"); } } else - throw new UserInformationException(string.Format("The source type \"{0}\" is not supported", entry.Key), "SourceTypeNotSupported"); + throw new UserInformationException($"The source type \"{entry.Key}\" is not supported", "SourceTypeNotSupported"); } } catch @@ -172,6 +190,9 @@ namespace Duplicati.Library.Main.Operation throw; } + if (results.Count == 0) + throw new UserInformationException("No sources were available for the backup", "NoSourcesAvailable"); + return results; } @@ -600,7 +621,7 @@ namespace Duplicati.Library.Main.Operation { long filesetid; using var counterToken = CancellationTokenSource.CreateLinkedTokenSource(m_taskReader.ProgressToken); - using (var source = GetSourceProvider(sources, m_options)) + using (var source = await GetSourceProvider(sources, m_options, m_taskReader.ProgressToken).ConfigureAwait(false)) { try { diff --git a/Duplicati/Library/Main/Operation/TestFilterHandler.cs b/Duplicati/Library/Main/Operation/TestFilterHandler.cs index b2468738f..b00d37f7a 100644 --- a/Duplicati/Library/Main/Operation/TestFilterHandler.cs +++ b/Duplicati/Library/Main/Operation/TestFilterHandler.cs @@ -48,7 +48,7 @@ namespace Duplicati.Library.Main.Operation { var stopToken = m_result.TaskControl.ProgressToken; - using (var provider = BackupHandler.GetSourceProvider(sources, m_options)) + using (var provider = await BackupHandler.GetSourceProvider(sources, m_options, stopToken).ConfigureAwait(false)) { Backup.Channels channels = new(); var source = Backup.FileEnumerationProcess.Run(channels, provider, null, diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index 4d32c8ae3..bccc5c9c8 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -1042,7 +1042,7 @@ namespace Duplicati.Library.Main return r; } - } + } /// /// Gets a flag indicating if advisory locking should be ignored @@ -1706,7 +1706,7 @@ namespace Duplicati.Library.Main } /// - /// Gets a flag indicating if compacting should not be done automatically + /// Gets a flag indicating if missing source elements should be ignored /// public bool AllowMissingSource { diff --git a/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs b/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs index 93bea7d8c..481af6065 100644 --- a/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs +++ b/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs @@ -20,6 +20,7 @@ // DEALINGS IN THE SOFTWARE. using System.IO.Pipelines; +using System.Runtime.CompilerServices; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using Duplicati.Library.Utility; @@ -32,11 +33,11 @@ namespace Duplicati.Library.SourceProvider; /// The parent backend /// The path of the entry /// True if the entry is a folder -/// True if the entry is a meta entry +/// True if the entry is a meta entry /// The creation time of the entry /// The last modification time of the entry /// The size of the entry -public class BackendSourceFileEntry(BackendSourceProvider parent, string path, bool isFolder, bool isMetaEntry, DateTime createdUtc, DateTime lastModificationUtc, long size) +public class BackendSourceFileEntry(BackendSourceProvider parent, string path, bool isFolder, bool isRootEntry, DateTime createdUtc, DateTime lastModificationUtc, long size) : ISourceProviderEntry { /// @@ -48,10 +49,10 @@ public class BackendSourceFileEntry(BackendSourceProvider parent, string path, b public bool IsFolder => isFolder; /// - public bool IsMetaEntry => isMetaEntry; + public bool IsMetaEntry => false; /// - public bool IsRootEntry => isMetaEntry; + public bool IsRootEntry => isRootEntry; /// public DateTime CreatedUtc => createdUtc; @@ -90,36 +91,93 @@ public class BackendSourceFileEntry(BackendSourceProvider parent, string path, b /// public string? HardlinkTargetId => null; + /// + /// An async enumerator for the entries + /// + private IAsyncEnumerator? preparedEnumerator = null; + + /// + /// A flag to indicate if the prepared enumerator has any entries + /// + private bool preparedEnumeratorAny = false; + + /// + /// Prepares the enumerator for this entry + /// + /// The cancellation token + /// The prepared enumerator + public async Task PrepareEnumerator(CancellationToken cancellationToken) + { + if (!isFolder || !isRootEntry) + throw new InvalidOperationException("PrepareEnumerator can only be called on root folders"); + + if (preparedEnumerator != null) + throw new InvalidOperationException("PrepareEnumerator can only be called once"); + + var result = EnumerateInternal(cancellationToken).GetAsyncEnumerator(cancellationToken); + var prev = Interlocked.Exchange(ref preparedEnumerator, result); + if (prev != null) + throw new InvalidOperationException("PrepareEnumerator can only be called once"); + + // Advance to the first entry, so we are sure it does not throw exceptions + preparedEnumeratorAny = await result.MoveNextAsync().ConfigureAwait(false); + } + /// - public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) + public async IAsyncEnumerable Enumerate([EnumeratorCancellation] CancellationToken cancellationToken) { if (!isFolder) throw new InvalidOperationException("Enumerate can only be called on folders"); - return parent.WrappedBackend.ListAsync(NormalizePathTo(path, '/'), cancellationToken) - // Remove the current and parent folder entries - .Where(x => !string.IsNullOrWhiteSpace(x.Name) && x.Name != "." && x.Name != "..") - // Remove sub-folder entries - .Where(x => !x.Name[0..^1].Contains('\\') && !x.Name[0..^1].Contains('/')) - // Convert to source file entries - .Select(x => + if (isRootEntry) + { + // If we have a prepared enumerator, consume and use it + var enumerator = Interlocked.Exchange(ref preparedEnumerator, null); + if (enumerator != null) { - var localPath = SystemIO.IO_OS.PathCombine(path, NormalizePathToLocalSystem(x.Name)); - if (x.IsFolder) - localPath = Util.AppendDirSeparator(localPath); + if (preparedEnumeratorAny) + { + // It has already been advanced, so return the current value + yield return enumerator.Current; + + while (await enumerator.MoveNextAsync()) + yield return enumerator.Current; + } + + yield break; + } + } + + // Otherwise, enumerate the entries + await foreach (var entry in EnumerateInternal(cancellationToken).ConfigureAwait(false)) + yield return entry; - return new BackendSourceFileEntry( - parent, - localPath, - x.IsFolder, - false, - x.Created, - x.LastModification, - x.Size - ); - }); } + private IAsyncEnumerable EnumerateInternal(CancellationToken cancellationToken) + => parent.WrappedBackend.ListAsync(NormalizePathTo(path, '/'), cancellationToken) + // Remove the current and parent folder entries + .Where(x => !string.IsNullOrWhiteSpace(x.Name) && x.Name != "." && x.Name != "..") + // Remove sub-folder entries + .Where(x => !x.Name[0..^1].Contains('\\') && !x.Name[0..^1].Contains('/')) + // Convert to source file entries + .Select(x => + { + var localPath = SystemIO.IO_OS.PathCombine(path, NormalizePathToLocalSystem(x.Name)); + if (x.IsFolder) + localPath = Util.AppendDirSeparator(localPath); + + return new BackendSourceFileEntry( + parent, + localPath, + x.IsFolder, + false, + x.Created, + x.LastModification, + x.Size + ); + }); + /// public async Task FileExists(string path, CancellationToken cancellationToken) { diff --git a/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs index 2ae3d9a71..85a8ba1b5 100644 --- a/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs +++ b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs @@ -49,9 +49,39 @@ public class BackendSourceProvider(IFolderEnabledBackend backend, string mounted /// public IList SupportedCommands => backend.SupportedCommands; + /// + /// The prepared root entry, if any + /// + private BackendSourceFileEntry? preparedRoot; + + /// + /// Flags if the provider has been initialized + /// + private int isInitialized = 0; + + /// + /// Creates a root entry + /// + /// The root entry + private BackendSourceFileEntry CreateRoot() + => new BackendSourceFileEntry(this, "", true, true, new DateTime(0), new DateTime(0), 0); + + /// + public async Task Initialize(CancellationToken cancellationToken) + { + // Only allow a single intiiialization call + if (Interlocked.Exchange(ref isInitialized, 1) != 0) + return; + + // Prepare the root entry + var root = CreateRoot(); + await root.PrepareEnumerator(cancellationToken).ConfigureAwait(false); + preparedRoot = root; + } + /// public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) - => new[] { new BackendSourceFileEntry(this, "", true, true, new DateTime(0), new DateTime(0), 0) }.ToAsyncEnumerable(); + => new[] { Interlocked.Exchange(ref preparedRoot, null) ?? CreateRoot() }.ToAsyncEnumerable(); /// public async Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) diff --git a/Duplicati/Library/SourceProvider/Builtin/Combiner.cs b/Duplicati/Library/SourceProvider/Builtin/Combiner.cs index ed886a028..5205c049e 100644 --- a/Duplicati/Library/SourceProvider/Builtin/Combiner.cs +++ b/Duplicati/Library/SourceProvider/Builtin/Combiner.cs @@ -37,6 +37,10 @@ public class Combiner(IEnumerable providers) : ISourceProvider /// private readonly List providers = providers.SelectMany(x => x is Combiner c ? c.providers.AsEnumerable() : [x]).ToList(); + /// + public Task Initialize(CancellationToken cancellationToken) + => Task.WhenAll(providers.Select(x => x.Initialize(cancellationToken))); + /// public async IAsyncEnumerable Enumerate([EnumeratorCancellation] CancellationToken cancellationToken) { diff --git a/Duplicati/Library/SourceProvider/Builtin/LocalFileSource.cs b/Duplicati/Library/SourceProvider/Builtin/LocalFileSource.cs index d4ae9a336..2568b4474 100644 --- a/Duplicati/Library/SourceProvider/Builtin/LocalFileSource.cs +++ b/Duplicati/Library/SourceProvider/Builtin/LocalFileSource.cs @@ -33,6 +33,9 @@ public class LocalFileSource(ISnapshotService snapshotService) : ISourceProvider /// public string MountedPath => string.Empty; + /// + public Task Initialize(CancellationToken cancellationToken) => Task.CompletedTask; + /// public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) => snapshotService.EnumerateFilesystemEntries().ToAsyncEnumerable(); diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index d3ce83d50..0874952e3 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -1565,7 +1565,38 @@ namespace Duplicati.Library.Utility return null; return url.Substring(0, idx); + } + /// + /// Returns a url that is safe to display, by removing any credentials + /// + /// The url to sanitize + /// The sanitized url + public static string GetUrlWithoutCredentials(string url) + { + // Assumed safe part of the url to show + const int maxShown = 25; + if (string.IsNullOrWhiteSpace(url)) + return url; + + // Use a reportable url without credentials + var sepIndex = Math.Max(0, url.IndexOf('|')) + 1; + var length = url.Length - sepIndex; + var shown = Math.Min(length, maxShown); + var hidden = length - maxShown; + var sanitizedUrl = $"{url[sepIndex..(sepIndex + shown)]}{new string('*', hidden)}"; + + // If we can parse it, this result is better + try + { + var uri = new Uri(url[sepIndex..]); + sanitizedUrl = new Uri($"{uri.Scheme}://{uri.Host}").SetPath(uri.Path).ToString(); + } + catch + { + } + + return sanitizedUrl; } } } From 1428d8cc6fc39efe01ca3b53b54c8b0f8659e65b Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 24 Feb 2025 12:53:48 +0100 Subject: [PATCH 226/362] Preserve order of input sources --- Duplicati/Library/Main/Controller.cs | 2 +- Duplicati/Library/Utility/Utility.cs | 11 +++++--- Duplicati/UnitTest/UtilityTests.cs | 41 ++++++++++++++-------------- 3 files changed, 28 insertions(+), 26 deletions(-) diff --git a/Duplicati/Library/Main/Controller.cs b/Duplicati/Library/Main/Controller.cs index abd3b311e..637caee3b 100644 --- a/Duplicati/Library/Main/Controller.cs +++ b/Duplicati/Library/Main/Controller.cs @@ -1012,7 +1012,7 @@ namespace Duplicati.Library.Main //Sanity check for duplicate files/folders ISet pathDuplicates; - sources = Library.Utility.Utility.GetUniqueItems(sources, Library.Utility.Utility.ClientFilenameStringComparer, out pathDuplicates).OrderBy(a => a).ToList(); + sources = Library.Utility.Utility.GetUniqueItems(sources, Library.Utility.Utility.ClientFilenameStringComparer, out pathDuplicates).ToList(); foreach (var pathDuplicate in pathDuplicates) Logging.Log.WriteVerboseMessage(LOGTAG, "RemoveDuplicateSource", "Removing duplicate source: {0}", pathDuplicate); diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index 0874952e3..038d3adad 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -981,7 +981,7 @@ namespace Duplicati.Library.Utility /// The collection to remove duplicate items from. /// The duplicate items in . /// The unique items from . - public static ISet GetUniqueItems(IEnumerable collection, out ISet duplicateItems) + public static IList GetUniqueItems(IEnumerable collection, out ISet duplicateItems) { return GetUniqueItems(collection, EqualityComparer.Default, out duplicateItems); } @@ -994,18 +994,21 @@ namespace Duplicati.Library.Utility /// The implementation to use when comparing values in the collection. /// The duplicate items in . /// The unique items from . - public static ISet GetUniqueItems(IEnumerable collection, IEqualityComparer comparer, out ISet duplicateItems) + public static IList GetUniqueItems(IEnumerable collection, IEqualityComparer comparer, out ISet duplicateItems) { var uniqueItems = new HashSet(comparer); + var results = new List(); duplicateItems = new HashSet(comparer); foreach (var item in collection) { - if (!uniqueItems.Add(item)) + if (uniqueItems.Add(item)) + results.Add(item); + else duplicateItems.Add(item); } - return uniqueItems; + return results; } // diff --git a/Duplicati/UnitTest/UtilityTests.cs b/Duplicati/UnitTest/UtilityTests.cs index c4dd5792c..35ca8b92c 100644 --- a/Duplicati/UnitTest/UtilityTests.cs +++ b/Duplicati/UnitTest/UtilityTests.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Utility; @@ -349,8 +349,7 @@ namespace Duplicati.UnitTest string[] duplicateItems = { "A", "c" }; // Test with default comparer. - ISet actualDuplicateItems; - ISet actualUniqueItems = Utility.GetUniqueItems(collection, out actualDuplicateItems); + var actualUniqueItems = Utility.GetUniqueItems(collection, out var actualDuplicateItems); CollectionAssert.AreEquivalent(uniqueItems, actualUniqueItems); CollectionAssert.AreEquivalent(duplicateItems, actualDuplicateItems); From 30b1cd59b8add78f9c73a5483e46ec4389729be1 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 24 Feb 2025 13:04:35 +0100 Subject: [PATCH 227/362] Fixed cases where Windows paths were sent to the remote destination --- .../Library/SourceProvider/Builtin/BackendSourceFileEntry.cs | 4 ++-- .../Library/SourceProvider/Builtin/BackendSourceProvider.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs b/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs index 481af6065..b9e134014 100644 --- a/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs +++ b/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs @@ -186,7 +186,7 @@ public class BackendSourceFileEntry(BackendSourceProvider parent, string path, b try { - var entry = await parent.WrappedBackend.GetEntryAsync(path, cancellationToken); + var entry = await parent.WrappedBackend.GetEntryAsync(NormalizePathTo(path, '/'), cancellationToken); return entry != null && !entry.IsFolder; } catch (FileNotFoundException) @@ -267,7 +267,7 @@ public class BackendSourceFileEntry(BackendSourceProvider parent, string path, b { try { - await streamingBackend.GetAsync(path, pipe.Writer.AsStream(), cancellationToken).ConfigureAwait(false); + await streamingBackend.GetAsync(NormalizePathTo(path, '/'), pipe.Writer.AsStream(), cancellationToken).ConfigureAwait(false); } catch (Exception ex) { diff --git a/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs index 85a8ba1b5..8e9c39853 100644 --- a/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs +++ b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs @@ -86,7 +86,7 @@ public class BackendSourceProvider(IFolderEnabledBackend backend, string mounted /// public async Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) { - var entry = await backend.GetEntryAsync(path, cancellationToken).ConfigureAwait(false); + var entry = await backend.GetEntryAsync(BackendSourceFileEntry.NormalizePathTo(path, '/'), cancellationToken).ConfigureAwait(false); return entry == null ? null : BackendSourceFileEntry.FromFileEntry(this, path, entry); } From 6238067f32352e9a81e7f59ca2f754d268ca71c1 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 24 Feb 2025 13:17:08 +0100 Subject: [PATCH 228/362] Updated logic for S3 backend, so the client implementations return the relative (non-prefixed) path --- Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs | 6 ------ Duplicati/Library/Backend/S3/S3AwsClient.cs | 4 ++-- Duplicati/Library/Backend/S3/S3Backend.cs | 8 -------- 3 files changed, 2 insertions(+), 16 deletions(-) diff --git a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs index 66d27e56b..62b5b8dc1 100644 --- a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs +++ b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs @@ -132,13 +132,7 @@ namespace Duplicati.Library.Backend public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { await foreach (IFileEntry file in Connection.ListBucketAsync(m_bucket, m_prefix, false, cancelToken).ConfigureAwait(false)) - { - ((FileEntry)file).Name = file.Name.Substring(m_prefix.Length); - if (file.Name.StartsWith("/", StringComparison.Ordinal) && !m_prefix.StartsWith("/", StringComparison.Ordinal)) - ((FileEntry)file).Name = file.Name.Substring(1); - yield return file; - } } public async Task PutAsync(string remotename, string localname, CancellationToken cancelToken) diff --git a/Duplicati/Library/Backend/S3/S3AwsClient.cs b/Duplicati/Library/Backend/S3/S3AwsClient.cs index 4527826dd..5fd16627b 100644 --- a/Duplicati/Library/Backend/S3/S3AwsClient.cs +++ b/Duplicati/Library/Backend/S3/S3AwsClient.cs @@ -278,7 +278,7 @@ namespace Duplicati.Library.Backend // Because the prefixes are returned, and not the folder objects // we do not get the folder modification date - yield return new Common.IO.FileEntry( + yield return new FileEntry( obj.Substring(prefix.Length), -1, new DateTime(0), @@ -293,7 +293,7 @@ namespace Duplicati.Library.Backend if (obj.Key == prefix || !obj.Key.StartsWith(prefix)) continue; - yield return new Common.IO.FileEntry( + yield return new FileEntry( obj.Key.Substring(prefix.Length), obj.Size, obj.LastModified, diff --git a/Duplicati/Library/Backend/S3/S3Backend.cs b/Duplicati/Library/Backend/S3/S3Backend.cs index 91877df04..4f081ea1c 100644 --- a/Duplicati/Library/Backend/S3/S3Backend.cs +++ b/Duplicati/Library/Backend/S3/S3Backend.cs @@ -281,15 +281,7 @@ namespace Duplicati.Library.Backend public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { await foreach (IFileEntry file in Connection.ListBucketAsync(m_bucket, m_prefix, m_recurseLists, cancelToken)) - { - ((FileEntry)file).Name = file.Name.Substring(m_prefix.Length); - - //Fix for a bug in Duplicati 1.0 beta 3 and earlier, where filenames are incorrectly prefixed with a slash - if (file.Name.StartsWith("/", StringComparison.Ordinal) && !m_prefix.StartsWith("/", StringComparison.Ordinal)) - ((FileEntry)file).Name = file.Name.Substring(1); - yield return file; - } } public async Task PutAsync(string remotename, string localname, CancellationToken cancelToken) From ea07ac4288afdf06863de9691031d217ea956347 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 24 Feb 2025 13:24:58 +0100 Subject: [PATCH 229/362] Updated build logic to use target os --- .../Modules/Builtin/Duplicati.Library.Modules.Builtin.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Modules/Builtin/Duplicati.Library.Modules.Builtin.csproj b/Duplicati/Library/Modules/Builtin/Duplicati.Library.Modules.Builtin.csproj index 26b025594..b247c5d56 100644 --- a/Duplicati/Library/Modules/Builtin/Duplicati.Library.Modules.Builtin.csproj +++ b/Duplicati/Library/Modules/Builtin/Duplicati.Library.Modules.Builtin.csproj @@ -24,13 +24,13 @@ - + PreserveNewest - + PreserveNewest From 39ba3ab89eb151bc7b9a9f7b12ea6b93539c4956 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 24 Feb 2025 13:25:47 +0100 Subject: [PATCH 230/362] More exact rid match --- .../Modules/Builtin/Duplicati.Library.Modules.Builtin.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Modules/Builtin/Duplicati.Library.Modules.Builtin.csproj b/Duplicati/Library/Modules/Builtin/Duplicati.Library.Modules.Builtin.csproj index b247c5d56..b6d52061d 100644 --- a/Duplicati/Library/Modules/Builtin/Duplicati.Library.Modules.Builtin.csproj +++ b/Duplicati/Library/Modules/Builtin/Duplicati.Library.Modules.Builtin.csproj @@ -24,13 +24,13 @@ - + PreserveNewest - + PreserveNewest From 3bfd12bf76ee36164de0c9fad2bbf82af7a2bc1b Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 24 Feb 2025 13:32:26 +0100 Subject: [PATCH 231/362] Fixed rid detection --- .../Builtin/Duplicati.Library.Modules.Builtin.csproj | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Modules/Builtin/Duplicati.Library.Modules.Builtin.csproj b/Duplicati/Library/Modules/Builtin/Duplicati.Library.Modules.Builtin.csproj index b6d52061d..bed29c7a7 100644 --- a/Duplicati/Library/Modules/Builtin/Duplicati.Library.Modules.Builtin.csproj +++ b/Duplicati/Library/Modules/Builtin/Duplicati.Library.Modules.Builtin.csproj @@ -23,14 +23,20 @@ - - + + + + $([System.String]::Copy('$(RuntimeIdentifier)')) + $([System.String]::Substring('$(RuntimeIdentifier)', 0, 4)) + + + PreserveNewest - + PreserveNewest From 704485176ab57381324658b67d42a16b726f6ad2 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 24 Feb 2025 13:56:28 +0100 Subject: [PATCH 232/362] Clarified the `Path` parameter --- Duplicati/Library/Utility/Uri.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Duplicati/Library/Utility/Uri.cs b/Duplicati/Library/Utility/Uri.cs index bc7505a7c..0e918132e 100644 --- a/Duplicati/Library/Utility/Uri.cs +++ b/Duplicati/Library/Utility/Uri.cs @@ -57,7 +57,8 @@ namespace Duplicati.Library.Utility /// public readonly string Host; /// - /// The server path, e.g. index.html + /// The server path, e.g. index.html. + /// Note that the path does NOT have a leading /. /// public readonly string Path; /// @@ -457,7 +458,7 @@ namespace Duplicati.Library.Utility /// The query to parse public static NameValueCollection ParseQueryString(string query) { - return Library.Utility.Uri.ParseQueryString(query, true); + return ParseQueryString(query, true); } /// @@ -537,7 +538,7 @@ namespace Duplicati.Library.Utility { var builder = new UriBuilder(url) { - Path = (new UrlPath(ExtractPath(url)).Append(path)).ToString(), + Path = new UrlPath(ExtractPath(url)).Append(path).ToString(), Query = query != null ? BuildUriQuery(query) : null }; return builder.Uri.AbsoluteUri; @@ -551,7 +552,7 @@ namespace Duplicati.Library.Utility /// URL. public static string ExtractPath(string url) { - return (new Uri(url)).Path; + return new Uri(url).Path; } /// From 66aa3cd9d6605fcea78a70ea31add2197f137d58 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 25 Feb 2025 15:07:17 +0100 Subject: [PATCH 233/362] Added support for profiles when building for different channels. Changed the default config name to have a `duplicati` prefix. --- ReleaseBuilder/Build/Command.Compile.Post.cs | 2 +- ReleaseBuilder/Build/Command.CreatePackage.cs | 14 ++-- ReleaseBuilder/Build/Command.GpgSign.cs | 4 +- ReleaseBuilder/Build/Command.RuntimeConfig.cs | 80 ++++++++++--------- ReleaseBuilder/Build/Command.Upload.cs | 18 ++--- ReleaseBuilder/Build/Command.cs | 28 ++++--- ReleaseBuilder/Configuration.cs | 27 +++++-- ReleaseBuilder/EnvHelper.cs | 13 ++- ReleaseBuilder/Program.cs | 45 +++++------ 9 files changed, 128 insertions(+), 103 deletions(-) diff --git a/ReleaseBuilder/Build/Command.Compile.Post.cs b/ReleaseBuilder/Build/Command.Compile.Post.cs index 122576901..4f88d29e6 100644 --- a/ReleaseBuilder/Build/Command.Compile.Post.cs +++ b/ReleaseBuilder/Build/Command.Compile.Post.cs @@ -316,7 +316,7 @@ public static partial class Command /// An awaitable task static async Task SignWindowsExecutables(string buildDir, RuntimeConfig rtcfg) { - var cfg = Program.Configuration; + var cfg = rtcfg.Configuration; if (!rtcfg.UseAuthenticodeSigning) return; diff --git a/ReleaseBuilder/Build/Command.CreatePackage.cs b/ReleaseBuilder/Build/Command.CreatePackage.cs index fe636d207..9d28302d7 100644 --- a/ReleaseBuilder/Build/Command.CreatePackage.cs +++ b/ReleaseBuilder/Build/Command.CreatePackage.cs @@ -175,7 +175,7 @@ public static partial class Command { // # Notarize and staple takes a while... Console.WriteLine($"Performing notarize and staple of {packageFile} ..."); - await ProcessHelper.Execute(["xcrun", "notarytool", "submit", tempFile, "--keychain-profile", Program.Configuration.ConfigFiles.NotarizeProfile, "--wait"]); + await ProcessHelper.Execute(["xcrun", "notarytool", "submit", tempFile, "--keychain-profile", rtcfg.Configuration.ConfigFiles.NotarizeProfile, "--wait"]); await ProcessHelper.Execute(["xcrun", "stapler", "staple", tempFile]); } @@ -322,7 +322,7 @@ public static partial class Command .Replace(originalNamespace, wixv4Namespace)); wixArgs = [ - Program.Configuration.Commands.Wix!, + rtcfg.Configuration.Commands.Wix!, "build", "-define", $"HarvestPath={sourceFiles}", "-arch", msiArch, @@ -336,7 +336,7 @@ public static partial class Command { // wixl needs the UI extension wixArgs = [ - Program.Configuration.Commands.Wix!, + rtcfg.Configuration.Commands.Wix!, "--ext", "ui", "--extdir", Path.Combine(resourcesDir, "WixUIExtension"), "--define", $"HarvestPath={sourceFiles}", @@ -1180,11 +1180,11 @@ public static partial class Command private static async Task BuildDockerImages(string baseDir, string buildRoot, IEnumerable targets, RuntimeConfig rtcfg) { // Make sure any dangling buildx instances are removed - try { await ProcessHelper.Execute([Program.Configuration.Commands.Docker!, "buildx", "rm", "duplicati-builder"], codeIsError: _ => false, suppressStdErr: true); } + try { await ProcessHelper.Execute([rtcfg.Configuration.Commands.Docker!, "buildx", "rm", "duplicati-builder"], codeIsError: _ => false, suppressStdErr: true); } catch { } // Prepare multi-build - await ProcessHelper.Execute([Program.Configuration.Commands.Docker!, "buildx", "create", "--use", "--name", "duplicati-builder"]); + await ProcessHelper.Execute([rtcfg.Configuration.Commands.Docker!, "buildx", "create", "--use", "--name", "duplicati-builder"]); // Perform a distict build for each interface type, but keep the buildx instance foreach (var interfaceType in Enum.GetValues()) @@ -1239,7 +1239,7 @@ public static partial class Command var repo = $"{rtcfg.DockerRepo}{(interfaceType == InterfaceType.Agent ? "-agent" : "")}"; // Build the images - var args = new List { Program.Configuration.Commands.Docker!, "buildx", "build" }; + var args = new List { rtcfg.Configuration.Commands.Docker!, "buildx", "build" }; args.AddRange(tags.SelectMany(x => new[] { "-t", $"{repo}:{x.ToLowerInvariant()}" })); args.AddRange([ "--platform", string.Join(",", dockerArchs), @@ -1258,6 +1258,6 @@ public static partial class Command } // Clean up - await ProcessHelper.Execute([Program.Configuration.Commands.Docker!, "buildx", "rm", "duplicati-builder"]); + await ProcessHelper.Execute([rtcfg.Configuration.Commands.Docker!, "buildx", "rm", "duplicati-builder"]); } } diff --git a/ReleaseBuilder/Build/Command.GpgSign.cs b/ReleaseBuilder/Build/Command.GpgSign.cs index a574af037..4b203a014 100644 --- a/ReleaseBuilder/Build/Command.GpgSign.cs +++ b/ReleaseBuilder/Build/Command.GpgSign.cs @@ -53,7 +53,7 @@ public static partial class Command await ProcessHelper.Execute( [ - Program.Configuration.Commands.Gpg!, + rtcfg.Configuration.Commands.Gpg!, "--pinentry-mode", "loopback", "--passphrase-fd", "0", "--batch", "--yes", @@ -86,7 +86,7 @@ public static partial class Command static (string GpgId, string GpgPassphrase) GetGpgIdAndPassphrase(RuntimeConfig rtcfg) { using var ms = new MemoryStream(); - using var fs = File.OpenRead(Program.Configuration.ConfigFiles.GpgKeyfile); + using var fs = File.OpenRead(rtcfg.Configuration.ConfigFiles.GpgKeyfile); SharpAESCrypt.SharpAESCrypt.Decrypt(rtcfg.KeyfilePassword, fs, ms); var parts = System.Text.Encoding.UTF8.GetString(ms.ToArray()).Split('\n', 2, StringSplitOptions.RemoveEmptyEntries); return (parts[0], parts[1]); diff --git a/ReleaseBuilder/Build/Command.RuntimeConfig.cs b/ReleaseBuilder/Build/Command.RuntimeConfig.cs index a47401608..20ed732ce 100644 --- a/ReleaseBuilder/Build/Command.RuntimeConfig.cs +++ b/ReleaseBuilder/Build/Command.RuntimeConfig.cs @@ -27,24 +27,23 @@ public static partial class Command /// /// Setup of the current runtime information /// - private class RuntimeConfig + /// + /// Constructs a new + /// + /// The release info to use + /// The keyfile password to use + /// The sign keys + /// The changelog news + /// The command input + private class RuntimeConfig( + Configuration configuration, + ReleaseInfo releaseInfo, + IEnumerable signKeys, + string keyfilePassword, + string changelogNews, + CommandInput input + ) { - /// - /// Constructs a new - /// - /// The release info to use - /// The keyfile password to use - /// The sign keys - /// The changelog news - /// The command input - public RuntimeConfig(ReleaseInfo releaseInfo, IEnumerable signKeys, string keyfilePassword, string changelogNews, CommandInput input) - { - ReleaseInfo = releaseInfo; - SignKeys = signKeys; - KeyfilePassword = keyfilePassword; - ChangelogNews = changelogNews; - Input = input; - } /// /// The cached password for the pfx file @@ -54,27 +53,30 @@ public static partial class Command /// /// The commandline input /// - private CommandInput Input { get; } - + private CommandInput Input => input; + /// + /// The configuration to use + /// + public Configuration Configuration => configuration; /// /// The release info for this run /// - public ReleaseInfo ReleaseInfo { get; } + public ReleaseInfo ReleaseInfo => releaseInfo; /// /// The keyfile password for this run /// - public IEnumerable SignKeys { get; } + public IEnumerable SignKeys => signKeys; /// /// The primary password /// - public string KeyfilePassword { get; } + public string KeyfilePassword => keyfilePassword; /// /// The changelog news /// - public string ChangelogNews { get; } + public string ChangelogNews => changelogNews; /// /// Gets the PFX password and throws if not possible @@ -102,7 +104,7 @@ public static partial class Command return; } - if (Program.Configuration.IsAuthenticodePossible()) + if (Configuration.IsAuthenticodePossible()) _useAuthenticodeSigning = true; else { @@ -137,7 +139,7 @@ public static partial class Command if (!OperatingSystem.IsMacOS()) _useCodeSignSigning = false; - else if (Program.Configuration.IsCodeSignPossible()) + else if (Configuration.IsCodeSignPossible()) _useCodeSignSigning = true; else { @@ -166,7 +168,7 @@ public static partial class Command { try { - var res = await ProcessHelper.ExecuteWithOutput([Program.Configuration.Commands.Docker!, "ps"], suppressStdErr: true); + var res = await ProcessHelper.ExecuteWithOutput([Configuration.Commands.Docker!, "ps"], suppressStdErr: true); _dockerBuild = true; } catch @@ -203,7 +205,7 @@ public static partial class Command if (!OperatingSystem.IsMacOS()) _useNotarizeSigning = false; - else if (Program.Configuration.IsNotarizePossible()) + else if (Configuration.IsNotarizePossible()) _useNotarizeSigning = true; else { @@ -236,7 +238,7 @@ public static partial class Command return; } - if (Program.Configuration.IsGpgPossible()) + if (Configuration.IsGpgPossible()) _useGpgSigning = true; else { @@ -269,7 +271,7 @@ public static partial class Command return; } - if (Program.Configuration.IsAwsUploadPossible()) + if (Configuration.IsAwsUploadPossible()) _useS3Upload = true; else { @@ -303,7 +305,7 @@ public static partial class Command return; } - if (Program.Configuration.IsGithubUploadPossible()) + if (Configuration.IsGithubUploadPossible()) _useGithubUpload = true; else { @@ -336,7 +338,7 @@ public static partial class Command return; } - if (Program.Configuration.IsUpdateServerReloadPossible()) + if (Configuration.IsUpdateServerReloadPossible()) _useUpdateServerReload = true; else { @@ -370,7 +372,7 @@ public static partial class Command return; } - if (Program.Configuration.IsDiscourseAnnouncePossible()) + if (Configuration.IsDiscourseAnnouncePossible()) _useDiscourseAnnounce = true; else { @@ -451,7 +453,7 @@ public static partial class Command /// Password for the password file /// The Authenticode password private string GetAuthenticodePassword(string keyfilepassword) - => EncryptionHelper.DecryptPasswordFile(Program.Configuration.ConfigFiles.AuthenticodePasswordFile, keyfilepassword).Trim(); + => EncryptionHelper.DecryptPasswordFile(Configuration.ConfigFiles.AuthenticodePasswordFile, keyfilepassword).Trim(); /// /// Performs authenticode signing if enabled @@ -461,8 +463,8 @@ public static partial class Command public Task AuthenticodeSign(string file) => UseAuthenticodeSigning ? ProcessRunner.OsslCodeSign( - Program.Configuration.Commands.OsslSignCode!, - Program.Configuration.ConfigFiles.AuthenticodePfxFile, + Configuration.Commands.OsslSignCode!, + Configuration.ConfigFiles.AuthenticodePfxFile, PfxPassword, file) : Task.CompletedTask; @@ -476,8 +478,8 @@ public static partial class Command public Task Codesign(string file, string entitlements) => UseCodeSignSigning ? ProcessRunner.MacOSCodeSign( - Program.Configuration.Commands.Codesign!, - Program.Configuration.ConfigFiles.CodesignIdentity, + Configuration.Commands.Codesign!, + Configuration.ConfigFiles.CodesignIdentity, entitlements, file ) @@ -491,8 +493,8 @@ public static partial class Command public Task Productsign(string file) => UseCodeSignSigning ? ProcessRunner.MacOSProductSign( - Program.Configuration.Commands.Productsign!, - Program.Configuration.ConfigFiles.CodesignIdentity, + Configuration.Commands.Productsign!, + Configuration.ConfigFiles.CodesignIdentity, file ) : Task.CompletedTask; diff --git a/ReleaseBuilder/Build/Command.Upload.cs b/ReleaseBuilder/Build/Command.Upload.cs index c311f5bbb..c16a3d9b7 100644 --- a/ReleaseBuilder/Build/Command.Upload.cs +++ b/ReleaseBuilder/Build/Command.Upload.cs @@ -114,10 +114,10 @@ public static partial class Command Console.WriteLine($"Uploading {files.Count()} files ({Duplicati.Library.Utility.Utility.FormatSizeString(totalSize)}) to S3..."); var chain = new Amazon.Runtime.CredentialManagement.CredentialProfileStoreChain(); - if (!chain.TryGetAWSCredentials(Program.Configuration.ConfigFiles.AwsUploadProfile, out var awsCredentials)) - throw new Exception($"The aws-cli profile '{Program.Configuration.ConfigFiles.AwsUploadProfile}' could not be found."); + if (!chain.TryGetAWSCredentials(rtcfg.Configuration.ConfigFiles.AwsUploadProfile, out var awsCredentials)) + throw new Exception($"The aws-cli profile '{rtcfg.Configuration.ConfigFiles.AwsUploadProfile}' could not be found."); - chain.TryGetProfile(Program.Configuration.ConfigFiles.AwsUploadProfile, out var awsProfile); + chain.TryGetProfile(rtcfg.Configuration.ConfigFiles.AwsUploadProfile, out var awsProfile); var config = new AmazonS3Config { RegionEndpoint = awsProfile?.Region ?? RegionEndpoint.USEast1, @@ -134,7 +134,7 @@ public static partial class Command await Duplicati.Library.Utility.RetryHelper.Retry(() => fileTransferUtility.UploadAsync( file.Path, - Program.Configuration.ConfigFiles.AwsUploadBucket, + rtcfg.Configuration.ConfigFiles.AwsUploadBucket, $"{rtcfg.ReleaseInfo.Channel.ToString().ToLowerInvariant()}/{file.Name}" ), 3, TimeSpan.FromSeconds(1), CancellationToken.None); @@ -152,9 +152,9 @@ public static partial class Command var source = $"{rtcfg.ReleaseInfo.Channel.ToString().ToLowerInvariant()}/latest-v2.manifest"; await Duplicati.Library.Utility.RetryHelper.Retry(() => client.CopyObjectAsync( - Program.Configuration.ConfigFiles.AwsUploadBucket, + rtcfg.Configuration.ConfigFiles.AwsUploadBucket, source, - Program.Configuration.ConfigFiles.AwsUploadBucket, + rtcfg.Configuration.ConfigFiles.AwsUploadBucket, target ), 3, TimeSpan.FromSeconds(1), CancellationToken.None); } @@ -175,7 +175,7 @@ public static partial class Command var totalSize = files.Sum(f => new FileInfo(f.Path).Length); Console.WriteLine($"Uploading {files.Count()} files ({Duplicati.Library.Utility.Utility.FormatSizeString(totalSize)}) to Github..."); - var ghtoken = File.ReadAllText(Program.Configuration.ConfigFiles.GithubTokenFile).Trim(); + var ghtoken = File.ReadAllText(rtcfg.Configuration.ConfigFiles.GithubTokenFile).Trim(); var owner = "duplicati"; var repo = "duplicati"; @@ -236,7 +236,7 @@ public static partial class Command using var client = new HttpClient(); var req = new HttpRequestMessage(HttpMethod.Post, "https://updates.duplicati.com/reload"); - var reloadToken = Program.Configuration.ConfigFiles.ReloadUpdatesApiKey; + var reloadToken = rtcfg.Configuration.ConfigFiles.ReloadUpdatesApiKey; req.Headers.Add("X-API-KEY", reloadToken); req.Content = JsonContent.Create(new[] { $"{rtcfg.ReleaseInfo.Channel.ToString().ToLowerInvariant()}/latest-v2.json", @@ -258,7 +258,7 @@ public static partial class Command using var client = new HttpClient(); var req = new HttpRequestMessage(HttpMethod.Post, "https://forum.duplicati.com/posts"); - var discourseToken = File.ReadAllText(Program.Configuration.ConfigFiles.DiscourseTokenFile).Trim().Split(":", 2); + var discourseToken = File.ReadAllText(rtcfg.Configuration.ConfigFiles.DiscourseTokenFile).Trim().Split(":", 2); req.Headers.Add("Api-Username", discourseToken[0]); req.Headers.Add("Api-Key", discourseToken[1]); req.Headers.Add("Accept", "application/json"); diff --git a/ReleaseBuilder/Build/Command.cs b/ReleaseBuilder/Build/Command.cs index 46ea9f935..ce95610bd 100644 --- a/ReleaseBuilder/Build/Command.cs +++ b/ReleaseBuilder/Build/Command.cs @@ -394,6 +394,7 @@ public static partial class Command static async Task DoBuild(CommandInput input) { Console.WriteLine($"Building {input.Channel} release ..."); + var configuration = Configuration.Create(input.Channel); var buildTargets = input.Targets; @@ -404,15 +405,15 @@ public static partial class Command throw new FileNotFoundException($"Solution file not found: {input.SolutionFile.FullName}"); // This could be fixed, so we will throw an exception if the build is not possible - if (buildTargets.Any(x => x.Package == PackageType.MSI) && !Program.Configuration.IsMSIBuildPossible()) + if (buildTargets.Any(x => x.Package == PackageType.MSI) && !configuration.IsMSIBuildPossible()) throw new Exception("WiX toolset not configured, cannot build MSI files"); // This will be fixed in the future, but requires a new http-interface for Synology DSM - if (buildTargets.Any(x => x.Package == PackageType.SynologySpk) && !Program.Configuration.IsSynologyPkgPossible()) + if (buildTargets.Any(x => x.Package == PackageType.SynologySpk) && !configuration.IsSynologyPkgPossible()) throw new Exception("Synology SPK files are currently not supported"); // This will not work, so to make it easier for non-MacOS developers, we will remove the MacOS packages - if (buildTargets.Any(x => x.Package == PackageType.MacPkg || x.Package == PackageType.DMG) && !Program.Configuration.IsMacPkgBuildPossible()) + if (buildTargets.Any(x => x.Package == PackageType.MacPkg || x.Package == PackageType.DMG) && !configuration.IsMacPkgBuildPossible()) { Console.WriteLine("MacOS packages requested but not running on MacOS, removing from build targets"); buildTargets = buildTargets.Where(x => x.Package != PackageType.MacPkg && x.Package != PackageType.DMG).ToArray(); @@ -481,13 +482,14 @@ public static partial class Command if (string.IsNullOrWhiteSpace(keyfilePassword)) keyfilePassword = ConsoleHelper.ReadPassword("Enter keyfile password"); - var primarySignKey = LoadKeyFile(Program.Configuration.ConfigFiles.UpdaterKeyfile.FirstOrDefault(), keyfilePassword, false); - var additionalKeys = Program.Configuration.ConfigFiles.UpdaterKeyfile + var primarySignKey = LoadKeyFile(configuration.ConfigFiles.UpdaterKeyfile.FirstOrDefault(), keyfilePassword, false); + var additionalKeys = configuration.ConfigFiles.UpdaterKeyfile .Skip(1) .Select(x => LoadKeyFile(x, keyfilePassword, true)); // Configure runtime environment var rtcfg = new RuntimeConfig( + configuration, releaseInfo, additionalKeys.Prepend(primarySignKey).ToList(), keyfilePassword, @@ -590,11 +592,11 @@ public static partial class Command }), Path.Combine(input.BuildPath.FullName, "packages"), version: releaseInfo.Version.ToString(), - incompatibleUpdateUrl: ReplaceVersionPlaceholders(Program.Configuration.ExtraSettings.UpdateFromIncompatibleVersionUrl, releaseInfo), - genericUpdatePageUrl: ReplaceVersionPlaceholders(Program.Configuration.ExtraSettings.GenericUpdatePageUrl, releaseInfo), + incompatibleUpdateUrl: ReplaceVersionPlaceholders(configuration.ExtraSettings.UpdateFromIncompatibleVersionUrl, releaseInfo), + genericUpdatePageUrl: ReplaceVersionPlaceholders(configuration.ExtraSettings.GenericUpdatePageUrl, releaseInfo), releaseType: releaseInfo.Channel.ToString().ToLowerInvariant(), packages: builtPackages.Select(x => new PackageEntry( - RemoteUrls: Program.Configuration.ExtraSettings.PackageUrls + RemoteUrls: configuration.ExtraSettings.PackageUrls .Select(u => ReplaceVersionPlaceholders(u, releaseInfo) .Replace("${FILENAME}", HttpUtility.UrlEncode(Path.GetFileName(x.CreatedFile))) @@ -784,7 +786,7 @@ public static partial class Command /// Modified files static Task> PrepareSourceDirectory(string baseDir, ReleaseInfo releaseInfo, RuntimeConfig rtcfg) { - var urlstring = string.Join(";", Program.Configuration.ExtraSettings.UpdaterUrls.Select(x => + var urlstring = string.Join(";", rtcfg.Configuration.ExtraSettings.UpdaterUrls.Select(x => ReplaceVersionPlaceholders(x, releaseInfo) .Replace("${FILENAME}", HttpUtility.UrlEncode("latest-v2.manifest")) )); @@ -832,8 +834,8 @@ public static partial class Command JsonSerializer.Serialize(new { Displayname = $"Duplicati v{releaseInfo.Version} - {releaseInfo.Channel}" }), Path.Combine(baseDir, "Duplicati", "Library", "AutoUpdater"), version: releaseInfo.Version.ToString(), - incompatibleUpdateUrl: ReplaceVersionPlaceholders(Program.Configuration.ExtraSettings.UpdateFromIncompatibleVersionUrl, releaseInfo), - genericUpdatePageUrl: ReplaceVersionPlaceholders(Program.Configuration.ExtraSettings.GenericUpdatePageUrl, releaseInfo), + incompatibleUpdateUrl: ReplaceVersionPlaceholders(rtcfg.Configuration.ExtraSettings.UpdateFromIncompatibleVersionUrl, releaseInfo), + genericUpdatePageUrl: ReplaceVersionPlaceholders(rtcfg.Configuration.ExtraSettings.GenericUpdatePageUrl, releaseInfo), releaseType: releaseInfo.Channel.ToString().ToLowerInvariant() ); @@ -868,7 +870,7 @@ public static partial class Command foreach (var target in targets) { - if (string.IsNullOrWhiteSpace(Program.Configuration.Commands.Npm)) + if (string.IsNullOrWhiteSpace(rtcfg.Configuration.Commands.Npm)) throw new Exception("NPM command not found, but required for building"); // Remove existing node_modules folder @@ -886,7 +888,7 @@ public static partial class Command EnvHelper.CopyDirectory(target, tmp, true); // Run npm install in the temporary folder - await ProcessHelper.Execute(new[] { Program.Configuration.Commands.Npm, "ci" }, workingDirectory: tmp); + await ProcessHelper.Execute(new[] { rtcfg.Configuration.Commands.Npm, "ci" }, workingDirectory: tmp); var basefolder = Directory.EnumerateDirectories(Path.Combine(tmp, "node_modules"), "*", SearchOption.AllDirectories) .Where(x => File.Exists(Path.Combine(x, "index.html"))) .OrderBy(x => x.Length) diff --git a/ReleaseBuilder/Configuration.cs b/ReleaseBuilder/Configuration.cs index 879843f10..6eb48c298 100644 --- a/ReleaseBuilder/Configuration.cs +++ b/ReleaseBuilder/Configuration.cs @@ -68,10 +68,11 @@ public record Configuration( /// /// Creates a new /// + /// The release channel /// The new configuration - public static Configuration Create() + public static Configuration Create(ReleaseChannel channel) => new( - ConfigFiles.Create(), + ConfigFiles.Create(channel), Commands.Create(), ExtraSettings.Create() ); @@ -283,12 +284,28 @@ public record ConfigFiles( /// /// Generates a new config files instance /// + /// The release channel /// The config files instance - public static ConfigFiles Create() + public static ConfigFiles Create(ReleaseChannel channel) { - ParseEnvironmentFile(ExpandEnv("BUILD_SETTINGS_FILE", "${HOME}/.config/build-settings")); - ParseEnvironmentFile(ExpandEnv("GATEKEEPER_SETTINGS_FILE", "${HOME}/.config/signkeys/Duplicati/macos-gatekeeper")); + // Grab the shared configuration + ParseEnvironmentFile(ExpandEnv("BUILD_SETTINGS_FILE", "${HOME}/.config/duplicati-build-settings")); + + // Override the configuration for the release type, if any + ParseEnvironmentFile(ExpandEnv(channel switch + { + ReleaseChannel.Stable or + ReleaseChannel.Beta or + ReleaseChannel.Experimental or + ReleaseChannel.Canary => "${HOME}/.config/signkeys/Duplicati/release-build-settings", + ReleaseChannel.Nightly => "${HOME}/.config/signkeys/Duplicati/nightly-build-settings-nightly", + ReleaseChannel.Debug => "${HOME}/.config/signkeys/Duplicati/debug-build-settings", + _ => throw new ArgumentOutOfRangeException(nameof(channel)) + })); + + // Override the configuration for the release channel, if any + ParseEnvironmentFile(ExpandEnv($"${{HOME}}/.config/signkeys/Duplicati/${channel.ToString().ToLowerInvariant()}-build-settings")); return new( ExpandEnv("UPDATER_KEYFILE", "${HOME}/.config/signkeys/Duplicati/updater-release.key").Split(Path.PathSeparator, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries), diff --git a/ReleaseBuilder/EnvHelper.cs b/ReleaseBuilder/EnvHelper.cs index c05e662b6..3a1ad185c 100644 --- a/ReleaseBuilder/EnvHelper.cs +++ b/ReleaseBuilder/EnvHelper.cs @@ -41,11 +41,20 @@ public static class EnvHelper if (string.IsNullOrWhiteSpace(value)) value = defaultValue ?? string.Empty; + return ExpandEnv(value); + } + + /// + /// Reads the environment key, and expands environment variables inside. + /// If no key is found, the default value is returned + /// + /// The key to use + /// The expanded string + public static string ExpandEnv(string value) // Bash-style env expansion "${name}", done after normal env expansion - return Regex.Replace(Environment.ExpandEnvironmentVariables(value), "\\${(?[^}]+)}", m => + => Regex.Replace(Environment.ExpandEnvironmentVariables(value), "\\${(?[^}]+)}", m => Environment.GetEnvironmentVariable(m.Groups["name"].Value) ?? string.Empty ); - } /// /// Reads the environment key, and expands environment variables inside. diff --git a/ReleaseBuilder/Program.cs b/ReleaseBuilder/Program.cs index 81ac16f80..f410d52c0 100644 --- a/ReleaseBuilder/Program.cs +++ b/ReleaseBuilder/Program.cs @@ -1,23 +1,23 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. using System.CommandLine; @@ -88,11 +88,6 @@ class Program .Distinct() .ToList(); - /// - /// The environment shared configuration - /// - public static readonly Configuration Configuration = Configuration.Create(); - /// /// The return code of the application; shared state /// From d5eba03c39be40e0b35245d79cc4c83a2c1a309e Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 25 Feb 2025 15:07:39 +0100 Subject: [PATCH 234/362] Deep signing MacOS --- ReleaseBuilder/ProcessRunner.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/ReleaseBuilder/ProcessRunner.cs b/ReleaseBuilder/ProcessRunner.cs index edc9cd509..1cb245598 100644 --- a/ReleaseBuilder/ProcessRunner.cs +++ b/ReleaseBuilder/ProcessRunner.cs @@ -83,6 +83,7 @@ public static class ProcessRunner public static Task MacOSCodeSign(string codesign, string codesignIdentity, string entitlementFile, string file) => ProcessHelper.Execute([ codesign, + "--deep", "--force", "--timestamp", "--options=runtime", From 90677fc124aee10b3b9013e97ded4068b4d3cdf5 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 25 Feb 2025 15:14:15 +0100 Subject: [PATCH 235/362] Missing update --- ReleaseBuilder/CreateKey/Command.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ReleaseBuilder/CreateKey/Command.cs b/ReleaseBuilder/CreateKey/Command.cs index bc0dca2d3..8443896d7 100644 --- a/ReleaseBuilder/CreateKey/Command.cs +++ b/ReleaseBuilder/CreateKey/Command.cs @@ -32,7 +32,7 @@ public static class Command var keyfileArgument = new Argument( name: "keyfile", description: "Path to keyfile to use for signing release manifests", - getDefaultValue: () => new FileInfo(Program.Configuration.ConfigFiles.UpdaterKeyfile.FirstOrDefault() ?? "./signkey.key") + getDefaultValue: () => new FileInfo(Configuration.Create(ReleaseChannel.Debug).ConfigFiles.UpdaterKeyfile.FirstOrDefault() ?? "./signkey.key") ); var command = new System.CommandLine.Command("create-key", "Creates a new key for signing releases") From 0b61e0d0bc549348884bb951ee1f35ee1122d153 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 25 Feb 2025 15:14:37 +0100 Subject: [PATCH 236/362] Yet another way to find out if the RID is for Windows --- .../Builtin/Duplicati.Library.Modules.Builtin.csproj | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/Duplicati/Library/Modules/Builtin/Duplicati.Library.Modules.Builtin.csproj b/Duplicati/Library/Modules/Builtin/Duplicati.Library.Modules.Builtin.csproj index bed29c7a7..e3406f9fa 100644 --- a/Duplicati/Library/Modules/Builtin/Duplicati.Library.Modules.Builtin.csproj +++ b/Duplicati/Library/Modules/Builtin/Duplicati.Library.Modules.Builtin.csproj @@ -24,19 +24,13 @@ - - - $([System.String]::Copy('$(RuntimeIdentifier)')) - $([System.String]::Substring('$(RuntimeIdentifier)', 0, 4)) - - - + PreserveNewest - + PreserveNewest From 123c12e2c303a20f7f4cb46330c281a00b814efa Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 25 Feb 2025 16:18:13 +0100 Subject: [PATCH 237/362] Whitespace --- Duplicati/Library/Main/Options.cs | 30 +++++++++++++++--------------- Duplicati/Library/Main/Strings.cs | 30 +++++++++++++++--------------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index c0d681aaa..d27d8d04b 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs index 6ee298520..0eb2af090 100644 --- a/Duplicati/Library/Main/Strings.cs +++ b/Duplicati/Library/Main/Strings.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using Duplicati.Library.Localization.Short; From c5ebf4fa71a5b34f5143afd6a6eabd9c489acadd Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 25 Feb 2025 16:19:04 +0100 Subject: [PATCH 238/362] Added the restore channel depth option --- .../Main/Operation/Restore/Channels.cs | 2 +- .../Library/Main/Operation/RestoreHandler.cs | 1 + Duplicati/Library/Main/Options.cs | 20 +++++++++++++++++++ Duplicati/Library/Main/Strings.cs | 2 ++ 4 files changed, 24 insertions(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Operation/Restore/Channels.cs b/Duplicati/Library/Main/Operation/Restore/Channels.cs index 9b83773da..9ed844e59 100644 --- a/Duplicati/Library/Main/Operation/Restore/Channels.cs +++ b/Duplicati/Library/Main/Operation/Restore/Channels.cs @@ -35,7 +35,7 @@ namespace Duplicati.Library.Main.Operation.Restore /// The buffer size for the channels. The buffer size is the number of /// messages that can be queued up before the sender blocks. /// - public static readonly int BufferSize = 8; + public static int BufferSize; /// /// Channel between and . diff --git a/Duplicati/Library/Main/Operation/RestoreHandler.cs b/Duplicati/Library/Main/Operation/RestoreHandler.cs index 829e91715..736eea5b1 100644 --- a/Duplicati/Library/Main/Operation/RestoreHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreHandler.cs @@ -331,6 +331,7 @@ namespace Duplicati.Library.Main.Operation using var setup_log_timer = new Logging.Timer(LOGTAG, "RestoreNetworkSetup", "RestoreNetworkSetup"); // Create the channels between BlockManager and FileProcessor + Restore.Channels.BufferSize = m_options.RestoreChannelBufferSize; var fileprocessor_requests = new Channel[m_options.RestoreFileProcessors].Select(_ => ChannelManager.CreateChannel(buffersize: Restore.Channels.BufferSize)).ToArray(); var fileprocessor_responses = new Channel[m_options.RestoreFileProcessors].Select(_ => ChannelManager.CreateChannel(buffersize: Restore.Channels.BufferSize)).ToArray(); diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index d27d8d04b..df43370c2 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -129,6 +129,11 @@ namespace Duplicati.Library.Main /// private readonly int DEFAULT_RESTORE_VOLUME_DOWNLOADERS = Math.Max(1, Environment.ProcessorCount / 2); + /// + /// The default value for the size of the channel buffers during restore + /// + private readonly int DEFAULT_RESTORE_CHANNEL_BUFFER_SIZE = 8; + /// /// An enumeration that describes the supported strategies for an optimization /// @@ -474,6 +479,7 @@ namespace Duplicati.Library.Main new CommandLineArgument("restore-volume-decompressors", CommandLineArgument.ArgumentType.Integer, Strings.Options.RestoreVolumeDecompressorsShort, Strings.Options.RestoreVolumeDecompressorsLong, DEFAULT_RESTORE_VOLUME_DECOMPRESSORS.ToString()), new CommandLineArgument("restore-volume-decryptors", CommandLineArgument.ArgumentType.Integer, Strings.Options.RestoreVolumeDecryptorsShort, Strings.Options.RestoreVolumeDecryptorsLong, DEFAULT_RESTORE_VOLUME_DECRYPTORS.ToString()), new CommandLineArgument("restore-volume-downloaders", CommandLineArgument.ArgumentType.Integer, Strings.Options.RestoreVolumeDownloadersShort, Strings.Options.RestoreVolumeDownloadersLong, DEFAULT_RESTORE_VOLUME_DOWNLOADERS.ToString()), + new CommandLineArgument("restore-channel-buffer-size", CommandLineArgument.ArgumentType.Integer, Strings.Options.RestoreChannelBufferSizeShort, Strings.Options.RestoreChannelBufferSizeLong, DEFAULT_RESTORE_CHANNEL_BUFFER_SIZE.ToString()), new CommandLineArgument("internal-profiling", CommandLineArgument.ArgumentType.Boolean, Strings.Options.InternalProfilingShort, Strings.Options.InternalProfilingLong, "false"), }); @@ -2191,6 +2197,20 @@ namespace Duplicati.Library.Main } } + public int RestoreChannelBufferSize + { + get + { + if (!m_options.TryGetValue("restore-channel-buffer-size", out string v)) + v = null; + + if (string.IsNullOrEmpty(v)) + return DEFAULT_RESTORE_CHANNEL_BUFFER_SIZE; + else + return int.Parse(v); + } + } + /// /// Toggles whether internal profiling is enabled and should be logged. /// diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs index 0eb2af090..6894b66eb 100644 --- a/Duplicati/Library/Main/Strings.cs +++ b/Duplicati/Library/Main/Strings.cs @@ -345,6 +345,8 @@ namespace Duplicati.Library.Main.Strings public static string RestoreVolumeDecryptorsLong { get { return LC.L("Use this option to set the number of concurrent FileDecryptor processes used during restore. A FileDecryptor processes one volume at a time, and increasing the number of FileDecryptors may improve restore performance if the bottleneck is decryption."); } } public static string RestoreVolumeDownloadersShort { get { return LC.L("Number of concurrent FileDownloader processes used during restore"); } } public static string RestoreVolumeDownloadersLong { get { return LC.L("Use this option to set the number of concurrent FileDownloader processes used during restore. A FileDownloader processes one volume at a time, and increasing the number of FileDownloaders may improve restore performance if the bottleneck is downloading."); } } + public static string RestoreChannelBufferSizeShort { get { return LC.L("Size of buffers of the channels used during restore"); } } + public static string RestoreChannelBufferSizeLong { get { return LC.L("Use this option to set the size of the buffers of the channels used during restore. The buffers are used to allow for better asynchronous communication between the processes in the restore flow. Increasing the buffer size may improve restore performance."); } } public static string InternalProfilingShort { get { return LC.L("Enable internal profiling"); } } public static string InternalProfilingLong { get { return LC.L("Use this option to enable internal profiling. Profiling is used to measure the performance of the internal code. The profiling data is written to the log file and can be used to identify performance bottlenecks."); } } } From 3a2468483e3e650974f0dc3a71553cf325c3c58a Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 25 Feb 2025 16:20:56 +0100 Subject: [PATCH 239/362] Made process helper ignore empty/null arguments --- ReleaseBuilder/EnvHelper.cs | 2 +- ReleaseBuilder/ProcessHelper.cs | 49 +++++++++++++++++++++------------ ReleaseBuilder/ProcessRunner.cs | 7 ++--- 3 files changed, 35 insertions(+), 23 deletions(-) diff --git a/ReleaseBuilder/EnvHelper.cs b/ReleaseBuilder/EnvHelper.cs index 3a1ad185c..faadddbba 100644 --- a/ReleaseBuilder/EnvHelper.cs +++ b/ReleaseBuilder/EnvHelper.cs @@ -206,7 +206,7 @@ public static class EnvHelper var targetEntry = Path.GetFileName(path); // Use docker to set the ownership - await ProcessHelper.Execute(new[] { "docker", "run", "--mount", $"type=bind,source={baseFolder},target=/opt/mount", "alpine:latest", "chown", recursive ? "-R" : "", $"{uid}:{gid}", Path.Combine("/opt/mount", targetEntry) }); + await ProcessHelper.Execute(["docker", "run", "--mount", $"type=bind,source={baseFolder},target=/opt/mount", "alpine:latest", "chown", recursive ? "-R" : "", $"{uid}:{gid}", Path.Combine("/opt/mount", targetEntry)]); } /// diff --git a/ReleaseBuilder/ProcessHelper.cs b/ReleaseBuilder/ProcessHelper.cs index 7a6f74e45..b5b67a26d 100644 --- a/ReleaseBuilder/ProcessHelper.cs +++ b/ReleaseBuilder/ProcessHelper.cs @@ -38,10 +38,14 @@ public static class ProcessHelper /// If true, stderr is not forwarded to the console /// Function to write to stdin /// An awaitable task - public static async Task Execute(IEnumerable command, string? workingDirectory = null, CancellationToken cancellationToken = default, Func? codeIsError = null, bool suppressStdErr = false, Func? writeStdIn = null) + public static async Task Execute(IEnumerable command, string? workingDirectory = null, CancellationToken cancellationToken = default, Func? codeIsError = null, bool suppressStdErr = false, Func? writeStdIn = null) { if (!command.Any()) throw new ArgumentException("Needs at least one command", nameof(command)); + var executable = command.First(); + if (string.IsNullOrWhiteSpace(executable)) + throw new ArgumentException("Executable name cannot be empty", nameof(command)); + workingDirectory ??= Environment.CurrentDirectory; if (!Directory.Exists(workingDirectory)) @@ -49,7 +53,7 @@ public static class ProcessHelper codeIsError ??= (x) => x != 0; - var p = Process.Start(new ProcessStartInfo(command.First(), command.Skip(1)) + var p = Process.Start(new ProcessStartInfo(executable, command.Skip(1).Where(x => !string.IsNullOrEmpty(x)).Select(x => x!)) { WindowStyle = ProcessWindowStyle.Hidden, WorkingDirectory = workingDirectory, @@ -57,7 +61,7 @@ public static class ProcessHelper RedirectStandardOutput = false, RedirectStandardInput = writeStdIn != null, UseShellExecute = false, - }) ?? throw new Exception($"Failed to launch process {command.First()}, null returned"); + }) ?? throw new Exception($"Failed to launch process {executable}, null returned"); // Forward error messages to stderr var t = suppressStdErr @@ -69,7 +73,7 @@ public static class ProcessHelper await p.WaitForExitAsync(cancellationToken).ConfigureAwait(false); if (codeIsError(p.ExitCode)) - throw new Exception($"Execution of {command.First()} gave error code {p.ExitCode}"); + throw new Exception($"Execution of {executable} gave error code {p.ExitCode}"); await t.ConfigureAwait(false); } @@ -84,7 +88,7 @@ public static class ProcessHelper /// Default value is null which will treat anything non-zero as an error /// If true, stderr is not forwarded to the console /// An awaitable task - public static async Task ExecuteAll(IEnumerable> commands, string? workingDirectory = null, CancellationToken cancellationToken = default, Func? codeIsError = null, bool suppressStdErr = false) + public static async Task ExecuteAll(IEnumerable> commands, string? workingDirectory = null, CancellationToken cancellationToken = default, Func? codeIsError = null, bool suppressStdErr = false) { foreach (var c in commands) await Execute(c, workingDirectory, cancellationToken, codeIsError, suppressStdErr).ConfigureAwait(false); @@ -101,10 +105,13 @@ public static class ProcessHelper /// If true, stderr is not forwarded to the console /// Function to write to stdin /// The output from stdout - public static async Task ExecuteWithOutput(IEnumerable command, string? workingDirectory = null, CancellationToken cancellationToken = default, Func? codeIsError = null, bool suppressStdErr = false, Func? writeStdIn = null) + public static async Task ExecuteWithOutput(IEnumerable command, string? workingDirectory = null, CancellationToken cancellationToken = default, Func? codeIsError = null, bool suppressStdErr = false, Func? writeStdIn = null) { if (!command.Any()) throw new ArgumentException("Needs at least one command", nameof(command)); + var executable = command.First(); + if (string.IsNullOrWhiteSpace(executable)) + throw new ArgumentException("Executable name cannot be empty", nameof(command)); workingDirectory ??= Environment.CurrentDirectory; if (!Directory.Exists(workingDirectory)) @@ -112,7 +119,7 @@ public static class ProcessHelper codeIsError ??= (x) => x != 0; - var p = Process.Start(new ProcessStartInfo(command.First(), command.Skip(1)) + var p = Process.Start(new ProcessStartInfo(executable, command.Skip(1).Where(x => !string.IsNullOrEmpty(x)).Select(x => x!)) { WindowStyle = ProcessWindowStyle.Hidden, WorkingDirectory = workingDirectory, @@ -120,7 +127,7 @@ public static class ProcessHelper RedirectStandardOutput = true, RedirectStandardInput = writeStdIn != null, UseShellExecute = false, - }) ?? throw new Exception($"Failed to launch process {command.First()}, null returned"); + }) ?? throw new Exception($"Failed to launch process {executable}, null returned"); var tstdout = p.StandardOutput.ReadToEndAsync(cancellationToken); var tstderr = suppressStdErr @@ -132,7 +139,7 @@ public static class ProcessHelper await p.WaitForExitAsync(cancellationToken).ConfigureAwait(false); if (codeIsError(p.ExitCode)) - throw new Exception($"Execution of {command.First()} gave error code {p.ExitCode}"); + throw new Exception($"Execution of {executable} gave error code {p.ExitCode}"); await tstderr.ConfigureAwait(false); return await tstdout.ConfigureAwait(false); @@ -151,10 +158,13 @@ public static class ProcessHelper /// If true, stderr is not forwarded to the console /// Function to write to stdin /// The output from stdout - public static async Task ExecuteWithOutput(IEnumerable command, Stream stdout, string? workingDirectory = null, CancellationToken cancellationToken = default, Func? codeIsError = null, bool suppressStdErr = false, Func? writeStdIn = null) + public static async Task ExecuteWithOutput(IEnumerable command, Stream stdout, string? workingDirectory = null, CancellationToken cancellationToken = default, Func? codeIsError = null, bool suppressStdErr = false, Func? writeStdIn = null) { if (!command.Any()) throw new ArgumentException("Needs at least one command", nameof(command)); + var executable = command.First(); + if (string.IsNullOrWhiteSpace(executable)) + throw new ArgumentException("Executable name cannot be empty", nameof(command)); workingDirectory ??= Environment.CurrentDirectory; if (!Directory.Exists(workingDirectory)) @@ -162,7 +172,7 @@ public static class ProcessHelper codeIsError ??= (x) => x != 0; - var p = Process.Start(new ProcessStartInfo(command.First(), command.Skip(1)) + var p = Process.Start(new ProcessStartInfo(executable, command.Skip(1).Where(x => !string.IsNullOrWhiteSpace(x)).Select(x => x!)) { WindowStyle = ProcessWindowStyle.Hidden, WorkingDirectory = workingDirectory, @@ -170,7 +180,7 @@ public static class ProcessHelper RedirectStandardOutput = true, RedirectStandardInput = writeStdIn != null, UseShellExecute = false, - }) ?? throw new Exception($"Failed to launch process {command.First()}, null returned"); + }) ?? throw new Exception($"Failed to launch process {executable}, null returned"); var tstdout = p.StandardOutput.BaseStream.CopyToAsync(stdout, cancellationToken); var tstderr = suppressStdErr @@ -182,7 +192,7 @@ public static class ProcessHelper await p.WaitForExitAsync(cancellationToken).ConfigureAwait(false); if (codeIsError(p.ExitCode)) - throw new Exception($"Execution of {command.First()} gave error code {p.ExitCode}"); + throw new Exception($"Execution of {executable} gave error code {p.ExitCode}"); await tstderr.ConfigureAwait(false); await tstdout.ConfigureAwait(false); @@ -200,10 +210,13 @@ public static class ProcessHelper /// Default value is null which will treat anything non-zero as an error /// Function to write to stdin /// The output from stdout - public static async Task ExecuteWithLog(IEnumerable command, string? workingDirectory = null, CancellationToken cancellationToken = default, Func? codeIsError = null, string? logFolder = null, Func? logFilename = null, Func? writeStdIn = null) + public static async Task ExecuteWithLog(IEnumerable command, string? workingDirectory = null, CancellationToken cancellationToken = default, Func? codeIsError = null, string? logFolder = null, Func? logFilename = null, Func? writeStdIn = null) { if (!command.Any()) throw new ArgumentException("Needs at least one command", nameof(command)); + var executable = command.First(); + if (string.IsNullOrWhiteSpace(executable)) + throw new ArgumentException("Executable name cannot be empty", nameof(command)); workingDirectory ??= Environment.CurrentDirectory; if (!Directory.Exists(workingDirectory)) @@ -213,7 +226,7 @@ public static class ProcessHelper codeIsError ??= (x) => x != 0; - var p = Process.Start(new ProcessStartInfo(command.First(), command.Skip(1)) + var p = Process.Start(new ProcessStartInfo(executable, command.Skip(1).Where(x => !string.IsNullOrWhiteSpace(x)).Select(x => x!)) { WindowStyle = ProcessWindowStyle.Hidden, WorkingDirectory = workingDirectory, @@ -221,9 +234,9 @@ public static class ProcessHelper RedirectStandardOutput = true, RedirectStandardInput = writeStdIn != null, UseShellExecute = false, - }) ?? throw new Exception($"Failed to launch process {command.First()}, null returned"); + }) ?? throw new Exception($"Failed to launch process {executable}, null returned"); - logFilename ??= (pid, isStdOut) => $"{command.First()}-{p.Id}.{(isStdOut ? "stdout" : "stderr")}.log"; + logFilename ??= (pid, isStdOut) => $"{executable}-{p.Id}.{(isStdOut ? "stdout" : "stderr")}.log"; using var logstdout = File.Create(Path.Combine(logFolder, logFilename(p.Id, true))); using var logstderr = File.Create(Path.Combine(logFolder, logFilename(p.Id, false))); @@ -236,7 +249,7 @@ public static class ProcessHelper await p.WaitForExitAsync(cancellationToken).ConfigureAwait(false); if (codeIsError(p.ExitCode)) - throw new Exception($"Execution of {command.First()} gave error code {p.ExitCode}"); + throw new Exception($"Execution of {executable} gave error code {p.ExitCode}"); await t1.ConfigureAwait(false); await t2.ConfigureAwait(false); diff --git a/ReleaseBuilder/ProcessRunner.cs b/ReleaseBuilder/ProcessRunner.cs index 1cb245598..910bd0dba 100644 --- a/ReleaseBuilder/ProcessRunner.cs +++ b/ReleaseBuilder/ProcessRunner.cs @@ -52,20 +52,19 @@ public static class ProcessRunner var tmp = Path.GetTempFileName(); File.Delete(tmp); - var args = new[] { + await ProcessHelper.Execute([ osslsigncode, "sign", "-pkcs12", pfxfile, "-pass", pfxpassword, "-n", OSSLOrganization, "-i", OSSLUrl, "-h", hashalg, - first ? "" : "-nest", + first ? null : "-nest", "-t", $"http://timestamp.digicert.com?alg={hashalg}", "-in", executable, "-out", tmp - }; + ]); - await ProcessHelper.Execute(args.Where(x => !string.IsNullOrWhiteSpace(x))); File.Move(tmp, executable, true); first = false; From b86794791b64ba8e283e73b1fbe75c6038654256 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 25 Feb 2025 16:35:46 +0100 Subject: [PATCH 240/362] Reallocating byte[] buffer takes up too long time if done for each file --- .../Main/Operation/Restore/FileProcessor.cs | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs index abef8218f..241ed6d27 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs @@ -77,6 +77,7 @@ namespace Duplicati.Library.Main.Operation.Restore { using var filehasher = HashFactory.CreateHasher(options.FileHashAlgorithm); using var blockhasher = HashFactory.CreateHasher(options.BlockHashAlgorithm); + var buffer = new byte[options.Blocksize]; while (true) { @@ -93,7 +94,7 @@ namespace Duplicati.Library.Main.Operation.Restore sw_work?.Start(); // Verify the target file blocks that may already exist. - var (bytes_verified, missing_blocks, verified_blocks) = await VerifyTargetBlocks(file, blocks, filehasher, blockhasher, options, results).ConfigureAwait(false); + var (bytes_verified, missing_blocks, verified_blocks) = await VerifyTargetBlocks(file, blocks, filehasher, blockhasher, buffer, options, results).ConfigureAwait(false); long bytes_written = 0; if (blocks.Length != missing_blocks.Count + verified_blocks.Count) { @@ -124,7 +125,7 @@ namespace Duplicati.Library.Main.Operation.Restore } else { - await CopyOldTargetBlocksToNewTarget(file, new_file, verified_blocks).ConfigureAwait(false); + await CopyOldTargetBlocksToNewTarget(file, new_file, buffer, verified_blocks).ConfigureAwait(false); } } else @@ -148,7 +149,7 @@ namespace Duplicati.Library.Main.Operation.Restore if (missing_blocks.Count > 0 && options.UseLocalBlocks) { // Verify the local blocks at the original restore path that may be used to restore the file. - (bytes_written, missing_blocks) = await VerifyLocalBlocks(file, missing_blocks, blocks.Length, filehasher, blockhasher, options, results, block_request).ConfigureAwait(false); + (bytes_written, missing_blocks) = await VerifyLocalBlocks(file, missing_blocks, blocks.Length, filehasher, blockhasher, buffer, options, results, block_request).ConfigureAwait(false); } bool empty_file_or_symlink = false; @@ -368,7 +369,7 @@ namespace Duplicati.Library.Main.Operation.Restore /// The old target file. /// The new target file. /// The blocks in the old file that were verified. - private static async Task CopyOldTargetBlocksToNewTarget(FileRequest old_file, FileRequest new_file, List verified_blocks) + private static async Task CopyOldTargetBlocksToNewTarget(FileRequest old_file, FileRequest new_file, byte[] buffer, List verified_blocks) { using var fs_old = SystemIO.IO_OS.FileOpenRead(old_file.TargetPath); using var fs_new = SystemIO.IO_OS.FileOpenWrite(new_file.TargetPath); @@ -378,7 +379,6 @@ namespace Duplicati.Library.Main.Operation.Restore fs_old.Seek(block.BlockOffset * block.BlockSize, SeekOrigin.Begin); fs_new.Seek(block.BlockOffset * block.BlockSize, SeekOrigin.Begin); - var buffer = new byte[block.BlockSize]; await fs_old.ReadAsync(buffer, 0, buffer.Length).ConfigureAwait(false); await fs_new.WriteAsync(buffer, 0, buffer.Length).ConfigureAwait(false); ; } @@ -493,7 +493,7 @@ namespace Duplicati.Library.Main.Operation.Restore /// The Duplicati configuration options. /// The restoration results. /// An awaitable `Task`, which returns a collection of data blocks that are missing. - private static async Task<(long, List, List)> VerifyTargetBlocks(FileRequest file, BlockRequest[] blocks, System.Security.Cryptography.HashAlgorithm filehasher, System.Security.Cryptography.HashAlgorithm blockhasher, Options options, RestoreResults results) + private static async Task<(long, List, List)> VerifyTargetBlocks(FileRequest file, BlockRequest[] blocks, System.Security.Cryptography.HashAlgorithm filehasher, System.Security.Cryptography.HashAlgorithm blockhasher, byte[] buffer, Options options, RestoreResults results) { long bytes_read = 0; List missing_blocks = []; @@ -506,7 +506,6 @@ namespace Duplicati.Library.Main.Operation.Restore try { using var f = SystemIO.IO_OS.FileOpenRead(file.TargetPath); - var buffer = new byte[options.Blocksize]; for (int i = 0; i < blocks.Length; i++) { var read = await f.ReadAsync(buffer, 0, (int)blocks[i].BlockSize).ConfigureAwait(false); @@ -595,7 +594,7 @@ namespace Duplicati.Library.Main.Operation.Restore /// The restoration results. /// The channel to request blocks from the block manager. Used to inform the block manager which blocks are already present. /// An awaitable `Task`, which returns a collection of data blocks that are missing. - private static async Task<(long, List)> VerifyLocalBlocks(FileRequest file, List blocks, long total_blocks, System.Security.Cryptography.HashAlgorithm filehasher, System.Security.Cryptography.HashAlgorithm blockhasher, Options options, RestoreResults results, IChannel block_request) + private static async Task<(long, List)> VerifyLocalBlocks(FileRequest file, List blocks, long total_blocks, System.Security.Cryptography.HashAlgorithm filehasher, System.Security.Cryptography.HashAlgorithm blockhasher, byte[] buffer, Options options, RestoreResults results, IChannel block_request) { if (file.TargetPath == file.OriginalPath) { @@ -618,7 +617,6 @@ namespace Duplicati.Library.Main.Operation.Restore SystemIO.IO_OS.FileOpenRead(file.TargetPath) : null) : SystemIO.IO_OS.FileOpenReadWrite(file.TargetPath); - var buffer = new byte[options.Blocksize]; long bytes_read = 0; long bytes_written = 0; From b62a1f760073a51b1f7b14a8b4067ff1d15f45c9 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 25 Feb 2025 16:36:30 +0100 Subject: [PATCH 241/362] Added missing comments in filelister --- Duplicati/Library/Main/Operation/Restore/FileLister.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Operation/Restore/FileLister.cs b/Duplicati/Library/Main/Operation/Restore/FileLister.cs index 15e7cc18b..c400f8a53 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileLister.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileLister.cs @@ -43,6 +43,7 @@ namespace Duplicati.Library.Main.Operation.Restore /// Runs the file lister process that lists the files that need to be restored /// and sends them to the . /// + /// The named channels for the restore operation. /// The restore database, which is queried for the file list. /// The restore options /// The restore results @@ -60,7 +61,7 @@ namespace Duplicati.Library.Main.Operation.Restore try { sw_prework?.Start(); - var files = db.GetFilesAndSymlinksToRestore(true).OrderByDescending(x => x.Length).ToArray(); + var files = db.GetFilesAndSymlinksToRestore(true).OrderByDescending(x => x.Length).ToArray(); // Get started on big files first result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_DownloadingRemoteFiles); sw_prework?.Stop(); From 5c7a9337220906caf4db3dbc5f068e08de3b8d3a Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 25 Feb 2025 16:37:49 +0100 Subject: [PATCH 242/362] FileLister did not report its internal profling properly --- .../Main/Operation/Restore/FileLister.cs | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/FileLister.cs b/Duplicati/Library/Main/Operation/Restore/FileLister.cs index c400f8a53..6da3be7e6 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileLister.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileLister.cs @@ -58,6 +58,9 @@ namespace Duplicati.Library.Main.Operation.Restore { Stopwatch sw_prework = options.InternalProfiling ? new() : null; Stopwatch sw_write = options.InternalProfiling ? new() : null; + + bool threw_exception = false; + try { sw_prework?.Start(); @@ -67,26 +70,25 @@ namespace Duplicati.Library.Main.Operation.Restore sw_write?.Start(); foreach (var file in files) - { await self.Output.WriteAsync(file).ConfigureAwait(false); - } sw_write?.Stop(); } - catch (RetiredException) - { - Logging.Log.WriteVerboseMessage(LOGTAG, "RetiredProcess", null, "File lister retired"); - - if (options.InternalProfiling) - { - Logging.Log.WriteProfilingMessage(LOGTAG, "InternalTimings", $"Prework: {sw_prework.ElapsedMilliseconds}ms, Write: {sw_write.ElapsedMilliseconds}ms"); - Console.WriteLine($"File lister - Prework: {sw_prework.ElapsedMilliseconds}ms, Write: {sw_write.ElapsedMilliseconds}ms"); - } - } catch (Exception ex) { Logging.Log.WriteErrorMessage(LOGTAG, "FileListerError", ex, "Error during file listing"); + threw_exception = true; throw; } + finally + { + if (!threw_exception) + Logging.Log.WriteVerboseMessage(LOGTAG, "RetiredProcess", null, "File lister retired"); + + if (options.InternalProfiling) + { + Logging.Log.WriteProfilingMessage(LOGTAG, "InternalTimings", $"Prework: {sw_prework.ElapsedMilliseconds}ms, Write: {sw_write.ElapsedMilliseconds}ms"); + } + } }); } } From d7b8d13bc03c7e5fef2b6006d3046adb398b2c76 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 25 Feb 2025 16:39:28 +0100 Subject: [PATCH 243/362] VolumeDecryptor and VolumeManager were unnecessarily sending around and keeping track of disposed encrypted volumes --- Duplicati/Library/Main/Operation/Restore/VolumeDecryptor.cs | 2 +- Duplicati/Library/Main/Operation/Restore/VolumeManager.cs | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/VolumeDecryptor.cs b/Duplicati/Library/Main/Operation/Restore/VolumeDecryptor.cs index 2555cbf0e..0b76a9a7a 100644 --- a/Duplicati/Library/Main/Operation/Restore/VolumeDecryptor.cs +++ b/Duplicati/Library/Main/Operation/Restore/VolumeDecryptor.cs @@ -72,7 +72,7 @@ namespace Duplicati.Library.Main.Operation.Restore sw_write?.Start(); // Pass the decrypted volume to the `VolumeDecompressor` process. - await self.Output.WriteAsync((volume_id, volume, bvr)).ConfigureAwait(false); + await self.Output.WriteAsync((volume_id, bvr)).ConfigureAwait(false); sw_write?.Stop(); } } diff --git a/Duplicati/Library/Main/Operation/Restore/VolumeManager.cs b/Duplicati/Library/Main/Operation/Restore/VolumeManager.cs index 0c636973c..842de33d5 100644 --- a/Duplicati/Library/Main/Operation/Restore/VolumeManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/VolumeManager.cs @@ -59,7 +59,6 @@ namespace Duplicati.Library.Main.Operation.Restore async self => { Dictionary cache = []; - Dictionary files = []; Dictionary> in_flight = []; Stopwatch sw_cache_set = options.InternalProfiling ? new() : null; @@ -84,8 +83,6 @@ namespace Duplicati.Library.Main.Operation.Restore sw_cache_evict?.Start(); cache.Remove(request.VolumeID, out var reader); reader?.Dispose(); - files.Remove(request.VolumeID, out var tempfile); - tempfile?.Dispose(); sw_cache_evict?.Stop(); } else @@ -111,11 +108,10 @@ namespace Duplicati.Library.Main.Operation.Restore } break; } - case (long volume_id, TempFile temp_file, BlockVolumeReader reader): + case (long volume_id, BlockVolumeReader reader): { sw_cache_set?.Start(); cache[volume_id] = reader; - files[volume_id] = temp_file; sw_cache_set?.Stop(); sw_wakeup?.Start(); foreach (var request in in_flight[volume_id]) From 616d8b75a725b3e4122b2eb7ad326cb45b63ffdb Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 25 Feb 2025 16:26:32 +0100 Subject: [PATCH 244/362] Simplified MacOS code signing --- ReleaseBuilder/Build/Command.Compile.Post.cs | 9 +---- ReleaseBuilder/Build/Command.CreatePackage.cs | 35 ++++++------------- ReleaseBuilder/Build/Command.GitPush.cs | 16 ++++----- .../Build/Command.PackageSupport.cs | 2 +- ReleaseBuilder/Build/Command.RuntimeConfig.cs | 6 ++-- ReleaseBuilder/Build/Command.cs | 2 +- ReleaseBuilder/ProcessRunner.cs | 5 +-- 7 files changed, 28 insertions(+), 47 deletions(-) diff --git a/ReleaseBuilder/Build/Command.Compile.Post.cs b/ReleaseBuilder/Build/Command.Compile.Post.cs index 4f88d29e6..d33b195b0 100644 --- a/ReleaseBuilder/Build/Command.Compile.Post.cs +++ b/ReleaseBuilder/Build/Command.Compile.Post.cs @@ -248,14 +248,7 @@ public static partial class Command var licenseTarget = Path.Combine(tmpApp, "Contents", "Licenses"); Directory.Move(Path.Combine(binDir, "licenses"), licenseTarget); - // Apply code signing, if requested - if (rtcfg.UseCodeSignSigning) - { - var entitlementFile = Path.Combine(resourcesDir, "Entitlements.plist"); - await PackageSupport.SignMacOSBinaries(rtcfg, binDir, entitlementFile); - await rtcfg.Codesign(tmpApp, entitlementFile); - } - + // Make files executable if (!OperatingSystem.IsWindows()) foreach (var f in Directory.EnumerateFiles(binDir, "*.launchagent.plist", SearchOption.TopDirectoryOnly)) File.SetUnixFileMode(f, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead | UnixFileMode.OtherRead); diff --git a/ReleaseBuilder/Build/Command.CreatePackage.cs b/ReleaseBuilder/Build/Command.CreatePackage.cs index 9d28302d7..5dce8e11e 100644 --- a/ReleaseBuilder/Build/Command.CreatePackage.cs +++ b/ReleaseBuilder/Build/Command.CreatePackage.cs @@ -364,7 +364,7 @@ public static partial class Command /// The package target to create the file for /// The runtime config /// An awaitable task - static async Task PrepareAndReSignAppBundle(string appFolder, string installerDir, PackageTarget target, RuntimeConfig rtcfg) + static async Task PrepareAndSignAppBundle(string appFolder, string installerDir, PackageTarget target, RuntimeConfig rtcfg) { await PackageSupport.InstallPackageIdentifier(Path.Combine(appFolder, "Contents", "MacOS"), target); @@ -372,15 +372,8 @@ public static partial class Command if (rtcfg.UseCodeSignSigning) { var entitlementFile = Path.Combine(installerDir, "Entitlements.plist"); - var updates = new[] { Path.Combine(appFolder, "Contents", "MacOS", "package_type_id.txt") } - .Concat( - ExecutableRenames.Values.Select(x => Path.Combine(appFolder, "Contents", "MacOS", x)) - .Where(File.Exists) - ) - .Append(appFolder); - - foreach (var x in updates) - await rtcfg.Codesign(x, entitlementFile); + await PackageSupport.SignMacOSBinaries(rtcfg, appFolder, entitlementFile); + await rtcfg.Codesign(appFolder, true, entitlementFile); } } @@ -398,9 +391,8 @@ public static partial class Command var mountDir = Path.Combine(buildRoot, "mount"); if (Directory.Exists(mountDir)) { - await ProcessHelper.Execute([ - "hdiutil", "detach", mountDir, "-quiet", "-force", - ], workingDirectory: buildRoot, codeIsError: _ => false); + await ProcessHelper.Execute(["hdiutil", "detach", mountDir, "-quiet", "-force",], + workingDirectory: buildRoot, codeIsError: _ => false); Directory.Delete(mountDir, false); } @@ -433,9 +425,7 @@ public static partial class Command // Change the dmg name var dmgname = $"Duplicati {rtcfg.ReleaseInfo.ReleaseName}"; Console.WriteLine($"Setting dmg name to {dmgname}"); - await ProcessHelper.Execute([ - "diskutil", "quiet", "rename", mountDir, dmgname - ], workingDirectory: mountDir); + await ProcessHelper.Execute(["diskutil", "quiet", "rename", mountDir, dmgname], workingDirectory: mountDir); // Make the Duplicati.app structure, root folder should exist var appFolder = Path.Combine(mountDir, rtcfg.MacOSAppName); @@ -444,7 +434,7 @@ public static partial class Command // Place the prepared folder EnvHelper.CopyDirectory(Path.Combine(buildRoot, $"{target.BuildTargetString}-{rtcfg.MacOSAppName}"), appFolder, recursive: true); - await PrepareAndReSignAppBundle(appFolder, resourcesDir, target, rtcfg); + await PrepareAndSignAppBundle(appFolder, resourcesDir, target, rtcfg); // Set permissions inside DMG file if (!OperatingSystem.IsWindows()) @@ -460,8 +450,7 @@ public static partial class Command File.Delete(templateDmg); Directory.Delete(mountDir, false); - if (rtcfg.UseCodeSignSigning) - await rtcfg.Codesign(dmgFile, Path.Combine(resourcesDir, "Entitlements.plist")); + await rtcfg.Codesign(dmgFile, false, Path.Combine(resourcesDir, "Entitlements.plist")); } /// @@ -488,7 +477,7 @@ public static partial class Command // Place the prepared folder EnvHelper.CopyDirectory(Path.Combine(buildRoot, $"{target.BuildTargetString}-{rtcfg.MacOSAppName}"), appFolder, recursive: true); - await PrepareAndReSignAppBundle(appFolder, installerDir, target, rtcfg); + await PrepareAndSignAppBundle(appFolder, installerDir, target, rtcfg); // Copy the source script files var scripts = new[] { "daemon", "daemon-scripts", "app-scripts" }; @@ -662,11 +651,7 @@ public static partial class Command } // Apply code signing, if requested - if (rtcfg.UseCodeSignSigning) - { - var entitlementFile = Path.Combine(installerDir, "Entitlements.plist"); - await PackageSupport.SignMacOSBinaries(rtcfg, binFolder, entitlementFile); - } + await PackageSupport.SignMacOSBinaries(rtcfg, binFolder, Path.Combine(installerDir, "Entitlements.plist")); var payloadPkgFile = target.Interface switch { diff --git a/ReleaseBuilder/Build/Command.GitPush.cs b/ReleaseBuilder/Build/Command.GitPush.cs index 7b7a1b4fd..0ccbc207c 100644 --- a/ReleaseBuilder/Build/Command.GitPush.cs +++ b/ReleaseBuilder/Build/Command.GitPush.cs @@ -39,29 +39,29 @@ public static partial class Command File.WriteAllText(Path.Combine(baseDir, "ReleaseBuilder", "build_version.txt"), releaseInfo.Version.ToString()); // Add modified files - await ProcessHelper.Execute(new[] { + await ProcessHelper.Execute([ "git", "add", "ReleaseBuilder/build_version.txt", "changelog.txt" - }, workingDirectory: baseDir); + ], workingDirectory: baseDir); // Make a commit - await ProcessHelper.Execute(new[] { + await ProcessHelper.Execute([ "git", "commit", "-m", $"Version bump to v{releaseInfo.Version}-{releaseInfo.ReleaseName}", "-m", "You can download this build from: ", "-m", $"Binaries: https://updates.duplicati.com/{releaseInfo.Channel.ToString().ToLowerInvariant()}/?version={releaseInfo.Version}", "-m", $"Signature file: https://updates.duplicati.com/{releaseInfo.Channel.ToString().ToLowerInvariant()}/duplicati-{releaseInfo.ReleaseName}.signatures.zip" - }, workingDirectory: baseDir); + ], workingDirectory: baseDir); // And tag the release - await ProcessHelper.Execute(new[] { + await ProcessHelper.Execute([ "git", "tag", $"v{releaseInfo.Version}-{releaseInfo.ReleaseName}", - }, workingDirectory: baseDir); + ], workingDirectory: baseDir); // Then push the release - await ProcessHelper.Execute(new[] { "git", "push", "-u", "origin", "head" }, workingDirectory: baseDir); - await ProcessHelper.Execute(new[] { "git", "push", "--tags" }, workingDirectory: baseDir); + await ProcessHelper.Execute(["git", "push", "-u", "origin", "head"], workingDirectory: baseDir); + await ProcessHelper.Execute(["git", "push", "--tags"], workingDirectory: baseDir); } } } diff --git a/ReleaseBuilder/Build/Command.PackageSupport.cs b/ReleaseBuilder/Build/Command.PackageSupport.cs index aa085e1b5..39567a09b 100644 --- a/ReleaseBuilder/Build/Command.PackageSupport.cs +++ b/ReleaseBuilder/Build/Command.PackageSupport.cs @@ -115,7 +115,7 @@ public static partial class Command .ToList(); foreach (var f in signtargets) - await rtcfg.Codesign(f, entitlementFile); + await rtcfg.Codesign(f, false, entitlementFile); } } } diff --git a/ReleaseBuilder/Build/Command.RuntimeConfig.cs b/ReleaseBuilder/Build/Command.RuntimeConfig.cs index 20ed732ce..7693cbab4 100644 --- a/ReleaseBuilder/Build/Command.RuntimeConfig.cs +++ b/ReleaseBuilder/Build/Command.RuntimeConfig.cs @@ -473,15 +473,17 @@ public static partial class Command /// Performs codesign on the given file /// /// The file to sign + /// A value indicating if deep signing should be used /// The entitlements to apply /// An awaitable task - public Task Codesign(string file, string entitlements) + public Task Codesign(string file, bool deep, string entitlements) => UseCodeSignSigning ? ProcessRunner.MacOSCodeSign( Configuration.Commands.Codesign!, Configuration.ConfigFiles.CodesignIdentity, entitlements, - file + file, + deep ) : Task.CompletedTask; diff --git a/ReleaseBuilder/Build/Command.cs b/ReleaseBuilder/Build/Command.cs index ce95610bd..cd7d009bb 100644 --- a/ReleaseBuilder/Build/Command.cs +++ b/ReleaseBuilder/Build/Command.cs @@ -888,7 +888,7 @@ public static partial class Command EnvHelper.CopyDirectory(target, tmp, true); // Run npm install in the temporary folder - await ProcessHelper.Execute(new[] { rtcfg.Configuration.Commands.Npm, "ci" }, workingDirectory: tmp); + await ProcessHelper.Execute([rtcfg.Configuration.Commands.Npm, "ci"], workingDirectory: tmp); var basefolder = Directory.EnumerateDirectories(Path.Combine(tmp, "node_modules"), "*", SearchOption.AllDirectories) .Where(x => File.Exists(Path.Combine(x, "index.html"))) .OrderBy(x => x.Length) diff --git a/ReleaseBuilder/ProcessRunner.cs b/ReleaseBuilder/ProcessRunner.cs index 910bd0dba..9127839b4 100644 --- a/ReleaseBuilder/ProcessRunner.cs +++ b/ReleaseBuilder/ProcessRunner.cs @@ -78,11 +78,12 @@ public static class ProcessRunner /// The identity used for codesign /// The entitlements to activate for the file /// The file to sign + /// Whether to sign deeply /// An awaitable task - public static Task MacOSCodeSign(string codesign, string codesignIdentity, string entitlementFile, string file) + public static Task MacOSCodeSign(string codesign, string codesignIdentity, string entitlementFile, string file, bool deep) => ProcessHelper.Execute([ codesign, - "--deep", + (deep ? "--deep" : null), "--force", "--timestamp", "--options=runtime", From e773907eed4d0794c5f55a59d90150d1480439f5 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 25 Feb 2025 16:45:55 +0100 Subject: [PATCH 245/362] Changed from allocating a new byte[] buffer in the decompressor to use the Buffers.ArrayPool --- .../Main/Operation/Restore/BlockManager.cs | 22 ++++++++-------- .../Main/Operation/Restore/FileProcessor.cs | 25 +++++++++++++------ .../Operation/Restore/VolumeDecompressor.cs | 3 ++- 3 files changed, 31 insertions(+), 19 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs index 6ee1ea0a5..155d6bf21 100644 --- a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs @@ -20,6 +20,7 @@ // DEALINGS IN THE SOFTWARE. using System; +using System.Buffers; using System.Collections.Concurrent; using System.Collections.Generic; using System.Data; @@ -143,11 +144,12 @@ namespace Duplicati.Library.Main.Operation.Restore /// /// The block request to check. /// - public void CheckCounts(BlockRequest blockRequest) + public async Task CheckCounts(BlockRequest blockRequest) { long error_block_id = -1; long error_volume_id = -1; var emit_evict = false; + byte[] data = null; lock (m_blockcount_lock) { @@ -163,6 +165,7 @@ namespace Duplicati.Library.Main.Operation.Restore { // Evict the block from the cache and check if the volume is no longer needed. m_blockcount.Remove(blockRequest.BlockID); + data = m_block_cache?.Get(blockRequest.BlockID); m_block_cache?.Remove(blockRequest.BlockID); } else // block_count < 0 @@ -177,7 +180,6 @@ namespace Duplicati.Library.Main.Operation.Restore } else if (vol_count == 0) { - // Notify the `VolumeManager` that it should evict the volume. m_volumecount.Remove(blockRequest.VolumeID); blockRequest.CacheDecrEvict = true; emit_evict = true; @@ -189,8 +191,12 @@ namespace Duplicati.Library.Main.Operation.Restore sw_checkcounts?.Stop(); } + if (data != null) + ArrayPool.Shared.Return(data); + + // Notify the `VolumeManager` that it should evict the volume. if (emit_evict) - m_volume_request.Write(blockRequest); + await m_volume_request.WriteAsync(blockRequest).ConfigureAwait(false); if (error_block_id != -1) { @@ -215,7 +221,6 @@ namespace Duplicati.Library.Main.Operation.Restore // Check if the block is already in the cache, and return it if it is. if (m_block_cache != null && m_block_cache.TryGetValue(block_request.BlockID, out byte[] value)) { - CheckCounts(block_request); return Task.FromResult(value); } @@ -230,12 +235,7 @@ namespace Duplicati.Library.Main.Operation.Restore } sw_get_wait?.Stop(); - return new_tcs.Task.ContinueWith(t => - { - if (t.IsCompletedSuccessfully) - CheckCounts(block_request); - return t.Result; - }); + return new_tcs.Task; } /// @@ -405,7 +405,7 @@ namespace Duplicati.Library.Main.Operation.Restore { sw_cache?.Start(); // Target file already had the block. - cache.CheckCounts(block_request); + await cache.CheckCounts(block_request).ConfigureAwait(false); sw_cache?.Stop(); } else diff --git a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs index 241ed6d27..84e0b1e96 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs @@ -244,6 +244,7 @@ namespace Duplicati.Library.Main.Operation.Restore sw_resp?.Start(); var data = await block_response.ReadAsync().ConfigureAwait(false); sw_resp?.Stop(); + sw_req?.Start(); if (j < missing_blocks.Count - burst) { @@ -253,7 +254,7 @@ namespace Duplicati.Library.Main.Operation.Restore sw_work?.Start(); // Hash the block to verify the file hash - filehasher.TransformBlock(data, 0, data.Length, data, 0); + filehasher.TransformBlock(data, 0, (int)blocks[i].BlockSize, data, 0); if (options.Dryrun) { // Simulate writing the block @@ -262,19 +263,22 @@ namespace Duplicati.Library.Main.Operation.Restore else { // Write the block to the file - await fs.WriteAsync(data).ConfigureAwait(false); + await fs.WriteAsync(data, 0, (int)blocks[i].BlockSize).ConfigureAwait(false); } // Keep track of metrics - bytes_written += data.Length; + bytes_written += blocks[i].BlockSize; j++; + sw_req?.Start(); + blocks[i].CacheDecrEvict = true; + await block_request.WriteAsync(blocks[i]).ConfigureAwait(false); + sw_req?.Stop(); } else { // Not a missing block, so read from the file - var data = new byte[blocks[i].BlockSize]; - var read = await fs?.ReadAsync(data, 0, data.Length); - filehasher.TransformBlock(data, 0, read, data, 0); + var read = await fs?.ReadAsync(buffer, 0, buffer.Length); + filehasher.TransformBlock(buffer, 0, read, buffer, 0); } } @@ -472,11 +476,18 @@ namespace Duplicati.Library.Main.Operation.Restore sw_req?.Start(); await block_request.WriteAsync(block).ConfigureAwait(false); sw_req?.Stop(); + sw_resp?.Start(); var data = await block_response.ReadAsync().ConfigureAwait(false); sw_resp?.Stop(); + sw_work?.Start(); - ms.Write(data, 0, data.Length); + ms.Write(data, 0, (int)block.BlockSize); + + sw_req?.Start(); + block.CacheDecrEvict = true; + await block_request.WriteAsync(block).ConfigureAwait(false); + sw_req?.Stop(); } ms.Seek(0, SeekOrigin.Begin); diff --git a/Duplicati/Library/Main/Operation/Restore/VolumeDecompressor.cs b/Duplicati/Library/Main/Operation/Restore/VolumeDecompressor.cs index a1f4afa0f..161660425 100644 --- a/Duplicati/Library/Main/Operation/Restore/VolumeDecompressor.cs +++ b/Duplicati/Library/Main/Operation/Restore/VolumeDecompressor.cs @@ -20,6 +20,7 @@ // DEALINGS IN THE SOFTWARE. using System; +using System.Buffers; using System.Diagnostics; using System.Threading.Tasks; using CoCoL; @@ -69,7 +70,7 @@ namespace Duplicati.Library.Main.Operation.Restore sw_read?.Stop(); sw_decompress?.Start(); - var data = new byte[block_request.BlockSize]; + var data = ArrayPool.Shared.Rent(options.Blocksize); lock (volume_reader) // The BlockVolumeReader is not thread-safe { volume_reader.ReadBlock(block_request.BlockHash, data); From bf3a26ffcddc8e6fcfa3242a54c6f29f4df6b905 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 25 Feb 2025 16:49:46 +0100 Subject: [PATCH 246/362] Added more internal profiling timers to the FileProcessor to get more high-resolution profiling --- .../Main/Operation/Restore/FileProcessor.cs | 52 +++++++++++++++---- 1 file changed, 41 insertions(+), 11 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs index 84e0b1e96..dfadaf805 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs @@ -72,6 +72,16 @@ namespace Duplicati.Library.Main.Operation.Restore Stopwatch sw_req = options.InternalProfiling ? new() : null; Stopwatch sw_resp = options.InternalProfiling ? new() : null; Stopwatch sw_work = options.InternalProfiling ? new() : null; + Stopwatch sw_work_verify_target = options.InternalProfiling ? new() : null; + Stopwatch sw_work_retarget = options.InternalProfiling ? new() : null; + Stopwatch sw_work_notify_local = options.InternalProfiling ? new() : null; + Stopwatch sw_work_verify_local = options.InternalProfiling ? new() : null; + Stopwatch sw_work_empty_file = options.InternalProfiling ? new() : null; + Stopwatch sw_work_hash = options.InternalProfiling ? new() : null; + Stopwatch sw_work_read = options.InternalProfiling ? new() : null; + Stopwatch sw_work_write = options.InternalProfiling ? new() : null; + Stopwatch sw_work_results = options.InternalProfiling ? new() : null; + Stopwatch sw_work_meta = options.InternalProfiling ? new() : null; try { @@ -92,17 +102,18 @@ namespace Duplicati.Library.Main.Operation.Restore var blocks = db.GetBlocksFromFile(file.BlocksetID).ToArray(); sw_block?.Stop(); - sw_work?.Start(); + sw_work_verify_target?.Start(); // Verify the target file blocks that may already exist. var (bytes_verified, missing_blocks, verified_blocks) = await VerifyTargetBlocks(file, blocks, filehasher, blockhasher, buffer, options, results).ConfigureAwait(false); long bytes_written = 0; + sw_work_verify_target?.Stop(); if (blocks.Length != missing_blocks.Count + verified_blocks.Count) { Logging.Log.WriteErrorMessage(LOGTAG, "BlockCountMismatch", null, $"Block count mismatch for {file.TargetPath} - expected: {blocks.Length}, actual: {missing_blocks.Count + verified_blocks.Count}"); - sw_work?.Stop(); continue; } + sw_work_retarget?.Start(); // Check if the target file needs to be retargeted if (missing_blocks.Count > 0 && !options.Overwrite && SystemIO.IO_OS.FileExists(file.TargetPath)) { @@ -136,7 +147,9 @@ namespace Duplicati.Library.Main.Operation.Restore file = new_file; } } + sw_work_retarget?.Stop(); + sw_work_notify_local?.Start(); // Notify to the cache that we use local blocks. foreach (var block in verified_blocks) { @@ -145,16 +158,20 @@ namespace Duplicati.Library.Main.Operation.Restore await block_request.WriteAsync(block).ConfigureAwait(false); sw_req?.Stop(); } + sw_work_notify_local?.Stop(); + sw_work_verify_local?.Start(); if (missing_blocks.Count > 0 && options.UseLocalBlocks) { // Verify the local blocks at the original restore path that may be used to restore the file. (bytes_written, missing_blocks) = await VerifyLocalBlocks(file, missing_blocks, blocks.Length, filehasher, blockhasher, buffer, options, results, block_request).ConfigureAwait(false); } + sw_work_verify_local?.Stop(); bool empty_file_or_symlink = false; if (file.BlocksetID != LocalDatabase.SYMLINK_BLOCKSET_ID && (blocks.Length == 0 || (blocks.Length == 1 && blocks[0].BlockSize == 0))) { + sw_work_empty_file?.Start(); empty_file_or_symlink = true; if (options.Dryrun) { @@ -178,6 +195,7 @@ namespace Duplicati.Library.Main.Operation.Restore await block_request.WriteAsync(blocks[0]).ConfigureAwait(false); } } + sw_work_empty_file?.Stop(); } else if (missing_blocks.Count > 0) { @@ -187,7 +205,9 @@ namespace Duplicati.Library.Main.Operation.Restore continue; } + sw_work_hash?.Start(); filehasher.Initialize(); + sw_work_hash?.Stop(); FileStream fs = null; try @@ -216,7 +236,6 @@ namespace Duplicati.Library.Main.Operation.Restore fs = SystemIO.IO_OS.FileOpenReadWrite(file.TargetPath); } - sw_work?.Stop(); sw_req?.Start(); // Burst the block requests to speed up the restore int burst = Channels.BufferSize; @@ -233,14 +252,12 @@ namespace Duplicati.Library.Main.Operation.Restore fs.SetLength(file.Length); } - sw_work?.Start(); for (int i = 0; i < blocks.Length; i++) { // Each response block is not verified against the block hash, as the processes downloading the blocks should do that. if (j < missing_blocks.Count && missing_blocks[j].BlockOffset == i) { // Read the block from the response and issue a new request, if more blocks are missing - sw_work?.Stop(); sw_resp?.Start(); var data = await block_response.ReadAsync().ConfigureAwait(false); sw_resp?.Stop(); @@ -251,10 +268,11 @@ namespace Duplicati.Library.Main.Operation.Restore await block_request.WriteAsync(missing_blocks[j + burst]).ConfigureAwait(false); } sw_req?.Stop(); - sw_work?.Start(); // Hash the block to verify the file hash + sw_work_hash?.Start(); filehasher.TransformBlock(data, 0, (int)blocks[i].BlockSize, data, 0); + sw_work_hash?.Stop(); if (options.Dryrun) { // Simulate writing the block @@ -263,7 +281,9 @@ namespace Duplicati.Library.Main.Operation.Restore else { // Write the block to the file + sw_work_write?.Start(); await fs.WriteAsync(data, 0, (int)blocks[i].BlockSize).ConfigureAwait(false); + sw_work_write?.Stop(); } // Keep track of metrics @@ -277,8 +297,12 @@ namespace Duplicati.Library.Main.Operation.Restore else { // Not a missing block, so read from the file + sw_work_read?.Start(); var read = await fs?.ReadAsync(buffer, 0, buffer.Length); + sw_work_read?.Stop(); + sw_work_hash?.Start(); filehasher.TransformBlock(buffer, 0, read, buffer, 0); + sw_work_hash?.Stop(); } } @@ -288,7 +312,11 @@ namespace Duplicati.Library.Main.Operation.Restore } // Verify the file hash + sw_work_hash?.Start(); filehasher.TransformFinalBlock([], 0, 0); + sw_work_hash?.Stop(); + + sw_work?.Start(); if (Convert.ToBase64String(filehasher.Hash) != file.Hash) { Logging.Log.WriteErrorMessage(LOGTAG, "FileHashMismatch", null, $"File hash mismatch for {file.TargetPath} - expected: {file.Hash}, actual: {Convert.ToBase64String(filehasher.Hash)}"); @@ -307,6 +335,7 @@ namespace Duplicati.Library.Main.Operation.Restore fs.SetLength(file.Length); } } + sw_work?.Stop(); } catch (Exception) { @@ -326,10 +355,13 @@ namespace Duplicati.Library.Main.Operation.Restore if (!options.SkipMetadata) { - empty_file_or_symlink |= await RestoreMetadata(db, file, block_request, block_response, options, sw_meta, sw_work, sw_req, sw_resp).ConfigureAwait(false); + empty_file_or_symlink |= await RestoreMetadata(db, file, block_request, block_response, options, sw_meta, sw_work_meta, sw_req, sw_resp).ConfigureAwait(false); + sw_work_meta?.Stop(); } // TODO legacy restore doesn't count metadata restore as a restored file. + + sw_work_results?.Start(); if (empty_file_or_symlink || bytes_written > 0) { // Keep track of the restored files and their sizes @@ -339,7 +371,7 @@ namespace Duplicati.Library.Main.Operation.Restore results.SizeOfRestoredFiles += bytes_written; } } - sw_work?.Stop(); + sw_work_results?.Stop(); Logging.Log.WriteVerboseMessage(LOGTAG, "RestoredFile", "Restored file {0}", file.TargetPath); } @@ -350,7 +382,7 @@ namespace Duplicati.Library.Main.Operation.Restore if (options.InternalProfiling) { - Logging.Log.WriteProfilingMessage(LOGTAG, "InternalTimings", $"File: {sw_file.ElapsedMilliseconds}ms, Block: {sw_block.ElapsedMilliseconds}ms, Meta: {sw_meta.ElapsedMilliseconds}ms, Req: {sw_req.ElapsedMilliseconds}ms, Resp: {sw_resp.ElapsedMilliseconds}ms, Work: {sw_work.ElapsedMilliseconds}ms"); + Logging.Log.WriteProfilingMessage(LOGTAG, "InternalTimings", $"File: {sw_file.ElapsedMilliseconds}ms, Block: {sw_block.ElapsedMilliseconds}ms, Meta: {sw_meta.ElapsedMilliseconds}ms, Req: {sw_req.ElapsedMilliseconds}ms, Resp: {sw_resp.ElapsedMilliseconds}ms, Work: {sw_work.ElapsedMilliseconds}ms, VerifyTarget: {sw_work_verify_target.ElapsedMilliseconds}ms, Retarget: {sw_work_retarget.ElapsedMilliseconds}ms, NotifyLocal: {sw_work_notify_local.ElapsedMilliseconds}ms, VerifyLocal: {sw_work_verify_local.ElapsedMilliseconds}ms, EmptyFile: {sw_work_empty_file.ElapsedMilliseconds}ms, Hash: {sw_work_hash.ElapsedMilliseconds}ms, Read: {sw_work_read.ElapsedMilliseconds}ms, Write: {sw_work_write.ElapsedMilliseconds}ms, Results: {sw_work_results.ElapsedMilliseconds}ms, Meta: {sw_work_meta.ElapsedMilliseconds}ms"); } return; } @@ -462,11 +494,9 @@ namespace Duplicati.Library.Main.Operation.Restore /// The stopwatch for internal profiling of the block responses. private static async Task RestoreMetadata(LocalRestoreDatabase db, FileRequest file, IChannel block_request, IChannel block_response, Options options, Stopwatch sw_meta, Stopwatch sw_work, Stopwatch sw_req, Stopwatch sw_resp) { - sw_work?.Stop(); sw_meta?.Start(); var blocks = db.GetMetadataBlocksFromFile(file.ID).ToArray(); sw_meta?.Stop(); - sw_work?.Start(); using var ms = new MemoryStream(); From 42ec5b633cab490ea7200754dc13531c802148b3 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 25 Feb 2025 17:41:35 +0100 Subject: [PATCH 247/362] Restore block cache max option didn't match the description --- Duplicati/Library/Main/Options.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index c0d681aaa..3b539d932 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -2071,7 +2071,7 @@ namespace Duplicati.Library.Main if (max_cache > 0 && max_cache < Blocksize) throw new ArgumentOutOfRangeException(nameof(max_cache), string.Format("The maximum cache size cannot be less than the blocksize if not explicitly 0: {0} < {1}", max_cache, Blocksize)); - return max_cache; + return max_cache / Blocksize; } } From 6d7faf97bb77b15288b8cf4aaa3b14429f01bd03 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 25 Feb 2025 17:41:57 +0100 Subject: [PATCH 248/362] Whitespace --- .../Main/Database/LocalRestoreDatabase.cs | 30 +++++++++---------- Duplicati/Library/Main/Options.cs | 30 +++++++++---------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index fde1e870f..039f861b6 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index 3b539d932..b1b2894f9 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; From a49c66bcb5d508112031fc6c1304992b7a7ff956 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 25 Feb 2025 17:42:30 +0100 Subject: [PATCH 249/362] Added caching of localrestoredatabase data --- .../Main/Database/LocalRestoreDatabase.cs | 63 +++++++++++++++++++ .../Main/Operation/Restore/FileLister.cs | 9 ++- .../Main/Operation/Restore/FileProcessor.cs | 6 +- 3 files changed, 73 insertions(+), 5 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index 039f861b6..b95817273 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -19,6 +19,7 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Data; using System.Linq; @@ -43,6 +44,8 @@ namespace Duplicati.Library.Main.Database /// protected string m_tempfiletable; protected string m_tempblocktable; + ConcurrentDictionary block2cache = new(); + ConcurrentDictionary block2metacache = new(); protected string m_latestblocktable; protected string m_fileprogtable; protected string m_totalprogtable; @@ -86,6 +89,10 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) { + //cmd.ExecuteNonQuery("PRAGMA temp_store = MEMORY"); + //cmd.ExecuteNonQuery("PRAGMA journal_mode = OFF"); + //cmd.ExecuteNonQuery("PRAGMA cache_size = -2000000"); + // How to handle METADATA? cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", m_fileprogtable)); cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (" @@ -347,6 +354,58 @@ namespace Duplicati.Library.Main.Database return new Tuple(0, 0); } + public void BuildCaches() + { + using (var cmd = m_connection.CreateCommand()) + { + var blocksetids = cmd.ExecuteReaderEnumerable(@"SELECT ""Blockset"".""ID"" FROM ""Blockset""").Select(r => r.ConvertValueToInt64(0)).ToList(); + cmd.CommandText = @$" + SELECT Block.ID, Block.Hash, Block.Size, Block.VolumeID + FROM BlocksetEntry INNER JOIN Block + ON BlocksetEntry.BlockID = Block.ID + WHERE BlocksetEntry.BlocksetID = ?"; + cmd.AddParameter(); + foreach (var bsid in blocksetids) + { + cmd.SetParameterValue(0, bsid); + using var reader = cmd.ExecuteReader(); + List brlist = []; + for (long i = 0; reader.Read(); i++) + brlist.Add(new BlockRequest(reader.ConvertValueToInt64(0), i, reader.ConvertValueToString(1), reader.ConvertValueToInt64(2), reader.ConvertValueToInt64(3), false)); + block2cache[bsid] = [.. brlist]; + } + Console.WriteLine($"Block cache loaded with {block2cache.Count} entries"); + } + + using (var cmd = m_connection.CreateCommand()) + { + var fileids = cmd.ExecuteReaderEnumerable($@" + SELECT F.ID + FROM ""{m_tempfiletable}"" F + LEFT JOIN Blockset B ON F.BlocksetID = B.ID + WHERE F.BlocksetID != {FOLDER_BLOCKSET_ID}").Select(r => r.ConvertValueToInt64(0)).ToList(); + + cmd.CommandText = $@" + SELECT Block.ID, Block.Hash, Block.Size, Block.VolumeID + FROM ""{m_tempfiletable}"" + INNER JOIN Metadataset ON ""{m_tempfiletable}"".MetadataID = Metadataset.ID + INNER JOIN BlocksetEntry ON Metadataset.BlocksetID = BlocksetEntry.BlocksetID + INNER JOIN Block ON BlocksetEntry.BlockID = Block.ID + WHERE ""{m_tempfiletable}"".ID = ? + "; + cmd.AddParameter(); + foreach (var fileID in fileids) + { + cmd.SetParameterValue(0, fileID); + using var reader = cmd.ExecuteReader(); + List brlist = []; + for (long i = 0; reader.Read(); i++) + brlist.Add(new BlockRequest(reader.ConvertValueToInt64(0), i, reader.ConvertValueToString(1), reader.ConvertValueToInt64(2), reader.ConvertValueToInt64(3), false)); + block2metacache[fileID] = [.. brlist]; + } + } + } + public string GetFirstPath() { using (var cmd = m_connection.CreateCommand()) @@ -1018,6 +1077,8 @@ namespace Duplicati.Library.Main.Database /// A list of needed to restore the given file. public IEnumerable GetBlocksFromFile(long blocksetID) { + //return block2cache[blocksetID]; + using var cmd = m_connection.CreateCommand(); cmd.CommandText = @$" SELECT Block.ID, Block.Hash, Block.Size, Block.VolumeID @@ -1040,6 +1101,8 @@ namespace Duplicati.Library.Main.Database /// A list of needed to restore the metadata of the given file. public IEnumerable GetMetadataBlocksFromFile(long fileID) { + //return block2metacache[fileID]; + using var cmd = m_connection.CreateCommand(); cmd.CommandText = $@" SELECT Block.ID, Block.Hash, Block.Size, Block.VolumeID diff --git a/Duplicati/Library/Main/Operation/Restore/FileLister.cs b/Duplicati/Library/Main/Operation/Restore/FileLister.cs index 6da3be7e6..493665092 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileLister.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileLister.cs @@ -28,7 +28,7 @@ using Duplicati.Library.Main.Database; namespace Duplicati.Library.Main.Operation.Restore { - + // TODO double check docstrings /// /// Process that holds the files that this particular restore operation needs to restore. /// @@ -56,6 +56,7 @@ namespace Duplicati.Library.Main.Operation.Restore }, async self => { + Stopwatch sw_cache = options.InternalProfiling ? new() : null; Stopwatch sw_prework = options.InternalProfiling ? new() : null; Stopwatch sw_write = options.InternalProfiling ? new() : null; @@ -63,6 +64,10 @@ namespace Duplicati.Library.Main.Operation.Restore try { + sw_cache?.Start(); + //db.BuildCaches(); + sw_cache?.Stop(); + sw_prework?.Start(); var files = db.GetFilesAndSymlinksToRestore(true).OrderByDescending(x => x.Length).ToArray(); // Get started on big files first result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_DownloadingRemoteFiles); @@ -86,7 +91,7 @@ namespace Duplicati.Library.Main.Operation.Restore if (options.InternalProfiling) { - Logging.Log.WriteProfilingMessage(LOGTAG, "InternalTimings", $"Prework: {sw_prework.ElapsedMilliseconds}ms, Write: {sw_write.ElapsedMilliseconds}ms"); + Logging.Log.WriteProfilingMessage(LOGTAG, "InternalTimings", $"Cache: {sw_cache.ElapsedMilliseconds}ms, Prework: {sw_prework.ElapsedMilliseconds}ms, Write: {sw_write.ElapsedMilliseconds}ms"); } } }); diff --git a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs index dfadaf805..3e98a0a47 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs @@ -99,7 +99,7 @@ namespace Duplicati.Library.Main.Operation.Restore // Get information about the blocks for the file // TODO rather than keeping all of the blocks in memory, we could do a single pass over the blocks using a cursor, only keeping the relevant block requests in memory. Maybe even only a single block request at a time. sw_block?.Start(); - var blocks = db.GetBlocksFromFile(file.BlocksetID).ToArray(); + var blocks = db.GetBlocksFromFile(file.BlocksetID).ToArray();//.Select(x => new BlockRequest(x.BlockID, x.BlockOffset, x.BlockHash, x.BlockSize, x.VolumeID, false)).ToArray(); sw_block?.Stop(); sw_work_verify_target?.Start(); @@ -282,7 +282,7 @@ namespace Duplicati.Library.Main.Operation.Restore { // Write the block to the file sw_work_write?.Start(); - await fs.WriteAsync(data, 0, (int)blocks[i].BlockSize).ConfigureAwait(false); + await fs.WriteAsync(data.AsMemory(0, (int)blocks[i].BlockSize)).ConfigureAwait(false); sw_work_write?.Stop(); } @@ -495,7 +495,7 @@ namespace Duplicati.Library.Main.Operation.Restore private static async Task RestoreMetadata(LocalRestoreDatabase db, FileRequest file, IChannel block_request, IChannel block_response, Options options, Stopwatch sw_meta, Stopwatch sw_work, Stopwatch sw_req, Stopwatch sw_resp) { sw_meta?.Start(); - var blocks = db.GetMetadataBlocksFromFile(file.ID).ToArray(); + var blocks = db.GetMetadataBlocksFromFile(file.ID);//.Select(x => new BlockRequest(x.BlockID, x.BlockOffset, x.BlockHash, x.BlockSize, x.VolumeID, false)).ToArray(); sw_meta?.Stop(); using var ms = new MemoryStream(); From b1fca9c3aa24dc128ed797fdbd2b2bcecc9dfc68 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 25 Feb 2025 21:54:10 +0100 Subject: [PATCH 250/362] Added logfile name to crash message --- ReleaseBuilder/ProcessHelper.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ReleaseBuilder/ProcessHelper.cs b/ReleaseBuilder/ProcessHelper.cs index b5b67a26d..8486fd2d9 100644 --- a/ReleaseBuilder/ProcessHelper.cs +++ b/ReleaseBuilder/ProcessHelper.cs @@ -249,7 +249,7 @@ public static class ProcessHelper await p.WaitForExitAsync(cancellationToken).ConfigureAwait(false); if (codeIsError(p.ExitCode)) - throw new Exception($"Execution of {executable} gave error code {p.ExitCode}"); + throw new Exception($"Execution of {executable} gave error code {p.ExitCode}, see log file {Path.Combine(logFolder, logFilename(p.Id, true))}"); await t1.ConfigureAwait(false); await t2.ConfigureAwait(false); From e1ae9dd9b4c5dd21eb43d1447d768a2230b1d123 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 25 Feb 2025 21:54:47 +0100 Subject: [PATCH 251/362] Added MacOS signing verification --- ReleaseBuilder/Build/Command.CreatePackage.cs | 7 +++++-- ReleaseBuilder/Build/Command.RuntimeConfig.cs | 13 +++++++++++++ ReleaseBuilder/ProcessRunner.cs | 15 +++++++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/ReleaseBuilder/Build/Command.CreatePackage.cs b/ReleaseBuilder/Build/Command.CreatePackage.cs index 5dce8e11e..21232ab44 100644 --- a/ReleaseBuilder/Build/Command.CreatePackage.cs +++ b/ReleaseBuilder/Build/Command.CreatePackage.cs @@ -372,8 +372,10 @@ public static partial class Command if (rtcfg.UseCodeSignSigning) { var entitlementFile = Path.Combine(installerDir, "Entitlements.plist"); - await PackageSupport.SignMacOSBinaries(rtcfg, appFolder, entitlementFile); - await rtcfg.Codesign(appFolder, true, entitlementFile); + // In principle, it should be enough to deep sign the bundle, but the signtool is broken + await PackageSupport.SignMacOSBinaries(rtcfg, Path.Combine(appFolder, "Contents", "MacOS"), entitlementFile); + await rtcfg.Codesign(appFolder, false, entitlementFile); + await rtcfg.VerifyCodeSign(appFolder); } } @@ -451,6 +453,7 @@ public static partial class Command Directory.Delete(mountDir, false); await rtcfg.Codesign(dmgFile, false, Path.Combine(resourcesDir, "Entitlements.plist")); + await rtcfg.VerifyCodeSign(dmgFile); } /// diff --git a/ReleaseBuilder/Build/Command.RuntimeConfig.cs b/ReleaseBuilder/Build/Command.RuntimeConfig.cs index 7693cbab4..7ee3dc4d3 100644 --- a/ReleaseBuilder/Build/Command.RuntimeConfig.cs +++ b/ReleaseBuilder/Build/Command.RuntimeConfig.cs @@ -487,6 +487,19 @@ public static partial class Command ) : Task.CompletedTask; + /// + /// Verifies the codesign of the given file or app bundle + /// + /// The file to verify + /// An awaitable task + public Task VerifyCodeSign(string file) + => UseCodeSignSigning + ? ProcessRunner.MacOSVerifyCodeSign( + Configuration.Commands.Codesign!, + file + ) + : Task.CompletedTask; + /// /// Performs productsign on the given file /// diff --git a/ReleaseBuilder/ProcessRunner.cs b/ReleaseBuilder/ProcessRunner.cs index 9127839b4..96bd28e7f 100644 --- a/ReleaseBuilder/ProcessRunner.cs +++ b/ReleaseBuilder/ProcessRunner.cs @@ -92,6 +92,21 @@ public static class ProcessRunner file ]); + /// + /// Verifies the MacOS codesign of a file or app bundle + /// + /// The path to the codesign binary + /// The file to verify + /// An awaitable task + public static Task MacOSVerifyCodeSign(string codesign, string file) + => ProcessHelper.Execute([ + codesign, + "--verify", + "--deep", + "--strict", + file + ]); + /// /// Runs MacOS codesign on a single file /// From 1c38061050b1473572109e11648daaf81f100a19 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 26 Feb 2025 09:20:20 +0100 Subject: [PATCH 252/362] Added support for authenticode signing with `jsign` and Yubikey / PIV. --- ReleaseBuilder/Build/Command.RuntimeConfig.cs | 62 +++++++++++++++---- ReleaseBuilder/Build/Command.cs | 9 +++ ReleaseBuilder/Configuration.cs | 26 ++++++-- ReleaseBuilder/ProcessRunner.cs | 43 ++++++++++++- 4 files changed, 122 insertions(+), 18 deletions(-) diff --git a/ReleaseBuilder/Build/Command.RuntimeConfig.cs b/ReleaseBuilder/Build/Command.RuntimeConfig.cs index 7ee3dc4d3..b78a40504 100644 --- a/ReleaseBuilder/Build/Command.RuntimeConfig.cs +++ b/ReleaseBuilder/Build/Command.RuntimeConfig.cs @@ -91,6 +91,11 @@ public static partial class Command /// private bool? _useAuthenticodeSigning; + /// + /// Cache value for checking if jsign tool should be used for authenticode signing + /// + private bool? _useJsignToolForAuthenticode; + /// /// Checks if Authenticode signing should be enabled /// @@ -104,18 +109,39 @@ public static partial class Command return; } - if (Configuration.IsAuthenticodePossible()) - _useAuthenticodeSigning = true; - else + if (Configuration.IsAuthenticodePossibleWithSignTool()) { - if (ConsoleHelper.ReadInput("Configuration missing for osslsigncode, continue without signing executables?", "Y", "n") == "Y") + _useAuthenticodeSigning = true; + _useJsignToolForAuthenticode = false; + return; + } + + if (Configuration.IsAuthenticodePossibleWithJsignTool()) + { + if (string.IsNullOrWhiteSpace(Input.SignkeyPin)) { - _useAuthenticodeSigning = false; - return; + var signKeyPin = EnvHelper.GetEnvKey("SIGNKEY_PIN", ""); + if (string.IsNullOrWhiteSpace(signKeyPin)) + signKeyPin = ConsoleHelper.ReadPassword("Enter the pin for the signing key"); + input = input with { SignkeyPin = signKeyPin }; } - throw new Exception("Configuration is not set up for osslsigncode"); + if (!string.IsNullOrWhiteSpace(Input.SignkeyPin)) + { + _useAuthenticodeSigning = true; + _useJsignToolForAuthenticode = true; + return; + } } + + if (ConsoleHelper.ReadInput("Configuration missing for jsign/signtool/osslsigncode, continue without signing executables?", "Y", "n") == "Y") + { + _useAuthenticodeSigning = false; + _useJsignToolForAuthenticode = false; + return; + } + + throw new Exception("Configuration is not set up for jsign/signtool/osslsigncode"); } } @@ -397,6 +423,11 @@ public static partial class Command /// public bool UseAuthenticodeSigning => _useAuthenticodeSigning!.Value; + /// + /// Returns a value indicating if authenticode signing is enabled + /// + public bool UseJsingToolForAuthenticode => _useJsignToolForAuthenticode!.Value; + /// /// Returns a value indicating if notarize is enabled /// @@ -461,13 +492,20 @@ public static partial class Command /// The file to sign /// An awaitable task public Task AuthenticodeSign(string file) - => UseAuthenticodeSigning - ? ProcessRunner.OsslCodeSign( - Configuration.Commands.OsslSignCode!, + { + if (!UseAuthenticodeSigning) + return Task.CompletedTask; + + if (UseJsingToolForAuthenticode) + return ProcessRunner.JsignCodeSign(Configuration.Commands.JSign!, Input.SignkeyPin, file); + else + return ProcessRunner.OsslCodeSign( + Configuration.Commands.SignCode!, Configuration.ConfigFiles.AuthenticodePfxFile, PfxPassword, - file) - : Task.CompletedTask; + file + ); + } /// /// Performs codesign on the given file diff --git a/ReleaseBuilder/Build/Command.cs b/ReleaseBuilder/Build/Command.cs index cd7d009bb..112fa1984 100644 --- a/ReleaseBuilder/Build/Command.cs +++ b/ReleaseBuilder/Build/Command.cs @@ -222,6 +222,12 @@ public static partial class Command var passwordOption = SharedOptions.passwordOption; + var signkeyPinOption = new Option( + name: "--signkey-pin", + description: "The pin to use for the signing key", + getDefaultValue: () => string.Empty + ); + var disableDockerPushOption = new Option( name: "--disable-docker-push", description: "Disables pushing the docker image to the repository", @@ -317,6 +323,7 @@ public static partial class Command disableAuthenticodeOption, disableCodeSignOption, passwordOption, + signkeyPinOption, macOsAppNameOption, disableDockerPushOption, dockerRepoOption, @@ -351,6 +358,7 @@ public static partial class Command /// If authenticode signing should be disabled /// If signcode should be disabled /// The password to use for the keyfile + /// The pin to use for the signing key /// If the docker push should be disabled /// The name of the MacOS app bundle /// The docker repository to push to @@ -376,6 +384,7 @@ public static partial class Command bool DisableAuthenticode, bool DisableSignCode, string Password, + string SignkeyPin, bool DisableDockerPush, string MacOSAppName, string DockerRepo, diff --git a/ReleaseBuilder/Configuration.cs b/ReleaseBuilder/Configuration.cs index 6eb48c298..b20b5934a 100644 --- a/ReleaseBuilder/Configuration.cs +++ b/ReleaseBuilder/Configuration.cs @@ -77,13 +77,26 @@ public record Configuration( ExtraSettings.Create() ); + /// - /// Checks if signing with authenticode is possible given the current configuration + /// Checks if signing with authenticode using jsign is possible given the current configuration /// /// A boolean indicating if signing is possible - public bool IsAuthenticodePossible() + public bool IsAuthenticodePossibleWithJsignTool() { - if (string.IsNullOrWhiteSpace(ConfigFiles.AuthenticodePasswordFile) || string.IsNullOrWhiteSpace(ConfigFiles.AuthenticodePfxFile) || string.IsNullOrWhiteSpace(Commands.OsslSignCode)) + if (string.IsNullOrWhiteSpace(Commands.JSign)) + return false; + + return true; + } + + /// + /// Checks if signing with authenticode is possible with signtool given the current configuration + /// + /// A boolean indicating if signing is possible + public bool IsAuthenticodePossibleWithSignTool() + { + if (string.IsNullOrWhiteSpace(ConfigFiles.AuthenticodePasswordFile) || string.IsNullOrWhiteSpace(ConfigFiles.AuthenticodePfxFile) || string.IsNullOrWhiteSpace(Commands.SignCode)) return false; if (!File.Exists(ConfigFiles.AuthenticodePasswordFile) || !File.Exists(ConfigFiles.AuthenticodePfxFile)) @@ -329,7 +342,8 @@ public record ConfigFiles( /// The "build" command /// The "gpg" command /// The "aws" command -/// The "osslsigncode" command +/// The "osslsigncode" command +/// The "jsign" command /// The "codesign" command /// The "productsign" command /// The "wix" command @@ -338,7 +352,8 @@ public record ConfigFiles( public record Commands( string Dotnet, string? Gpg, - string? OsslSignCode, + string? SignCode, + string? JSign, string? Codesign, string? Productsign, string? Wix, @@ -355,6 +370,7 @@ public record Commands( FindCommand("dotnet", "DOTNET") ?? throw new Exception("Failed to find the \"dotnet\" command"), FindCommand("gpg2", "GPG", FindCommand("gpg", "GPG")), FindCommand(OperatingSystem.IsWindows() ? "signtool.exe" : "osslsigncode", "SIGNTOOL"), + FindCommand("jsign", "JSIGNTOOL"), OperatingSystem.IsMacOS() ? FindCommand("codesign", "CODESIGN") : null, OperatingSystem.IsMacOS() ? FindCommand("productsign", "PRODUCTSIGN") : null, FindCommand(OperatingSystem.IsWindows() ? "wix" : "wixl", "WIX"), diff --git a/ReleaseBuilder/ProcessRunner.cs b/ReleaseBuilder/ProcessRunner.cs index 96bd28e7f..060013644 100644 --- a/ReleaseBuilder/ProcessRunner.cs +++ b/ReleaseBuilder/ProcessRunner.cs @@ -30,12 +30,43 @@ public static class ProcessRunner /// /// The company name to encode in the Authenticode certificate /// - private const string OSSLOrganization = "Duplicati"; + private const string OSSLOrganization = "Duplicati Inc"; /// /// The url to encode in the Authenticode certificate /// private const string OSSLUrl = "https://duplicati.com"; + /// + /// Performs code signing of the + /// + /// The path to the jsign binary + /// The password to decrypt the PFX file + /// The executable to sign, in-place + /// An awaitable task + public static async Task JsignCodeSign(string jsign, string keypin, string executable) + { + var first = true; + foreach (var hashalg in OSSLHashAlgs) + { + await ProcessHelper.Execute([ + jsign, "sign", + "--storetype", "PIV", + "--storepass", keypin, + "--alias", "AUTHENTICATION", + "--name", OSSLOrganization, + "--url", OSSLUrl, + "--alg", hashalg, + first ? "--replace" : null, + "--tsaurl", "http://ts.ssl.com", + "--tsmode", "RFC3161", + executable + ]); + + + first = false; + } + } + /// /// Performs code signing of the /// @@ -52,6 +83,16 @@ public static class ProcessRunner var tmp = Path.GetTempFileName(); File.Delete(tmp); + // It is also possible to use a PKCS#11 module for the key, like this: + // osslsigncode sign -h sha256 + // -pkcs11module /opt/homebrew/lib/libykcs11.dylib + // -certs public-key.pem + // -key 'pkcs11:pin-value={pin}' + // -ts http://ts.ssl.com + // -in {executable} + // -out {tmp} + + await ProcessHelper.Execute([ osslsigncode, "sign", "-pkcs12", pfxfile, From 395a5cb64f0b08e5e1a6c8d330469984e158c74f Mon Sep 17 00:00:00 2001 From: Olivier Chaze Date: Wed, 26 Feb 2025 14:47:20 +0100 Subject: [PATCH 253/362] Adding two Infomaiak Cloud locations --- .../Backend/OpenStack/OpenStackStorage.cs | 40 ++++++++++--------- Duplicati/Library/Backend/S3/S3Backend.cs | 40 ++++++++++--------- 2 files changed, 42 insertions(+), 38 deletions(-) diff --git a/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs b/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs index 2b626e0f6..2c3ccebc4 100644 --- a/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs +++ b/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -75,7 +75,9 @@ namespace Duplicati.Library.Backend.OpenStack new KeyValuePair("Infomaniak Swiss Backup cluster 1", "https://swiss-backup.infomaniak.com/identity/v3"), new KeyValuePair("Infomaniak Swiss Backup cluster 2", "https://swiss-backup02.infomaniak.com/identity/v3"), new KeyValuePair("Infomaniak Swiss Backup cluster 3", "https://swiss-backup03.infomaniak.com/identity/v3"), + new KeyValuePair("Infomaniak Swiss Backup cluster 4", "https://swiss-backup04.infomaniak.com/identity/v3"), new KeyValuePair("Infomaniak Public Cloud 1", "https://api.pub1.infomaniak.cloud/identity/v3"), + new KeyValuePair("Infomaniak Public Cloud 2", "https://api.pub2.infomaniak.cloud/identity/v3"), new KeyValuePair("Catalyst Cloud - nz-hlz-1 (NZ)", "https://api.nz-hlz-1.catalystcloud.io:5000/v3"), new KeyValuePair("Catalyst Cloud - nz-por-1 (NZ)", "https://api.nz-por-1.catalystcloud.io:5000/v3"), }; diff --git a/Duplicati/Library/Backend/S3/S3Backend.cs b/Duplicati/Library/Backend/S3/S3Backend.cs index 8df4c8355..ee92d801f 100644 --- a/Duplicati/Library/Backend/S3/S3Backend.cs +++ b/Duplicati/Library/Backend/S3/S3Backend.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Common.IO; @@ -66,7 +66,9 @@ namespace Duplicati.Library.Backend { "Infomaniak Swiss Backup cluster 1", "s3.swiss-backup.infomaniak.com" }, { "Infomaniak Swiss Backup cluster 2", "s3.swiss-backup02.infomaniak.com" }, { "Infomaniak Swiss Backup cluster 3", "s3.swiss-backup03.infomaniak.com" }, + { "Infomaniak Swiss Backup cluster 4", "s3.swiss-backup04.infomaniak.com" }, { "Infomaniak Public Cloud 1", "s3.pub1.infomaniak.cloud" }, + { "Infomaniak Public Cloud 2", "s3.pub2.infomaniak.cloud" }, { "さくらのクラウド (Sakura Cloud)", "s3.isk01.sakurastorage.jp" }, { "Seagate Lyve - US-East-1", "https://s3.us-east-1.lyvecloud.seagate.com" }, { "Seagate Lyve - US-West-1", "https://s3.us-west-1.lyvecloud.seagate.com" }, From 4bf8c4287b606caaef79697f5a7a9d02497cbd2a Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 26 Feb 2025 15:39:26 +0100 Subject: [PATCH 254/362] Fixed a missing cancellation token forward --- Duplicati/Library/Backend/Mega/MegaBackend.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Backend/Mega/MegaBackend.cs b/Duplicati/Library/Backend/Mega/MegaBackend.cs index f60b2af07..573133f0d 100644 --- a/Duplicati/Library/Backend/Mega/MegaBackend.cs +++ b/Duplicati/Library/Backend/Mega/MegaBackend.cs @@ -194,7 +194,7 @@ namespace Duplicati.Library.Backend.Mega public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { if (m_filecache == null) - await ResetFileCacheAsync(null, CancellationToken.None).ConfigureAwait(false); + await ResetFileCacheAsync(null, cancelToken).ConfigureAwait(false); foreach (var n in m_filecache.Values) { From d6ea2c57cd7c9216c5f577182fc3b19574d58d66 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 26 Feb 2025 15:55:50 +0100 Subject: [PATCH 255/362] Removed outdated comments --- Duplicati/Library/Backend/WEBDAV/WEBDAV.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Duplicati/Library/Backend/WEBDAV/WEBDAV.cs b/Duplicati/Library/Backend/WEBDAV/WEBDAV.cs index 70602a49f..8cd59c891 100644 --- a/Duplicati/Library/Backend/WEBDAV/WEBDAV.cs +++ b/Duplicati/Library/Backend/WEBDAV/WEBDAV.cs @@ -189,7 +189,7 @@ namespace Duplicati.Library.Backend try { using var response = await requestResources.HttpClient.SendAsync(requestResources.RequestMessage, HttpCompletionOption.ResponseContentRead, timeoutToken.Token).ConfigureAwait(false); - response.EnsureSuccessStatusCode(); // This replaces the if needed when Mono was used. + response.EnsureSuccessStatusCode(); using var stream = await response.Content.ReadAsStreamAsync(cancelToken).ConfigureAwait(false); doc.Load(stream); } @@ -307,7 +307,7 @@ namespace Duplicati.Library.Backend var response = await requestResources.HttpClient.SendAsync(requestResources.RequestMessage, combinedTokens.Token).ConfigureAwait(false); - response.EnsureSuccessStatusCode(); // This replaces the if needed when Mono was used. + response.EnsureSuccessStatusCode(); } catch (HttpRequestException wex) @@ -356,7 +356,7 @@ namespace Duplicati.Library.Backend using var response = await requestResources.HttpClient.SendAsync(requestResources.RequestMessage, combinedTokens.Token).ConfigureAwait(false); - response.EnsureSuccessStatusCode(); // This replaces the if needed when Mono was used. + response.EnsureSuccessStatusCode(); } #endregion @@ -428,7 +428,7 @@ namespace Duplicati.Library.Backend using var response = await requestResources.HttpClient.SendAsync(requestResources.RequestMessage, HttpCompletionOption.ResponseHeadersRead, cancelToken).ConfigureAwait(false); - response.EnsureSuccessStatusCode(); // This replaces the if needed when Mono was used. + response.EnsureSuccessStatusCode(); } catch (HttpRequestException wex) { From 692954ee9772385c4c9525ccf25b73afb55ad102 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 26 Feb 2025 16:08:55 +0100 Subject: [PATCH 256/362] Made the local restore database use a connection pool for getting block data --- .../Main/Database/LocalRestoreDatabase.cs | 55 ++++++++++++++++--- 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index b95817273..856a6d29e 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -44,8 +44,9 @@ namespace Duplicati.Library.Main.Database /// protected string m_tempfiletable; protected string m_tempblocktable; - ConcurrentDictionary block2cache = new(); - ConcurrentDictionary block2metacache = new(); + ConcurrentDictionary> block2cache = new(); + ConcurrentDictionary> block2metacache = new(); + ConcurrentBag connection_pool = new(); protected string m_latestblocktable; protected string m_fileprogtable; protected string m_totalprogtable; @@ -90,7 +91,7 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) { //cmd.ExecuteNonQuery("PRAGMA temp_store = MEMORY"); - //cmd.ExecuteNonQuery("PRAGMA journal_mode = OFF"); + //cmd.ExecuteNonQuery("PRAGMA journal_mode = WAL"); //cmd.ExecuteNonQuery("PRAGMA cache_size = -2000000"); // How to handle METADATA? @@ -1079,19 +1080,37 @@ namespace Duplicati.Library.Main.Database { //return block2cache[blocksetID]; - using var cmd = m_connection.CreateCommand(); + if (!connection_pool.TryTake(out var connection)) + { + //connection = CreateConnection(m_connection.ConnectionString); + connection = (System.Data.IDbConnection?)Activator.CreateInstance(Duplicati.Library.SQLiteHelper.SQLiteLoader.SQLiteConnectionType); + connection.ConnectionString = m_connection.ConnectionString + ";Cache=Shared;"; + connection.Open(); + using var cmd2 = connection.CreateCommand(); + cmd2.CommandText = "PRAGMA journal_mode=WAL;"; + cmd2.ExecuteNonQuery(); + cmd2.CommandText = "PRAGMA read_uncommitted = true;"; + cmd2.ExecuteNonQuery(); + } + + using var cmd = connection.CreateCommand(); + cmd.CommandText = @$" SELECT Block.ID, Block.Hash, Block.Size, Block.VolumeID FROM BlocksetEntry INNER JOIN Block ON BlocksetEntry.BlockID = Block.ID WHERE BlocksetEntry.BlocksetID = ?"; + cmd.AddParameter(); cmd.SetParameterValue(0, blocksetID); + using var reader = cmd.ExecuteReader(); for (long i = 0; reader.Read(); i++) { yield return new BlockRequest(reader.ConvertValueToInt64(0), i, reader.ConvertValueToString(1), reader.ConvertValueToInt64(2), reader.ConvertValueToInt64(3), false); } + + connection_pool.Add(connection); } /// @@ -1103,14 +1122,26 @@ namespace Duplicati.Library.Main.Database { //return block2metacache[fileID]; - using var cmd = m_connection.CreateCommand(); + if (!connection_pool.TryTake(out var connection)) + { + connection = (System.Data.IDbConnection?)Activator.CreateInstance(Duplicati.Library.SQLiteHelper.SQLiteLoader.SQLiteConnectionType); + connection.ConnectionString = m_connection.ConnectionString + ";Cache=Shared;"; + connection.Open(); + using var cmd2 = connection.CreateCommand(); + cmd2.CommandText = "PRAGMA journal_mode=WAL;"; + cmd2.ExecuteNonQuery(); + cmd2.CommandText = "PRAGMA read_uncommitted = true;"; + cmd2.ExecuteNonQuery(); + } + + using var cmd = connection.CreateCommand(); cmd.CommandText = $@" SELECT Block.ID, Block.Hash, Block.Size, Block.VolumeID - FROM ""{m_tempfiletable}"" - INNER JOIN Metadataset ON ""{m_tempfiletable}"".MetadataID = Metadataset.ID + FROM FileLookup + INNER JOIN Metadataset ON FileLookup.MetadataID = Metadataset.ID INNER JOIN BlocksetEntry ON Metadataset.BlocksetID = BlocksetEntry.BlocksetID INNER JOIN Block ON BlocksetEntry.BlockID = Block.ID - WHERE ""{m_tempfiletable}"".ID = ? + WHERE FileLookup.ID = ? "; cmd.AddParameter(); cmd.SetParameterValue(0, fileID); @@ -1119,6 +1150,8 @@ namespace Duplicati.Library.Main.Database { yield return new BlockRequest(reader.ConvertValueToInt64(0), i, reader.ConvertValueToString(1), reader.ConvertValueToInt64(2), reader.ConvertValueToInt64(3), false); } + + connection_pool.Add(connection); } /// @@ -1383,6 +1416,12 @@ namespace Duplicati.Library.Main.Database public override void Dispose() { + foreach (var connection in connection_pool) + { + connection.Close(); + connection.Dispose(); + } + connection_pool.Clear(); DropRestoreTable(); base.Dispose(); } From 6873162002792bba1bd4d30a6d9fa5c5cd66e6a5 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 26 Feb 2025 16:09:23 +0100 Subject: [PATCH 257/362] Additional internal profiling resolution --- .../Main/Operation/Restore/FileProcessor.cs | 3 +++ .../Main/Operation/Restore/VolumeDecompressor.cs | 14 ++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs index 3e98a0a47..960337dfc 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs @@ -513,11 +513,14 @@ namespace Duplicati.Library.Main.Operation.Restore sw_work?.Start(); ms.Write(data, 0, (int)block.BlockSize); + sw_work?.Stop(); sw_req?.Start(); block.CacheDecrEvict = true; await block_request.WriteAsync(block).ConfigureAwait(false); sw_req?.Stop(); + + sw_work?.Start(); } ms.Seek(0, SeekOrigin.Begin); diff --git a/Duplicati/Library/Main/Operation/Restore/VolumeDecompressor.cs b/Duplicati/Library/Main/Operation/Restore/VolumeDecompressor.cs index 161660425..eff1b1f30 100644 --- a/Duplicati/Library/Main/Operation/Restore/VolumeDecompressor.cs +++ b/Duplicati/Library/Main/Operation/Restore/VolumeDecompressor.cs @@ -55,7 +55,9 @@ namespace Duplicati.Library.Main.Operation.Restore { Stopwatch sw_read = options.InternalProfiling ? new() : null; Stopwatch sw_write = options.InternalProfiling ? new() : null; - Stopwatch sw_decompress = options.InternalProfiling ? new() : null; + Stopwatch sw_decompress1 = options.InternalProfiling ? new() : null; + Stopwatch sw_decompress2 = options.InternalProfiling ? new() : null; + Stopwatch sw_decompress3 = options.InternalProfiling ? new() : null; Stopwatch sw_verify = options.InternalProfiling ? new() : null; try @@ -69,13 +71,17 @@ namespace Duplicati.Library.Main.Operation.Restore var (block_request, volume_reader) = await self.Input.ReadAsync().ConfigureAwait(false); sw_read?.Stop(); - sw_decompress?.Start(); + sw_decompress1?.Start(); var data = ArrayPool.Shared.Rent(options.Blocksize); + sw_decompress1?.Stop(); + sw_decompress2?.Start(); lock (volume_reader) // The BlockVolumeReader is not thread-safe { + sw_decompress2?.Stop(); + sw_decompress3?.Start(); volume_reader.ReadBlock(block_request.BlockHash, data); + sw_decompress3?.Stop(); } - sw_decompress?.Stop(); sw_verify?.Start(); var hash = Convert.ToBase64String(block_hasher.ComputeHash(data, 0, (int)block_request.BlockSize)); @@ -97,7 +103,7 @@ namespace Duplicati.Library.Main.Operation.Restore if (options.InternalProfiling) { - Logging.Log.WriteProfilingMessage(LOGTAG, "InternalTimings", $"Read: {sw_read.ElapsedMilliseconds}ms, Write: {sw_write.ElapsedMilliseconds}ms, Decompress: {sw_decompress.ElapsedMilliseconds}ms, Verify: {sw_verify.ElapsedMilliseconds}ms"); + Logging.Log.WriteProfilingMessage(LOGTAG, "InternalTimings", $"Read: {sw_read.ElapsedMilliseconds}ms, Write: {sw_write.ElapsedMilliseconds}ms, Decompress allocate: {sw_decompress1.ElapsedMilliseconds}ms, Decompress lock: {sw_decompress2.ElapsedMilliseconds}ms, Decompress read: {sw_decompress3.ElapsedMilliseconds}ms, Verify: {sw_verify.ElapsedMilliseconds}ms"); } } catch (Exception ex) From 0e6e1bcbd83f06ef1126b695fdbb9e0dc8d955fb Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 26 Feb 2025 16:10:51 +0100 Subject: [PATCH 258/362] Currently, the block cache cannot compact while using the arraypool, as this doesn't properly return the buffers to the pool --- .../Main/Operation/Restore/BlockManager.cs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs index 155d6bf21..cf9a0870a 100644 --- a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs @@ -106,6 +106,7 @@ namespace Duplicati.Library.Main.Operation.Restore /// The options for the restore. /// private readonly Options m_options; + private MemoryCacheEntryOptions m_entry_options = new(); /// /// Initializes a new instance of the class. @@ -121,6 +122,15 @@ namespace Duplicati.Library.Main.Operation.Restore { var cache_options = new MemoryCacheOptions(); m_block_cache = new MemoryCache(cache_options); + m_entry_options.RegisterPostEvictionCallback((key, value, reason, state) => + { + m_block_cache.Remove(key); + lock (m_blockcount_lock) + { + m_returncount++; + } + ArrayPool.Shared.Return((byte[])value); + }); } this.readers = readers; sw_cacheevict = options.InternalProfiling ? new() : null; @@ -259,10 +269,11 @@ namespace Duplicati.Library.Main.Operation.Restore } sw_cacheevict?.Start(); - if (m_block_cache?.Count * m_options.Blocksize > m_options.RestoreCacheMax) - { - m_block_cache?.Compact(m_options.RestoreCacheEvict); - } + // TODO cache is broken due to the arraypool + //if (m_block_cache?.Count > m_options.RestoreCacheMax) + //{ + // m_block_cache?.Compact(m_options.RestoreCacheEvict); + //} sw_cacheevict?.Stop(); } From 71091551273d93283c51e87d1b0ba13067848720 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Wed, 26 Feb 2025 16:11:13 +0100 Subject: [PATCH 259/362] Additional block count reporting of the block manager --- .../Main/Operation/Restore/BlockManager.cs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs index cf9a0870a..692deb15d 100644 --- a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs @@ -106,6 +106,10 @@ namespace Duplicati.Library.Main.Operation.Restore /// The options for the restore. /// private readonly Options m_options; + + private long m_blockcount_total = 0; + private long m_returncount = 0; + private long m_autoevictcount = 0; private MemoryCacheEntryOptions m_entry_options = new(); /// @@ -144,6 +148,7 @@ namespace Duplicati.Library.Main.Operation.Restore var vc = m_volumecount.TryGetValue(volume_id, out var v); m_volumecount[volume_id] = vc ? v + 1 : 1; } + m_blockcount_total = m_blockcount.Count; } /// @@ -177,6 +182,7 @@ namespace Duplicati.Library.Main.Operation.Restore m_blockcount.Remove(blockRequest.BlockID); data = m_block_cache?.Get(blockRequest.BlockID); m_block_cache?.Remove(blockRequest.BlockID); + m_autoevictcount++; } else // block_count < 0 { @@ -202,7 +208,13 @@ namespace Duplicati.Library.Main.Operation.Restore } if (data != null) + { + lock (m_blockcount_lock) + { + m_returncount++; + } ArrayPool.Shared.Return(data); + } // Notify the `VolumeManager` that it should evict the volume. if (emit_evict) @@ -317,7 +329,12 @@ namespace Duplicati.Library.Main.Operation.Restore if (m_options.InternalProfiling) { - Logging.Log.WriteProfilingMessage(LOGTAG, "InternalTimings", $"Sleepable dictionary - CheckCounts: {sw_checkcounts.ElapsedMilliseconds}ms, Get wait: {sw_get_wait.ElapsedMilliseconds}ms, Cache evict: {sw_cacheevict.ElapsedMilliseconds}ms"); + Logging.Log.WriteProfilingMessage(LOGTAG, "InternalTimings", $"Sleepable dictionary - CheckCounts: {sw_checkcounts.ElapsedMilliseconds}ms, Get wait: {sw_get_wait.ElapsedMilliseconds}ms, Cache evict: {sw_cacheevict.ElapsedMilliseconds}ms, returned blocks: {m_returncount}, total blocks: {m_blockcount_total}, auto evicted: {m_autoevictcount}"); + } + + if (m_block_cache.Count > 0) + { + Logging.Log.WriteWarningMessage(LOGTAG, "BlockCacheError", null, $"Internal Block cache is not empty: {m_block_cache.Count}"); } } From e6ebafac14fb187151f90d50662a9c7578a5ead6 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 26 Feb 2025 22:11:22 +0100 Subject: [PATCH 260/362] Added link for easy switching to new UI --- Duplicati/Server/webroot/ngax/index.html | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Duplicati/Server/webroot/ngax/index.html b/Duplicati/Server/webroot/ngax/index.html index bdc81a44e..5f72a20fa 100755 --- a/Duplicati/Server/webroot/ngax/index.html +++ b/Duplicati/Server/webroot/ngax/index.html @@ -258,6 +258,10 @@
{{'Manual' | translate}}
+ +
+ {{'Use new UI' | translate}} +
From a5d53d4493fceee206db3374b985cd796f051f9b Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 26 Feb 2025 22:12:34 +0100 Subject: [PATCH 261/362] Version bump ngclient --- Duplicati/Server/webroot/ngclient/package-lock.json | 8 ++++---- Duplicati/Server/webroot/ngclient/package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Duplicati/Server/webroot/ngclient/package-lock.json b/Duplicati/Server/webroot/ngclient/package-lock.json index 8324e94cb..34c630c3e 100644 --- a/Duplicati/Server/webroot/ngclient/package-lock.json +++ b/Duplicati/Server/webroot/ngclient/package-lock.json @@ -9,13 +9,13 @@ "version": "1.0.0", "license": "MIT", "dependencies": { - "@duplicati/ngclient": "^0.0.83" + "@duplicati/ngclient": "^0.0.85" } }, "node_modules/@duplicati/ngclient": { - "version": "0.0.83", - "resolved": "https://registry.npmjs.org/@duplicati/ngclient/-/ngclient-0.0.83.tgz", - "integrity": "sha512-/kvFgch7Y60HhFPCMZxT20GebVggBG98/PpTJ3R5xr4cQVeOtkJgZb9Yds4LNiIBhiNBQvk93CN0FrhEDtzB6w==", + "version": "0.0.85", + "resolved": "https://registry.npmjs.org/@duplicati/ngclient/-/ngclient-0.0.85.tgz", + "integrity": "sha512-998IYLYD1KYY2q9l3z1EiqxyoVFCs3aCXKnZJtE3gK/PJcFagIAAbGWhxJ6RvXozn4dm+8PFcu2bcTJdE37qeQ==", "license": "MIT" } } diff --git a/Duplicati/Server/webroot/ngclient/package.json b/Duplicati/Server/webroot/ngclient/package.json index 59c9cf022..beed1e594 100644 --- a/Duplicati/Server/webroot/ngclient/package.json +++ b/Duplicati/Server/webroot/ngclient/package.json @@ -4,6 +4,6 @@ "description": "Modern Angular Client for Duplicati", "license": "MIT", "dependencies": { - "@duplicati/ngclient": "^0.0.83" + "@duplicati/ngclient": "^0.0.85" } } From bdfdc7e92a09e619f740837f06bfa2f9c86dee47 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 27 Feb 2025 10:54:47 +0100 Subject: [PATCH 262/362] MemoryCache evict now correctly handles the ArrayPool --- .../Main/Operation/Restore/BlockManager.cs | 38 ++++++++++++------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs index 692deb15d..32a3f4861 100644 --- a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs @@ -111,6 +111,7 @@ namespace Duplicati.Library.Main.Operation.Restore private long m_returncount = 0; private long m_autoevictcount = 0; private MemoryCacheEntryOptions m_entry_options = new(); + private ConcurrentDictionary m_active_readers = []; /// /// Initializes a new instance of the class. @@ -124,18 +125,21 @@ namespace Duplicati.Library.Main.Operation.Restore m_volume_request = volume_request; if (m_options.RestoreCacheMax > 0) { - var cache_options = new MemoryCacheOptions(); - m_block_cache = new MemoryCache(cache_options); - m_entry_options.RegisterPostEvictionCallback((key, value, reason, state) => + while (true) { - m_block_cache.Remove(key); lock (m_blockcount_lock) { - m_returncount++; + if (m_active_readers.TryGetValue((long)key, out var ac) && ac == 0) + { + m_block_cache.Remove(key); + m_returncount++; + break; + } } - ArrayPool.Shared.Return((byte[])value); - }); - } + await Task.Delay(10).ConfigureAwait(false); + } + ArrayPool.Shared.Return((byte[])value); + }); this.readers = readers; sw_cacheevict = options.InternalProfiling ? new() : null; sw_checkcounts = options.InternalProfiling ? new() : null; @@ -170,6 +174,8 @@ namespace Duplicati.Library.Main.Operation.Restore { sw_checkcounts?.Start(); + m_active_readers[blockRequest.BlockID] = m_active_readers.TryGetValue(blockRequest.BlockID, out var ac) ? ac - 1 : 0; + var block_count = m_blockcount.TryGetValue(blockRequest.BlockID, out var c) ? c - 1 : 0; if (block_count > 0) @@ -240,6 +246,11 @@ namespace Duplicati.Library.Main.Operation.Restore /// A `Task` holding the data block. public Task Get(BlockRequest block_request) { + lock (m_blockcount_lock) + { + m_active_readers[block_request.BlockID] = m_active_readers.TryGetValue(block_request.BlockID, out var c) ? c + 1 : 1; + } + // Check if the block is already in the cache, and return it if it is. if (m_block_cache != null && m_block_cache.TryGetValue(block_request.BlockID, out byte[] value)) { @@ -281,11 +292,10 @@ namespace Duplicati.Library.Main.Operation.Restore } sw_cacheevict?.Start(); - // TODO cache is broken due to the arraypool - //if (m_block_cache?.Count > m_options.RestoreCacheMax) - //{ - // m_block_cache?.Compact(m_options.RestoreCacheEvict); - //} + if (m_block_cache?.Count > m_options.RestoreCacheMax) + { + m_block_cache?.Compact(m_options.RestoreCacheEvict); + } sw_cacheevict?.Stop(); } @@ -332,7 +342,7 @@ namespace Duplicati.Library.Main.Operation.Restore Logging.Log.WriteProfilingMessage(LOGTAG, "InternalTimings", $"Sleepable dictionary - CheckCounts: {sw_checkcounts.ElapsedMilliseconds}ms, Get wait: {sw_get_wait.ElapsedMilliseconds}ms, Cache evict: {sw_cacheevict.ElapsedMilliseconds}ms, returned blocks: {m_returncount}, total blocks: {m_blockcount_total}, auto evicted: {m_autoevictcount}"); } - if (m_block_cache.Count > 0) + if (m_block_cache?.Count > 0) { Logging.Log.WriteWarningMessage(LOGTAG, "BlockCacheError", null, $"Internal Block cache is not empty: {m_block_cache.Count}"); } From 038e78ad06bf72bda220077087f42a4005fd572c Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 27 Feb 2025 11:17:11 +0100 Subject: [PATCH 263/362] The block cache is now always present, as it keeps track of blocks. In case of a cache size 0, every Set() will trigger an evict. --- .../Main/Operation/Restore/BlockManager.cs | 16 +++++++++------- Duplicati/Library/Main/Options.cs | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs index 32a3f4861..b869c2335 100644 --- a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs @@ -123,7 +123,9 @@ namespace Duplicati.Library.Main.Operation.Restore { m_options = options; m_volume_request = volume_request; - if (m_options.RestoreCacheMax > 0) + var cache_options = new MemoryCacheOptions(); + m_block_cache = new MemoryCache(cache_options); + m_entry_options.RegisterPostEvictionCallback(async (key, value, reason, state) => { while (true) { @@ -186,8 +188,8 @@ namespace Duplicati.Library.Main.Operation.Restore { // Evict the block from the cache and check if the volume is no longer needed. m_blockcount.Remove(blockRequest.BlockID); - data = m_block_cache?.Get(blockRequest.BlockID); - m_block_cache?.Remove(blockRequest.BlockID); + data = m_block_cache.Get(blockRequest.BlockID); + m_block_cache.Remove(blockRequest.BlockID); m_autoevictcount++; } else // block_count < 0 @@ -283,7 +285,7 @@ namespace Duplicati.Library.Main.Operation.Restore /// public void Set(BlockRequest blockRequest, byte[] value) { - m_block_cache?.Set(blockRequest.BlockID, value); + m_block_cache.Set(blockRequest.BlockID, value); // Notify any waiters that the block is available. if (m_waiters.TryRemove(blockRequest.BlockID, out var tcs)) @@ -292,9 +294,9 @@ namespace Duplicati.Library.Main.Operation.Restore } sw_cacheevict?.Start(); - if (m_block_cache?.Count > m_options.RestoreCacheMax) + if (m_block_cache.Count > m_options.RestoreCacheMax) { - m_block_cache?.Compact(m_options.RestoreCacheEvict); + m_block_cache.Compact(m_options.RestoreCacheMax == 0 ? 1.0 : m_options.RestoreCacheEvict); } sw_cacheevict?.Stop(); } @@ -342,7 +344,7 @@ namespace Duplicati.Library.Main.Operation.Restore Logging.Log.WriteProfilingMessage(LOGTAG, "InternalTimings", $"Sleepable dictionary - CheckCounts: {sw_checkcounts.ElapsedMilliseconds}ms, Get wait: {sw_get_wait.ElapsedMilliseconds}ms, Cache evict: {sw_cacheevict.ElapsedMilliseconds}ms, returned blocks: {m_returncount}, total blocks: {m_blockcount_total}, auto evicted: {m_autoevictcount}"); } - if (m_block_cache?.Count > 0) + if (m_block_cache.Count > 0) { Logging.Log.WriteWarningMessage(LOGTAG, "BlockCacheError", null, $"Internal Block cache is not empty: {m_block_cache.Count}"); } diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index b1b2894f9..1125588de 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -2057,7 +2057,7 @@ namespace Duplicati.Library.Main } /// - /// Gets the maximum number of data blocks to keep in the cache. If set to 0, the cache is disabled. + /// Gets the maximum number of data blocks to keep in the cache. If set to 0, the cache is effictively disabled, but some is still kept for bookkeeping. /// public long RestoreCacheMax { From a0eae4131b747d6cb4263debf8e68a88be10723d Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 27 Feb 2025 14:14:07 +0100 Subject: [PATCH 264/362] Cleaned up LocalRestoreDatabase --- .../Main/Database/LocalRestoreDatabase.cs | 124 +++++------------- 1 file changed, 31 insertions(+), 93 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index 856a6d29e..6dd245fa8 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -44,9 +44,7 @@ namespace Duplicati.Library.Main.Database /// protected string m_tempfiletable; protected string m_tempblocktable; - ConcurrentDictionary> block2cache = new(); - ConcurrentDictionary> block2metacache = new(); - ConcurrentBag connection_pool = new(); + protected ConcurrentBag m_connection_pool = []; protected string m_latestblocktable; protected string m_fileprogtable; protected string m_totalprogtable; @@ -90,10 +88,6 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) { - //cmd.ExecuteNonQuery("PRAGMA temp_store = MEMORY"); - //cmd.ExecuteNonQuery("PRAGMA journal_mode = WAL"); - //cmd.ExecuteNonQuery("PRAGMA cache_size = -2000000"); - // How to handle METADATA? cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", m_fileprogtable)); cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (" @@ -355,58 +349,6 @@ namespace Duplicati.Library.Main.Database return new Tuple(0, 0); } - public void BuildCaches() - { - using (var cmd = m_connection.CreateCommand()) - { - var blocksetids = cmd.ExecuteReaderEnumerable(@"SELECT ""Blockset"".""ID"" FROM ""Blockset""").Select(r => r.ConvertValueToInt64(0)).ToList(); - cmd.CommandText = @$" - SELECT Block.ID, Block.Hash, Block.Size, Block.VolumeID - FROM BlocksetEntry INNER JOIN Block - ON BlocksetEntry.BlockID = Block.ID - WHERE BlocksetEntry.BlocksetID = ?"; - cmd.AddParameter(); - foreach (var bsid in blocksetids) - { - cmd.SetParameterValue(0, bsid); - using var reader = cmd.ExecuteReader(); - List brlist = []; - for (long i = 0; reader.Read(); i++) - brlist.Add(new BlockRequest(reader.ConvertValueToInt64(0), i, reader.ConvertValueToString(1), reader.ConvertValueToInt64(2), reader.ConvertValueToInt64(3), false)); - block2cache[bsid] = [.. brlist]; - } - Console.WriteLine($"Block cache loaded with {block2cache.Count} entries"); - } - - using (var cmd = m_connection.CreateCommand()) - { - var fileids = cmd.ExecuteReaderEnumerable($@" - SELECT F.ID - FROM ""{m_tempfiletable}"" F - LEFT JOIN Blockset B ON F.BlocksetID = B.ID - WHERE F.BlocksetID != {FOLDER_BLOCKSET_ID}").Select(r => r.ConvertValueToInt64(0)).ToList(); - - cmd.CommandText = $@" - SELECT Block.ID, Block.Hash, Block.Size, Block.VolumeID - FROM ""{m_tempfiletable}"" - INNER JOIN Metadataset ON ""{m_tempfiletable}"".MetadataID = Metadataset.ID - INNER JOIN BlocksetEntry ON Metadataset.BlocksetID = BlocksetEntry.BlocksetID - INNER JOIN Block ON BlocksetEntry.BlockID = Block.ID - WHERE ""{m_tempfiletable}"".ID = ? - "; - cmd.AddParameter(); - foreach (var fileID in fileids) - { - cmd.SetParameterValue(0, fileID); - using var reader = cmd.ExecuteReader(); - List brlist = []; - for (long i = 0; reader.Read(); i++) - brlist.Add(new BlockRequest(reader.ConvertValueToInt64(0), i, reader.ConvertValueToString(1), reader.ConvertValueToInt64(2), reader.ConvertValueToInt64(3), false)); - block2metacache[fileID] = [.. brlist]; - } - } - } - public string GetFirstPath() { using (var cmd = m_connection.CreateCommand()) @@ -1001,6 +943,26 @@ namespace Duplicati.Library.Main.Database return new FilesAndMetadata(m_connection, m_tempfiletable, m_tempblocktable, blocksize, curvolume); } + /// + /// Returns a connection from the connection pool. + /// + /// A connection from the connection pool. + public IDbConnection GetConnectionFromPool() + { + if (!m_connection_pool.TryTake(out var connection)) + { + connection = (System.Data.IDbConnection)Activator.CreateInstance(Duplicati.Library.SQLiteHelper.SQLiteLoader.SQLiteConnectionType); + connection.ConnectionString = m_connection.ConnectionString + ";Cache=Shared;"; + connection.Open(); + + using var cmd = connection.CreateCommand(); + cmd.ExecuteNonQuery("PRAGMA journal_mode = WAL;"); + cmd.ExecuteNonQuery("PRAGMA read_uncommitted = true;"); + } + + return connection; + } + private class FileToRestore : IFileToRestore { public string Path { get; private set; } @@ -1078,21 +1040,7 @@ namespace Duplicati.Library.Main.Database /// A list of needed to restore the given file. public IEnumerable GetBlocksFromFile(long blocksetID) { - //return block2cache[blocksetID]; - - if (!connection_pool.TryTake(out var connection)) - { - //connection = CreateConnection(m_connection.ConnectionString); - connection = (System.Data.IDbConnection?)Activator.CreateInstance(Duplicati.Library.SQLiteHelper.SQLiteLoader.SQLiteConnectionType); - connection.ConnectionString = m_connection.ConnectionString + ";Cache=Shared;"; - connection.Open(); - using var cmd2 = connection.CreateCommand(); - cmd2.CommandText = "PRAGMA journal_mode=WAL;"; - cmd2.ExecuteNonQuery(); - cmd2.CommandText = "PRAGMA read_uncommitted = true;"; - cmd2.ExecuteNonQuery(); - } - + var connection = GetConnectionFromPool(); using var cmd = connection.CreateCommand(); cmd.CommandText = @$" @@ -1100,7 +1048,6 @@ namespace Duplicati.Library.Main.Database FROM BlocksetEntry INNER JOIN Block ON BlocksetEntry.BlockID = Block.ID WHERE BlocksetEntry.BlocksetID = ?"; - cmd.AddParameter(); cmd.SetParameterValue(0, blocksetID); @@ -1110,7 +1057,8 @@ namespace Duplicati.Library.Main.Database yield return new BlockRequest(reader.ConvertValueToInt64(0), i, reader.ConvertValueToString(1), reader.ConvertValueToInt64(2), reader.ConvertValueToInt64(3), false); } - connection_pool.Add(connection); + // Return the connection to the pool + m_connection_pool.Add(connection); } /// @@ -1120,21 +1068,9 @@ namespace Duplicati.Library.Main.Database /// A list of needed to restore the metadata of the given file. public IEnumerable GetMetadataBlocksFromFile(long fileID) { - //return block2metacache[fileID]; - - if (!connection_pool.TryTake(out var connection)) - { - connection = (System.Data.IDbConnection?)Activator.CreateInstance(Duplicati.Library.SQLiteHelper.SQLiteLoader.SQLiteConnectionType); - connection.ConnectionString = m_connection.ConnectionString + ";Cache=Shared;"; - connection.Open(); - using var cmd2 = connection.CreateCommand(); - cmd2.CommandText = "PRAGMA journal_mode=WAL;"; - cmd2.ExecuteNonQuery(); - cmd2.CommandText = "PRAGMA read_uncommitted = true;"; - cmd2.ExecuteNonQuery(); - } - + var connection = GetConnectionFromPool(); using var cmd = connection.CreateCommand(); + cmd.CommandText = $@" SELECT Block.ID, Block.Hash, Block.Size, Block.VolumeID FROM FileLookup @@ -1145,13 +1081,15 @@ namespace Duplicati.Library.Main.Database "; cmd.AddParameter(); cmd.SetParameterValue(0, fileID); + using var reader = cmd.ExecuteReader(); for (long i = 0; reader.Read(); i++) { yield return new BlockRequest(reader.ConvertValueToInt64(0), i, reader.ConvertValueToString(1), reader.ConvertValueToInt64(2), reader.ConvertValueToInt64(3), false); } - connection_pool.Add(connection); + // Return the connection to the pool + m_connection_pool.Add(connection); } /// @@ -1416,12 +1354,12 @@ namespace Duplicati.Library.Main.Database public override void Dispose() { - foreach (var connection in connection_pool) + foreach (var connection in m_connection_pool) { connection.Close(); connection.Dispose(); } - connection_pool.Clear(); + m_connection_pool.Clear(); DropRestoreTable(); base.Dispose(); } From 2fae2d510e4eb353d975537ce336ffc759aa096c Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 27 Feb 2025 14:16:06 +0100 Subject: [PATCH 265/362] Removed old block counters from BlockManager --- .../Library/Main/Operation/Restore/BlockManager.cs | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs index b869c2335..7df17ccfb 100644 --- a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs @@ -107,9 +107,6 @@ namespace Duplicati.Library.Main.Operation.Restore /// private readonly Options m_options; - private long m_blockcount_total = 0; - private long m_returncount = 0; - private long m_autoevictcount = 0; private MemoryCacheEntryOptions m_entry_options = new(); private ConcurrentDictionary m_active_readers = []; @@ -134,7 +131,6 @@ namespace Duplicati.Library.Main.Operation.Restore if (m_active_readers.TryGetValue((long)key, out var ac) && ac == 0) { m_block_cache.Remove(key); - m_returncount++; break; } } @@ -154,7 +150,6 @@ namespace Duplicati.Library.Main.Operation.Restore var vc = m_volumecount.TryGetValue(volume_id, out var v); m_volumecount[volume_id] = vc ? v + 1 : 1; } - m_blockcount_total = m_blockcount.Count; } /// @@ -190,7 +185,6 @@ namespace Duplicati.Library.Main.Operation.Restore m_blockcount.Remove(blockRequest.BlockID); data = m_block_cache.Get(blockRequest.BlockID); m_block_cache.Remove(blockRequest.BlockID); - m_autoevictcount++; } else // block_count < 0 { @@ -216,13 +210,7 @@ namespace Duplicati.Library.Main.Operation.Restore } if (data != null) - { - lock (m_blockcount_lock) - { - m_returncount++; - } ArrayPool.Shared.Return(data); - } // Notify the `VolumeManager` that it should evict the volume. if (emit_evict) @@ -341,7 +329,7 @@ namespace Duplicati.Library.Main.Operation.Restore if (m_options.InternalProfiling) { - Logging.Log.WriteProfilingMessage(LOGTAG, "InternalTimings", $"Sleepable dictionary - CheckCounts: {sw_checkcounts.ElapsedMilliseconds}ms, Get wait: {sw_get_wait.ElapsedMilliseconds}ms, Cache evict: {sw_cacheevict.ElapsedMilliseconds}ms, returned blocks: {m_returncount}, total blocks: {m_blockcount_total}, auto evicted: {m_autoevictcount}"); + Logging.Log.WriteProfilingMessage(LOGTAG, "InternalTimings", $"Sleepable dictionary - CheckCounts: {sw_checkcounts.ElapsedMilliseconds}ms, Get wait: {sw_get_wait.ElapsedMilliseconds}ms, Cache evict: {sw_cacheevict.ElapsedMilliseconds}ms"); } if (m_block_cache.Count > 0) From 2ebb021174ef5ba7629bc80fc2f5d16ec890da6f Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 27 Feb 2025 14:38:18 +0100 Subject: [PATCH 266/362] Cleaned up BlockManager --- .../Main/Operation/Restore/BlockManager.cs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs index 7df17ccfb..e9e351f51 100644 --- a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs @@ -106,9 +106,14 @@ namespace Duplicati.Library.Main.Operation.Restore /// The options for the restore. /// private readonly Options m_options; - - private MemoryCacheEntryOptions m_entry_options = new(); - private ConcurrentDictionary m_active_readers = []; + /// + /// The cache eviction options. Used for registering a callback when a block is evicted from the cache. + /// + private readonly MemoryCacheEntryOptions m_entry_options = new(); + /// + /// Dictionary to keep track of how many active readers are accessing each block. On eviction, the byte[] buffer can only be returned to the ArrayPool if there are no active readers. + /// + private readonly ConcurrentDictionary m_active_readers = []; /// /// Initializes a new instance of the class. @@ -159,7 +164,6 @@ namespace Duplicati.Library.Main.Operation.Restore /// the VolumeDownloader will be notified. /// /// The block request to check. - /// public async Task CheckCounts(BlockRequest blockRequest) { long error_block_id = -1; @@ -269,8 +273,8 @@ namespace Duplicati.Library.Main.Operation.Restore /// removed, the requester will be given a `Task` that will be /// completed when the block is available. /// - /// - /// + /// The block request related to the value. + /// The byte[] buffer holding the block data. public void Set(BlockRequest blockRequest, byte[] value) { m_block_cache.Set(blockRequest.BlockID, value); @@ -359,6 +363,7 @@ namespace Duplicati.Library.Main.Operation.Restore /// accessing the cache for the blocks, and writing the resulting /// blocks back to the `FileProcessor`. /// + /// The named channels for the restore operation. /// The database holding information about how many of each block this restore requires. /// The restore options. /// The channels for reading block requests from the `FileProcessor`. From a539af58ca9d4f45b19937b7307cc88b10ae2c14 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 27 Feb 2025 14:40:52 +0100 Subject: [PATCH 267/362] Removed unused code from FileLister --- Duplicati/Library/Main/Operation/Restore/FileLister.cs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/FileLister.cs b/Duplicati/Library/Main/Operation/Restore/FileLister.cs index 493665092..6da3be7e6 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileLister.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileLister.cs @@ -28,7 +28,7 @@ using Duplicati.Library.Main.Database; namespace Duplicati.Library.Main.Operation.Restore { - // TODO double check docstrings + /// /// Process that holds the files that this particular restore operation needs to restore. /// @@ -56,7 +56,6 @@ namespace Duplicati.Library.Main.Operation.Restore }, async self => { - Stopwatch sw_cache = options.InternalProfiling ? new() : null; Stopwatch sw_prework = options.InternalProfiling ? new() : null; Stopwatch sw_write = options.InternalProfiling ? new() : null; @@ -64,10 +63,6 @@ namespace Duplicati.Library.Main.Operation.Restore try { - sw_cache?.Start(); - //db.BuildCaches(); - sw_cache?.Stop(); - sw_prework?.Start(); var files = db.GetFilesAndSymlinksToRestore(true).OrderByDescending(x => x.Length).ToArray(); // Get started on big files first result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_DownloadingRemoteFiles); @@ -91,7 +86,7 @@ namespace Duplicati.Library.Main.Operation.Restore if (options.InternalProfiling) { - Logging.Log.WriteProfilingMessage(LOGTAG, "InternalTimings", $"Cache: {sw_cache.ElapsedMilliseconds}ms, Prework: {sw_prework.ElapsedMilliseconds}ms, Write: {sw_write.ElapsedMilliseconds}ms"); + Logging.Log.WriteProfilingMessage(LOGTAG, "InternalTimings", $"Prework: {sw_prework.ElapsedMilliseconds}ms, Write: {sw_write.ElapsedMilliseconds}ms"); } } }); From 6ef151f4af4ece0170d87a16edab54a9528b2070 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 27 Feb 2025 14:45:05 +0100 Subject: [PATCH 268/362] Cleaned up FileProcessor --- Duplicati/Library/Main/Operation/Restore/FileProcessor.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs index 960337dfc..7027923aa 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs @@ -52,6 +52,7 @@ namespace Duplicati.Library.Main.Operation.Restore /// /// Runs the file processor process that restores the files that need to be restored. /// + /// The named channels for the restore operation. /// The restore database, which is queried for blocks, corresponding volumes and metadata for the files. /// The channel to request blocks from the block manager. /// The channel to receive blocks from the block manager. @@ -99,7 +100,7 @@ namespace Duplicati.Library.Main.Operation.Restore // Get information about the blocks for the file // TODO rather than keeping all of the blocks in memory, we could do a single pass over the blocks using a cursor, only keeping the relevant block requests in memory. Maybe even only a single block request at a time. sw_block?.Start(); - var blocks = db.GetBlocksFromFile(file.BlocksetID).ToArray();//.Select(x => new BlockRequest(x.BlockID, x.BlockOffset, x.BlockHash, x.BlockSize, x.VolumeID, false)).ToArray(); + var blocks = db.GetBlocksFromFile(file.BlocksetID).ToArray(); sw_block?.Stop(); sw_work_verify_target?.Start(); @@ -495,7 +496,7 @@ namespace Duplicati.Library.Main.Operation.Restore private static async Task RestoreMetadata(LocalRestoreDatabase db, FileRequest file, IChannel block_request, IChannel block_response, Options options, Stopwatch sw_meta, Stopwatch sw_work, Stopwatch sw_req, Stopwatch sw_resp) { sw_meta?.Start(); - var blocks = db.GetMetadataBlocksFromFile(file.ID);//.Select(x => new BlockRequest(x.BlockID, x.BlockOffset, x.BlockHash, x.BlockSize, x.VolumeID, false)).ToArray(); + var blocks = db.GetMetadataBlocksFromFile(file.ID); sw_meta?.Stop(); using var ms = new MemoryStream(); @@ -534,6 +535,7 @@ namespace Duplicati.Library.Main.Operation.Restore /// The metadata about the blocks that make up the file. /// A hasher for the file. /// A hasher for a data block. + /// A buffer to read and write data blocks. /// The Duplicati configuration options. /// The restoration results. /// An awaitable `Task`, which returns a collection of data blocks that are missing. @@ -634,6 +636,7 @@ namespace Duplicati.Library.Main.Operation.Restore /// The total number of blocks for the file. /// A hasher for the file. /// A hasher for the data block. + /// A buffer to read and write data blocks. /// The Duplicati configuration options. /// The restoration results. /// The channel to request blocks from the block manager. Used to inform the block manager which blocks are already present. From b9eaa224b8a4a8b02782c18a3f505f43e37a611c Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 27 Feb 2025 14:46:04 +0100 Subject: [PATCH 269/362] Renamed variables in VolumeDecompressor --- .../Operation/Restore/VolumeDecompressor.cs | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/VolumeDecompressor.cs b/Duplicati/Library/Main/Operation/Restore/VolumeDecompressor.cs index eff1b1f30..9809a478b 100644 --- a/Duplicati/Library/Main/Operation/Restore/VolumeDecompressor.cs +++ b/Duplicati/Library/Main/Operation/Restore/VolumeDecompressor.cs @@ -55,9 +55,9 @@ namespace Duplicati.Library.Main.Operation.Restore { Stopwatch sw_read = options.InternalProfiling ? new() : null; Stopwatch sw_write = options.InternalProfiling ? new() : null; - Stopwatch sw_decompress1 = options.InternalProfiling ? new() : null; - Stopwatch sw_decompress2 = options.InternalProfiling ? new() : null; - Stopwatch sw_decompress3 = options.InternalProfiling ? new() : null; + Stopwatch sw_decompress_alloc = options.InternalProfiling ? new() : null; + Stopwatch sw_decompress_locking = options.InternalProfiling ? new() : null; + Stopwatch sw_decompress_read = options.InternalProfiling ? new() : null; Stopwatch sw_verify = options.InternalProfiling ? new() : null; try @@ -71,16 +71,16 @@ namespace Duplicati.Library.Main.Operation.Restore var (block_request, volume_reader) = await self.Input.ReadAsync().ConfigureAwait(false); sw_read?.Stop(); - sw_decompress1?.Start(); + sw_decompress_alloc?.Start(); var data = ArrayPool.Shared.Rent(options.Blocksize); - sw_decompress1?.Stop(); - sw_decompress2?.Start(); + sw_decompress_alloc?.Stop(); + sw_decompress_locking?.Start(); lock (volume_reader) // The BlockVolumeReader is not thread-safe { - sw_decompress2?.Stop(); - sw_decompress3?.Start(); + sw_decompress_locking?.Stop(); + sw_decompress_read?.Start(); volume_reader.ReadBlock(block_request.BlockHash, data); - sw_decompress3?.Stop(); + sw_decompress_read?.Stop(); } sw_verify?.Start(); @@ -103,7 +103,7 @@ namespace Duplicati.Library.Main.Operation.Restore if (options.InternalProfiling) { - Logging.Log.WriteProfilingMessage(LOGTAG, "InternalTimings", $"Read: {sw_read.ElapsedMilliseconds}ms, Write: {sw_write.ElapsedMilliseconds}ms, Decompress allocate: {sw_decompress1.ElapsedMilliseconds}ms, Decompress lock: {sw_decompress2.ElapsedMilliseconds}ms, Decompress read: {sw_decompress3.ElapsedMilliseconds}ms, Verify: {sw_verify.ElapsedMilliseconds}ms"); + Logging.Log.WriteProfilingMessage(LOGTAG, "InternalTimings", $"Read: {sw_read.ElapsedMilliseconds}ms, Write: {sw_write.ElapsedMilliseconds}ms, Decompress allocate: {sw_decompress_alloc.ElapsedMilliseconds}ms, Decompress lock: {sw_decompress_locking.ElapsedMilliseconds}ms, Decompress read: {sw_decompress_read.ElapsedMilliseconds}ms, Verify: {sw_verify.ElapsedMilliseconds}ms"); } } catch (Exception ex) From 97df02c57c633e1cf0006d1abbb3e3580080e892 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 27 Feb 2025 14:48:08 +0100 Subject: [PATCH 270/362] Fixed missing docstrings --- Duplicati/Library/Main/Operation/Restore/VolumeDecompressor.cs | 1 + Duplicati/Library/Main/Operation/Restore/VolumeDecryptor.cs | 2 ++ 2 files changed, 3 insertions(+) diff --git a/Duplicati/Library/Main/Operation/Restore/VolumeDecompressor.cs b/Duplicati/Library/Main/Operation/Restore/VolumeDecompressor.cs index 9809a478b..2c2c4da6f 100644 --- a/Duplicati/Library/Main/Operation/Restore/VolumeDecompressor.cs +++ b/Duplicati/Library/Main/Operation/Restore/VolumeDecompressor.cs @@ -42,6 +42,7 @@ namespace Duplicati.Library.Main.Operation.Restore /// /// Runs the volume decompressor process. /// + /// The named channels for the restore operation. /// The restore options public static Task Run(Channels channels, Options options) { diff --git a/Duplicati/Library/Main/Operation/Restore/VolumeDecryptor.cs b/Duplicati/Library/Main/Operation/Restore/VolumeDecryptor.cs index 0b76a9a7a..8f3be3273 100644 --- a/Duplicati/Library/Main/Operation/Restore/VolumeDecryptor.cs +++ b/Duplicati/Library/Main/Operation/Restore/VolumeDecryptor.cs @@ -42,6 +42,8 @@ namespace Duplicati.Library.Main.Operation.Restore /// /// Runs the volume decryptor process. /// + /// The named channels for the restore operation. + /// The backend manager. /// The restore options. public static Task Run(Channels channels, IBackendManager backend, Options options) { From dbe413d8891fbb599fb5bd0341845008ae4a0c9b Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 28 Feb 2025 10:26:07 +0100 Subject: [PATCH 271/362] Fixed the restore BlockManager block counting logic --- .../Main/Operation/Restore/BlockManager.cs | 33 +++++++++++++++---- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs index e9e351f51..4f3849239 100644 --- a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs @@ -129,19 +129,28 @@ namespace Duplicati.Library.Main.Operation.Restore m_block_cache = new MemoryCache(cache_options); m_entry_options.RegisterPostEvictionCallback(async (key, value, reason, state) => { + bool was_present = false; while (true) { lock (m_blockcount_lock) { - if (m_active_readers.TryGetValue((long)key, out var ac) && ac == 0) + if (m_active_readers.TryGetValue((long)key, out var ac)) { - m_block_cache.Remove(key); - break; + if (ac == 0) + { + m_block_cache.Remove(key); + was_present = true; + break; + } } + else + break; } + await Task.Delay(10).ConfigureAwait(false); } - ArrayPool.Shared.Return((byte[])value); + if (was_present) + ArrayPool.Shared.Return((byte[])value); }); this.readers = readers; sw_cacheevict = options.InternalProfiling ? new() : null; @@ -175,7 +184,11 @@ namespace Duplicati.Library.Main.Operation.Restore { sw_checkcounts?.Start(); - m_active_readers[blockRequest.BlockID] = m_active_readers.TryGetValue(blockRequest.BlockID, out var ac) ? ac - 1 : 0; + var active = m_active_readers.TryGetValue(blockRequest.BlockID, out var ac) ? ac - 1 : 0; + if (active == 0) + m_active_readers.Remove(blockRequest.BlockID, out var _); + else + m_active_readers[blockRequest.BlockID] = active; var block_count = m_blockcount.TryGetValue(blockRequest.BlockID, out var c) ? c - 1 : 0; @@ -321,7 +334,12 @@ namespace Duplicati.Library.Main.Operation.Restore { var blocks = m_blockcount.Where(x => x.Value != 0).Select(x => x.Key).ToArray(); var blockids = string.Join(", ", blocks); - Logging.Log.WriteWarningMessage(LOGTAG, "BlockCountError", null, $"Block count in SleepableDictionarys block table is not zero: {blockcount}{Environment.NewLine}");//Blocks: {blockids}"); + Logging.Log.WriteWarningMessage(LOGTAG, "BlockCountError", null, $"Block count in SleepableDictionarys block table is not zero: {blockcount}{Environment.NewLine}"); + } + + if (m_active_readers.Count > 0) + { + Logging.Log.WriteWarningMessage(LOGTAG, "BlockCountError", null, $"There are still {m_active_readers.Count} files being read by {m_active_readers.Sum(x => x.Value)} readers"); } if (volumecount != 0) @@ -338,7 +356,8 @@ namespace Duplicati.Library.Main.Operation.Restore if (m_block_cache.Count > 0) { - Logging.Log.WriteWarningMessage(LOGTAG, "BlockCacheError", null, $"Internal Block cache is not empty: {m_block_cache.Count}"); + Logging.Log.WriteWarningMessage(LOGTAG, "BlockCacheMismatch", null, $"Internal Block cache is not empty: {m_block_cache.Count}"); + Logging.Log.WriteWarningMessage(LOGTAG, "BlockCacheMismatch", null, $"Block counts in cache ({m_blockcount.Count}): {string.Join(", ", m_blockcount.Select(x => x.Value))}"); } } From 6e08affd1a549e6074e89e19313533d4fe71e417 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 28 Feb 2025 10:27:02 +0100 Subject: [PATCH 272/362] Rather than having the tempfile database create new ID's for each entry, reuse the File.ID so the connections that do not have access to the temp table can access the same data. --- .../Main/Database/LocalRestoreDatabase.cs | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index 6dd245fa8..10aef6e8c 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -242,7 +242,7 @@ namespace Duplicati.Library.Main.Database if (filter == null || filter.Empty) { // Simple case, restore everything - cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") SELECT ""File"".""Path"", ""File"".""BlocksetID"", ""File"".""MetadataID"", 0 FROM ""File"", ""FilesetEntry"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FilesetEntry"".""FilesetID"" = ? ", m_tempfiletable); + cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") SELECT ""File"".""ID"", ""File"".""Path"", ""File"".""BlocksetID"", ""File"".""MetadataID"", 0 FROM ""File"", ""FilesetEntry"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FilesetEntry"".""FilesetID"" = ? ", m_tempfiletable); cmd.AddParameter(filesetId); cmd.ExecuteNonQuery(); } @@ -266,7 +266,7 @@ namespace Duplicati.Library.Main.Database cmd.ExecuteNonQuery(); } - cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") SELECT ""File"".""Path"", ""File"".""BlocksetID"", ""File"".""MetadataID"", 0 FROM ""File"", ""FilesetEntry"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FilesetEntry"".""FilesetID"" = ? AND ""Path"" IN (SELECT DISTINCT ""Path"" FROM ""{1}"") ", m_tempfiletable, m_filenamestable); + cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") SELECT ""File"".""ID"", ""File"".""Path"", ""File"".""BlocksetID"", ""File"".""MetadataID"", 0 FROM ""File"", ""FilesetEntry"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FilesetEntry"".""FilesetID"" = ? AND ""Path"" IN (SELECT DISTINCT ""Path"" FROM ""{1}"") ", m_tempfiletable, m_filenamestable); cmd.SetParameterValue(0, filesetId); var c = cmd.ExecuteNonQuery(); @@ -296,13 +296,14 @@ namespace Duplicati.Library.Main.Database { // Restore but filter elements based on the filter expression // If this is too slow, we could add a special handler for wildcard searches too - cmd.CommandText = @"SELECT ""File"".""Path"", ""File"".""BlocksetID"", ""File"".""MetadataID"" FROM ""File"", ""FilesetEntry"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FilesetID"" = ?"; + cmd.CommandText = @"SELECT ""File"".""ID"", ""File"".""Path"", ""File"".""BlocksetID"", ""File"".""MetadataID"" FROM ""File"", ""FilesetEntry"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FilesetID"" = ?"; cmd.AddParameter(filesetId); - object[] values = new object[3]; + object[] values = new object[4]; using (var cmd2 = m_connection.CreateCommand()) { - cmd2.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") VALUES (?,?,?,0)", m_tempfiletable); + cmd2.CommandText = string.Format(@"INSERT INTO ""{0}"" (""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") VALUES (?,?,?,?,0)", m_tempfiletable); + cmd2.AddParameter(); cmd2.AddParameter(); cmd2.AddParameter(); cmd2.AddParameter(); @@ -311,11 +312,12 @@ namespace Duplicati.Library.Main.Database while (rd.Read()) { rd.GetValues(values); - if (values[0] != null && values[0] != DBNull.Value && Library.Utility.FilterExpression.Matches(filter, values[0].ToString())) + if (values[1] != null && values[1] != DBNull.Value && Library.Utility.FilterExpression.Matches(filter, values[1].ToString())) { cmd2.SetParameterValue(0, values[0]); cmd2.SetParameterValue(1, values[1]); cmd2.SetParameterValue(2, values[2]); + cmd2.SetParameterValue(3, values[3]); cmd2.ExecuteNonQuery(); } } @@ -323,6 +325,7 @@ namespace Duplicati.Library.Main.Database } //creating indexes after insertion is much faster + cmd.ExecuteNonQuery(string.Format(@"CREATE INDEX ""{0}_ID"" ON ""{0}"" (""ID"")", m_tempfiletable)); cmd.ExecuteNonQuery(string.Format(@"CREATE INDEX ""{0}_TargetPath"" ON ""{0}"" (""TargetPath"")", m_tempfiletable)); cmd.ExecuteNonQuery(string.Format(@"CREATE INDEX ""{0}_Path"" ON ""{0}"" (""Path"")", m_tempfiletable)); @@ -1073,11 +1076,11 @@ namespace Duplicati.Library.Main.Database cmd.CommandText = $@" SELECT Block.ID, Block.Hash, Block.Size, Block.VolumeID - FROM FileLookup - INNER JOIN Metadataset ON FileLookup.MetadataID = Metadataset.ID + FROM File + INNER JOIN Metadataset ON File.MetadataID = Metadataset.ID INNER JOIN BlocksetEntry ON Metadataset.BlocksetID = BlocksetEntry.BlocksetID INNER JOIN Block ON BlocksetEntry.BlockID = Block.ID - WHERE FileLookup.ID = ? + WHERE File.ID = ? "; cmd.AddParameter(); cmd.SetParameterValue(0, fileID); From c61cc778a836779745d20d933a4f91a821a2f672 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 28 Feb 2025 11:18:23 +0100 Subject: [PATCH 273/362] Log remote filesets This commit adds the number of known remote filesets to the log output. --- Duplicati/CommandLine/CLI/Commands.cs | 1 + .../Library/Interface/ResultInterfaces.cs | 83 ++++++++++--------- .../Library/Main/Operation/BackupHandler.cs | 1 + .../Main/Operation/FilelistProcessor.cs | 1 + Duplicati/Library/Main/ResultClasses.cs | 1 + Duplicati/Library/RestAPI/Runner.cs | 1 + 6 files changed, 47 insertions(+), 41 deletions(-) diff --git a/Duplicati/CommandLine/CLI/Commands.cs b/Duplicati/CommandLine/CLI/Commands.cs index 76425af0c..9aa3a8e99 100644 --- a/Duplicati/CommandLine/CLI/Commands.cs +++ b/Duplicati/CommandLine/CLI/Commands.cs @@ -668,6 +668,7 @@ namespace Duplicati.CommandLine if (parsedStats.KnownFileCount > 0) { output.MessageEvent(string.Format(" Remote files: {0}", parsedStats.KnownFileCount)); + output.MessageEvent(string.Format(" Remote filesets: {0}", parsedStats.KnownFilesets)); output.MessageEvent(string.Format(" Remote size: {0}", Library.Utility.Utility.FormatSizeString(parsedStats.KnownFileSize))); } diff --git a/Duplicati/Library/Interface/ResultInterfaces.cs b/Duplicati/Library/Interface/ResultInterfaces.cs index 1848e98ca..0c886c43b 100644 --- a/Duplicati/Library/Interface/ResultInterfaces.cs +++ b/Duplicati/Library/Interface/ResultInterfaces.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -38,7 +38,7 @@ namespace Duplicati.Library.Interface DateTime BeginTime { get; } DateTime EndTime { get; } TimeSpan Duration { get; } - + IEnumerable Errors { get; } IEnumerable Warnings { get; } IEnumerable Messages { get; } @@ -64,6 +64,7 @@ namespace Duplicati.Library.Interface long UnknownFileCount { get; set; } long KnownFileCount { get; set; } long KnownFileSize { get; set; } + long KnownFilesets { get; set; } DateTime LastBackupDate { get; set; } long BackupListCount { get; set; } long TotalQuotaSpace { get; set; } @@ -116,14 +117,14 @@ namespace Duplicati.Library.Interface IEnumerable LogMessages { get; } IEnumerable RemoteVolumes { get; } } - + public interface IDeleteResults : IBasicResults { IEnumerable> DeletedSets { get; } ICompactResults CompactResults { get; } bool Dryrun { get; } } - + public interface IBackupResults : IBasicResults, IBackendStatsticsReporter { long DeletedFiles { get; } @@ -143,16 +144,16 @@ namespace Duplicati.Library.Interface long ModifiedFolders { get; } long ModifiedSymlinks { get; } long AddedSymlinks { get; } - long DeletedSymlinks { get; } + long DeletedSymlinks { get; } bool PartialBackup { get; } bool Dryrun { get; } - + ICompactResults CompactResults { get; } IVacuumResults VacuumResults { get; } IDeleteResults DeleteResults { get; } IRepairResults RepairResults { get; } } - + public interface IRestoreResults : IBasicResults { long RestoredFiles { get; } @@ -163,10 +164,10 @@ namespace Duplicati.Library.Interface long DeletedFiles { get; } long DeletedFolders { get; } long DeletedSymlinks { get; } - + IRecreateDatabaseResults RecreateDatabaseResults { get; } } - + public interface IRecreateDatabaseResults : IBasicResults { } @@ -188,23 +189,23 @@ namespace Duplicati.Library.Interface IVacuumResults VacuumResults { get; } } - + public interface ICreateLogDatabaseResults : IBasicResults { string TargetPath { get; } } - + public interface IRestoreControlFilesResults : IBasicResults { IEnumerable Files { get; } } - + public interface IRepairResults : IBasicResults { IRecreateDatabaseResults RecreateDatabaseResults { get; } } - - + + /// /// The possible change types for an entry /// @@ -222,8 +223,8 @@ namespace Duplicati.Library.Interface /// The element was modified /// Modified - } - + } + /// /// The possible entry types /// @@ -242,16 +243,16 @@ namespace Duplicati.Library.Interface /// File } - + public interface IListChangesResults : IBasicResults { DateTime BaseVersionTimestamp { get; } DateTime CompareVersionTimestamp { get; } long BaseVersionIndex { get; } long CompareVersionIndex { get; } - - IEnumerable> ChangeDetails { get; } - + + IEnumerable> ChangeDetails { get; } + long AddedFolders { get; } long AddedSymlinks { get; } long AddedFiles { get; } @@ -263,14 +264,14 @@ namespace Duplicati.Library.Interface long ModifiedFolders { get; } long ModifiedSymlinks { get; } long ModifiedFiles { get; } - + long PreviousSize { get; } long CurrentSize { get; } long AddedSize { get; } long DeletedSize { get; } } - + /// /// The status of a change in a test entry /// @@ -298,7 +299,7 @@ namespace Duplicati.Library.Interface { IEnumerable>>> Verifications { get; } } - + public interface ITestFilterResults : IBasicResults { long FileSize { get; set; } diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 93a8e3222..ba4165515 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -530,6 +530,7 @@ namespace Duplicati.Library.Main.Operation if (result.BackendWriter != null) { result.BackendWriter.KnownFileCount = database.GetRemoteVolumes().Count(); + result.BackendWriter.KnownFilesets = database.GetRemoteVolumes().Count(x => x.Type == RemoteVolumeType.Files); result.BackendWriter.KnownFileSize = database.GetRemoteVolumes().Select(x => Math.Max(0, x.Size)).Sum(); result.BackendWriter.UnknownFileCount = 0; diff --git a/Duplicati/Library/Main/Operation/FilelistProcessor.cs b/Duplicati/Library/Main/Operation/FilelistProcessor.cs index f1d10fdfb..37cbdbd88 100644 --- a/Duplicati/Library/Main/Operation/FilelistProcessor.cs +++ b/Duplicati/Library/Main/Operation/FilelistProcessor.cs @@ -234,6 +234,7 @@ namespace Duplicati.Library.Main.Operation select n).ToList(); log.KnownFileCount = remotelist.Count; + log.KnownFilesets = filesets.Count; long knownFileSize = remotelist.Select(x => Math.Max(0, x.File.Size)).Sum(); log.KnownFileSize = knownFileSize; log.UnknownFileCount = unknownlist.Count; diff --git a/Duplicati/Library/Main/ResultClasses.cs b/Duplicati/Library/Main/ResultClasses.cs index 846332274..25123a65d 100644 --- a/Duplicati/Library/Main/ResultClasses.cs +++ b/Duplicati/Library/Main/ResultClasses.cs @@ -99,6 +99,7 @@ namespace Duplicati.Library.Main public long UnknownFileCount { get; set; } public long KnownFileCount { get; set; } public long KnownFileSize { get; set; } + public long KnownFilesets { get; set; } public DateTime LastBackupDate { get; set; } public long BackupListCount { get; set; } public long TotalQuotaSpace { get; set; } diff --git a/Duplicati/Library/RestAPI/Runner.cs b/Duplicati/Library/RestAPI/Runner.cs index 2014358c8..71171bf87 100644 --- a/Duplicati/Library/RestAPI/Runner.cs +++ b/Duplicati/Library/RestAPI/Runner.cs @@ -736,6 +736,7 @@ namespace Duplicati.Server backup.Metadata["TargetFilesSize"] = r.KnownFileSize.ToString(); backup.Metadata["TargetFilesCount"] = r.KnownFileCount.ToString(); + backup.Metadata["TargetFilesetsCount"] = r.KnownFilesets.ToString(); backup.Metadata["TargetSizeString"] = Duplicati.Library.Utility.Utility.FormatSizeString(r.KnownFileSize); } } From 3298b897e75a2c26ff5ec976f9ddd97ebc741173 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 28 Feb 2025 16:25:44 +0100 Subject: [PATCH 274/362] Version bump to v2.1.0.110-2.1.0.110_canary_2025-02-28 You can download this build from: Binaries: https://updates.duplicati.com/canary/?version=2.1.0.110 Signature file: https://updates.duplicati.com/canary/duplicati-2.1.0.110_canary_2025-02-28.signatures.zip --- ReleaseBuilder/build_version.txt | 2 +- changelog.txt | 53 ++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/ReleaseBuilder/build_version.txt b/ReleaseBuilder/build_version.txt index 60deb0e83..4e5414f9a 100644 --- a/ReleaseBuilder/build_version.txt +++ b/ReleaseBuilder/build_version.txt @@ -1 +1 @@ -2.1.0.109 \ No newline at end of file +2.1.0.110 \ No newline at end of file diff --git a/changelog.txt b/changelog.txt index 165df0823..e2acdeba2 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,56 @@ +2025-02-28 - 2.1.0.110_canary_2025-02-28 +========== +This release is a canary release intended to be used for testing. + +# Major changes in this version + +- Restore improvements +Building further on the new restore engine, this version has significant performance speedups during restore. + +- New signing keys +The builds are now signed by "Duplicati Inc" for MacOS and Windows. +The Windows signatures are now done with an extended validation (EV) certificate and should not trigger Windows Defender warnings. + +- New enumeration logic +The enumeration of files and folders has been rewritten to better support some edge cases. + +- Improved pagination support +All backends are updated to support the new list methods that supports pagination and cancellation. +The test method will now only load a single page of results, instead of listing all items. + +- Support for backend-as-source +This version has support for backups of both remote and local data. +In this version, S3, IDrive, SSH and CIFS sources are supported. +The UI does not yet support this, but it is possible to read data from a remote location by "mounting" it into the local filesystem. +For the commandline (and manual text entry in the UI) enter sources such as: +``` +// Linux/MacOS +@/mnt/s3-data|s3://example?auth-username=... + +// Windows +@X:\server1|cifs://server/share?auth-username=... +``` + +# Potentially breaking changes (from 2.1.0.109) + - Location of data folder has changed in some cases, refer to previous release for details + - Permissions are now applied to the data folder, refer to previous release for details + - New user interface, `ngclient` is the default; there is a button to switch between the two + +## Detailed list of changes: +- Improved script examples +- Fixed double-click TrayIcon on Windows, thanks @Alex-Heard +- Fixed an issue with CORS headers +- Parallel downloads for restore +- Report interface and port in Webserver startup message +- Fixed missing example script on Windows +- Fixed a case where a leading slash could be stripped from the path or a URL +- Added additional Infomaniak endpoints, thanks @ochaze +- Backend listing is now async for better pagination and cancellation support +- Updated signing support for the release builds +- Added a link from `ngax` to `ngclient` (the other way already exists) +- Added alternate source providers and new enumeration process +- Performance optimizations for restore process + 2025-02-11 - 2.1.0.109_canary_2025-02-11 ========== This release is a canary release intended to be used for testing. From 805ad42472d119bf7085163222bf75da6eb1c4ec Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 3 Mar 2025 17:08:36 +0100 Subject: [PATCH 275/362] The MemoryCache in the BlockManager was removing its own entry in its post-eviction callback --- Duplicati/Library/Main/Operation/Restore/BlockManager.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs index 4f3849239..a76d6cb5d 100644 --- a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs @@ -138,7 +138,6 @@ namespace Duplicati.Library.Main.Operation.Restore { if (ac == 0) { - m_block_cache.Remove(key); was_present = true; break; } From 0ea45b002335fd1cc499160a12eb3cc40be9776a Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 3 Mar 2025 17:09:20 +0100 Subject: [PATCH 276/362] Rather than sleeping on the active readers spinlock, the post-eviction callback should yield to whichever other thread is trying to take the lock. --- Duplicati/Library/Main/Operation/Restore/BlockManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs index a76d6cb5d..014d73f31 100644 --- a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs @@ -146,7 +146,7 @@ namespace Duplicati.Library.Main.Operation.Restore break; } - await Task.Delay(10).ConfigureAwait(false); + await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding); } if (was_present) ArrayPool.Shared.Return((byte[])value); From 96149bbcf3ee5e9b1687bca7c6d3623b6298f28e Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 3 Mar 2025 17:09:46 +0100 Subject: [PATCH 277/362] The MemoryCache in BlockManager cannot be null any more. --- Duplicati/Library/Main/Operation/Restore/BlockManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs index 014d73f31..138d92d45 100644 --- a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs @@ -258,7 +258,7 @@ namespace Duplicati.Library.Main.Operation.Restore } // Check if the block is already in the cache, and return it if it is. - if (m_block_cache != null && m_block_cache.TryGetValue(block_request.BlockID, out byte[] value)) + if (m_block_cache.TryGetValue(block_request.BlockID, out byte[] value)) { return Task.FromResult(value); } From ef2c7e5187f358848cb3dd3b00895d37d8e16f0e Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 3 Mar 2025 17:18:12 +0100 Subject: [PATCH 278/362] Keeping a reference to the temp file ensures it is not caught by the garbage collector, which in turn disposes the temp file, in turn in turn deleting it. --- Duplicati/Library/Main/Operation/Restore/VolumeDecryptor.cs | 2 +- Duplicati/Library/Main/Operation/Restore/VolumeManager.cs | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/VolumeDecryptor.cs b/Duplicati/Library/Main/Operation/Restore/VolumeDecryptor.cs index 8f3be3273..22fb03e88 100644 --- a/Duplicati/Library/Main/Operation/Restore/VolumeDecryptor.cs +++ b/Duplicati/Library/Main/Operation/Restore/VolumeDecryptor.cs @@ -74,7 +74,7 @@ namespace Duplicati.Library.Main.Operation.Restore sw_write?.Start(); // Pass the decrypted volume to the `VolumeDecompressor` process. - await self.Output.WriteAsync((volume_id, bvr)).ConfigureAwait(false); + await self.Output.WriteAsync((volume_id, tmpfile, bvr)).ConfigureAwait(false); sw_write?.Stop(); } } diff --git a/Duplicati/Library/Main/Operation/Restore/VolumeManager.cs b/Duplicati/Library/Main/Operation/Restore/VolumeManager.cs index 842de33d5..788c25790 100644 --- a/Duplicati/Library/Main/Operation/Restore/VolumeManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/VolumeManager.cs @@ -59,6 +59,7 @@ namespace Duplicati.Library.Main.Operation.Restore async self => { Dictionary cache = []; + Dictionary tmpfiles = []; Dictionary> in_flight = []; Stopwatch sw_cache_set = options.InternalProfiling ? new() : null; @@ -83,6 +84,8 @@ namespace Duplicati.Library.Main.Operation.Restore sw_cache_evict?.Start(); cache.Remove(request.VolumeID, out var reader); reader?.Dispose(); + tmpfiles.Remove(request.VolumeID, out var tmpfile); + tmpfile?.Dispose(); sw_cache_evict?.Stop(); } else @@ -108,10 +111,11 @@ namespace Duplicati.Library.Main.Operation.Restore } break; } - case (long volume_id, BlockVolumeReader reader): + case (long volume_id, TempFile tmpfile, BlockVolumeReader reader): { sw_cache_set?.Start(); cache[volume_id] = reader; + tmpfiles[volume_id] = tmpfile; sw_cache_set?.Stop(); sw_wakeup?.Start(); foreach (var request in in_flight[volume_id]) From 7c41949e9e602f91ca0f3dba2a7d11251c987d9e Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 3 Mar 2025 17:20:50 +0100 Subject: [PATCH 279/362] Disposing a TempFile through finalizing now gives a warning, as this should have been done by the instantiating entity. --- Duplicati/Library/Utility/TempFile.cs | 46 +++++++++++++++------------ 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/Duplicati/Library/Utility/TempFile.cs b/Duplicati/Library/Utility/TempFile.cs index 3a9db951c..20aed7386 100644 --- a/Duplicati/Library/Utility/TempFile.cs +++ b/Duplicati/Library/Utility/TempFile.cs @@ -1,23 +1,23 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; @@ -37,6 +37,7 @@ namespace Duplicati.Library.Utility private string m_path; private bool m_protect; + private bool m_disposed = false; #if DEBUG //In debug mode, we track the creation of temporary files, and encode the generating method into the name @@ -179,6 +180,8 @@ namespace Duplicati.Library.Utility protected void Dispose(bool disposing) { + m_disposed = true; + if (disposing) GC.SuppressFinalize(this); @@ -204,6 +207,9 @@ namespace Duplicati.Library.Utility ~TempFile() { + if (!m_disposed) + Logging.Log.WriteWarningMessage("TempFile", "Finalizer", null, "TempFile was not disposed, cleaning up"); + Dispose(false); } From ab256086167d25b6af4c613084779c4fe125388f Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 3 Mar 2025 17:20:57 +0100 Subject: [PATCH 280/362] Remove whitespace --- Duplicati/Library/Utility/TempFile.cs | 30 +++++++++++++-------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Utility/TempFile.cs b/Duplicati/Library/Utility/TempFile.cs index 20aed7386..aa9853ea0 100644 --- a/Duplicati/Library/Utility/TempFile.cs +++ b/Duplicati/Library/Utility/TempFile.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; From ccd10f8ead4bdb6fae7236c45ab0f865a3949189 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 3 Mar 2025 17:30:31 +0100 Subject: [PATCH 281/362] FileProcessor is now sending explicit copies of each request around. --- .../Main/Operation/Restore/FileProcessor.cs | 93 ++++++++++++++++--- 1 file changed, 80 insertions(+), 13 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs index 7027923aa..d393c5f71 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs @@ -154,9 +154,17 @@ namespace Duplicati.Library.Main.Operation.Restore // Notify to the cache that we use local blocks. foreach (var block in verified_blocks) { - block.CacheDecrEvict = true; sw_req?.Start(); - await block_request.WriteAsync(block).ConfigureAwait(false); + await block_request.WriteAsync( + new BlockRequest( + block.BlockID, + block.BlockOffset, + block.BlockHash, + block.BlockSize, + block.VolumeID, + true + ) + ).ConfigureAwait(false); sw_req?.Stop(); } sw_work_notify_local?.Stop(); @@ -192,8 +200,16 @@ namespace Duplicati.Library.Main.Operation.Restore fs.SetLength(0); if (missing_blocks.Count != 0) { - blocks[0].CacheDecrEvict = true; - await block_request.WriteAsync(blocks[0]).ConfigureAwait(false); + await block_request.WriteAsync( + new BlockRequest( + blocks[0].BlockID, + blocks[0].BlockOffset, + blocks[0].BlockHash, + blocks[0].BlockSize, + blocks[0].VolumeID, + true + ) + ).ConfigureAwait(false); } } sw_work_empty_file?.Stop(); @@ -243,7 +259,16 @@ namespace Duplicati.Library.Main.Operation.Restore int j = 0; for (int i = 0; i < (int)Math.Min(missing_blocks.Count, burst); i++) { - await block_request.WriteAsync(missing_blocks[i]).ConfigureAwait(false); + await block_request.WriteAsync( + new BlockRequest( + missing_blocks[i].BlockID, + missing_blocks[i].BlockOffset, + missing_blocks[i].BlockHash, + missing_blocks[i].BlockSize, + missing_blocks[i].VolumeID, + false + ) + ).ConfigureAwait(false); } sw_req?.Stop(); @@ -266,7 +291,16 @@ namespace Duplicati.Library.Main.Operation.Restore sw_req?.Start(); if (j < missing_blocks.Count - burst) { - await block_request.WriteAsync(missing_blocks[j + burst]).ConfigureAwait(false); + await block_request.WriteAsync( + new BlockRequest( + missing_blocks[j + burst].BlockID, + missing_blocks[j + burst].BlockOffset, + missing_blocks[j + burst].BlockHash, + missing_blocks[j + burst].BlockSize, + missing_blocks[j + burst].VolumeID, + false + ) + ).ConfigureAwait(false); } sw_req?.Stop(); @@ -291,8 +325,16 @@ namespace Duplicati.Library.Main.Operation.Restore bytes_written += blocks[i].BlockSize; j++; sw_req?.Start(); - blocks[i].CacheDecrEvict = true; - await block_request.WriteAsync(blocks[i]).ConfigureAwait(false); + await block_request.WriteAsync( + new BlockRequest( + blocks[i].BlockID, + blocks[i].BlockOffset, + blocks[i].BlockHash, + blocks[i].BlockSize, + blocks[i].VolumeID, + true + ) + ).ConfigureAwait(false); sw_req?.Stop(); } else @@ -505,7 +547,16 @@ namespace Duplicati.Library.Main.Operation.Restore { sw_work?.Stop(); sw_req?.Start(); - await block_request.WriteAsync(block).ConfigureAwait(false); + await block_request.WriteAsync( + new BlockRequest( + block.BlockID, + block.BlockOffset, + block.BlockHash, + block.BlockSize, + block.VolumeID, + false + ) + ).ConfigureAwait(false); sw_req?.Stop(); sw_resp?.Start(); @@ -517,8 +568,16 @@ namespace Duplicati.Library.Main.Operation.Restore sw_work?.Stop(); sw_req?.Start(); - block.CacheDecrEvict = true; - await block_request.WriteAsync(block).ConfigureAwait(false); + await block_request.WriteAsync( + new BlockRequest( + block.BlockID, + block.BlockOffset, + block.BlockHash, + block.BlockSize, + block.VolumeID, + true + ) + ).ConfigureAwait(false); sw_req?.Stop(); sw_work?.Start(); @@ -715,8 +774,16 @@ namespace Duplicati.Library.Main.Operation.Restore } bytes_read += read; bytes_written += read; - blocks[j].CacheDecrEvict = true; - await block_request.WriteAsync(blocks[j]).ConfigureAwait(false); + await block_request.WriteAsync( + new BlockRequest( + blocks[j].BlockID, + blocks[j].BlockOffset, + blocks[j].BlockHash, + blocks[j].BlockSize, + blocks[j].VolumeID, + true + ) + ).ConfigureAwait(false); verified_blocks.Add(blocks[j]); } } From 1650399fe630abeab6f660bb596cb46393885876 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 3 Mar 2025 17:32:58 +0100 Subject: [PATCH 282/362] Added explicit logging to trace a block request through the new restore network --- Duplicati/Library/Main/Operation/Restore/BlockManager.cs | 6 ++++++ Duplicati/Library/Main/Operation/Restore/VolumeManager.cs | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs index 138d92d45..77fa29281 100644 --- a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs @@ -252,6 +252,8 @@ namespace Duplicati.Library.Main.Operation.Restore /// A `Task` holding the data block. public Task Get(BlockRequest block_request) { + Logging.Log.WriteExplicitMessage(LOGTAG, "BlockCacheGet", "Getting block {0} from cache", block_request.BlockID); + lock (m_blockcount_lock) { m_active_readers[block_request.BlockID] = m_active_readers.TryGetValue(block_request.BlockID, out var c) ? c + 1 : 1; @@ -269,6 +271,8 @@ namespace Duplicati.Library.Main.Operation.Restore var new_tcs = m_waiters.GetOrAdd(block_request.BlockID, tcs); if (tcs == new_tcs) { + Logging.Log.WriteExplicitMessage(LOGTAG, "BlockCacheGet", "Requesting block {0} from volume {1}", block_request.BlockID, block_request.VolumeID); + // We are the first to request this block m_volume_request.Write(block_request); } @@ -289,6 +293,8 @@ namespace Duplicati.Library.Main.Operation.Restore /// The byte[] buffer holding the block data. public void Set(BlockRequest blockRequest, byte[] value) { + Logging.Log.WriteExplicitMessage(LOGTAG, "BlockCacheSet", "Setting block {0} in cache", blockRequest.BlockID); + m_block_cache.Set(blockRequest.BlockID, value); // Notify any waiters that the block is available. diff --git a/Duplicati/Library/Main/Operation/Restore/VolumeManager.cs b/Duplicati/Library/Main/Operation/Restore/VolumeManager.cs index 788c25790..700de7b88 100644 --- a/Duplicati/Library/Main/Operation/Restore/VolumeManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/VolumeManager.cs @@ -90,13 +90,16 @@ namespace Duplicati.Library.Main.Operation.Restore } else { + Logging.Log.WriteExplicitMessage(LOGTAG, "VolumeRequest", "Requesting block {0} from volume {1}", request.BlockID, request.VolumeID); sw_request?.Start(); if (cache.TryGetValue(request.VolumeID, out BlockVolumeReader reader)) { + Logging.Log.WriteExplicitMessage(LOGTAG, "VolumeRequest", "Block {0} found in cache", request.BlockID); await self.DecompressRequest.WriteAsync((request, reader)).ConfigureAwait(false); } else { + Logging.Log.WriteExplicitMessage(LOGTAG, "VolumeRequest", "Block {0} not found in cache, requesting volume {1}", request.BlockID, request.VolumeID); if (in_flight.TryGetValue(request.VolumeID, out var waiters)) { waiters.Add(request); @@ -120,6 +123,7 @@ namespace Duplicati.Library.Main.Operation.Restore sw_wakeup?.Start(); foreach (var request in in_flight[volume_id]) { + Logging.Log.WriteExplicitMessage(LOGTAG, "VolumeRequest", "Requesting block {0} from volume {1}", request.BlockID, volume_id); await self.DecompressRequest.WriteAsync((request, reader)).ConfigureAwait(false); } in_flight.Remove(volume_id); From 60fa777a8e943c7dafe15f277b06ba638fa35362 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 3 Mar 2025 17:34:27 +0100 Subject: [PATCH 283/362] Added a timeout for block requests --- .../Library/Main/Operation/Restore/BlockManager.cs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs index 77fa29281..140032238 100644 --- a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs @@ -273,6 +273,17 @@ namespace Duplicati.Library.Main.Operation.Restore { Logging.Log.WriteExplicitMessage(LOGTAG, "BlockCacheGet", "Requesting block {0} from volume {1}", block_request.BlockID, block_request.VolumeID); + // TODO add a debugging timeout. As this can potentially take a long time in a real setup with a slow machine and slew connection. + Task.Run(async () => + { + await Task.Delay(TimeSpan.FromMinutes(5)).ConfigureAwait(false); + if (!tcs.Task.IsCompleted) + { + tcs.SetException(new TimeoutException("Block request timeout")); + Logging.Log.WriteErrorMessage(LOGTAG, "BlockRequestTimeout", null, $"Block request for block {block_request.BlockID} timed out"); + } + }); + // We are the first to request this block m_volume_request.Write(block_request); } From cbb639651aeb9f9d5f51b35507fc0d84d71e5c39 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 3 Mar 2025 17:39:25 +0100 Subject: [PATCH 284/362] Added support for handling control messages. This enables automatic reporting setup via remote management. --- Duplicati/Agent/Program.cs | 45 +++---- .../RemoteControl/ExchangeDataTypes.cs | 38 +++++- .../RemoteControl/KeepRemoteConnection.cs | 124 ++++++++++++++++-- .../Library/RestAPI/Database/Connection.cs | 50 +++---- .../RestAPI/Database/ServerSettings.cs | 15 +++ Duplicati/Library/RestAPI/Runner.cs | 12 ++ Duplicati/Library/RestAPI/Scheduler.cs | 2 +- .../Endpoints/V1/ServerSetting.cs | 1 - .../ServiceCollectionsExtensions.cs | 1 + .../Services/IRemoteControllerHandler.cs | 58 ++++++++ .../Services/RemoteControllerHandler.cs | 120 +++++++++++++++++ .../Services/RemoteControllerService.cs | 58 +------- 12 files changed, 406 insertions(+), 118 deletions(-) create mode 100644 Duplicati/WebserverCore/Services/IRemoteControllerHandler.cs create mode 100644 Duplicati/WebserverCore/Services/RemoteControllerHandler.cs diff --git a/Duplicati/Agent/Program.cs b/Duplicati/Agent/Program.cs index 12451ea3d..c02ecbdd0 100644 --- a/Duplicati/Agent/Program.cs +++ b/Duplicati/Agent/Program.cs @@ -34,7 +34,7 @@ using Duplicati.Library.RemoteControl; using Duplicati.Library.RestAPI; using Duplicati.Library.Utility; using Duplicati.Server; -using Duplicati.WebserverCore.Abstractions; +using Duplicati.WebserverCore.Services; using Microsoft.Extensions.DependencyInjection; namespace Duplicati.Agent; @@ -46,12 +46,6 @@ public static class Program /// private static readonly string LogTag = Log.LogTagFromType(typeof(Program)); - /// - /// Create a random pre-shared key for the agent - /// - private static string? PreSharedKey = null; - - /// /// Commandline arguments for the agent /// @@ -275,7 +269,7 @@ public static class Program // Set the pre-shared key for the agent if (!agentConfig.DisablePreSharedKey) - WebserverCore.Middlewares.PreSharedKeyFilter.PreSharedKey = PreSharedKey = RandomNumberGenerator.GetHexString(128); + WebserverCore.Middlewares.PreSharedKeyFilter.PreSharedKey = RandomNumberGenerator.GetHexString(128); using var cts = new CancellationTokenSource(); @@ -313,8 +307,10 @@ public static class Program settings.CertificateUrl, settings.ServerCertificates, cts.Token, - c => ReKey(agentConfig, c), - (m) => OnMessage(m, agentConfig) + OnConnect, + m => ReKey(m, agentConfig), + OnControl, + OnMessage ) ); @@ -324,24 +320,19 @@ public static class Program } } - private static async Task OnMessage(KeepRemoteConnection.CommandMessage message, CommandLineArguments agentConfig) - { - Log.WriteMessage(LogMessageType.Verbose, LogTag, "OnMessage", "Received message: {0}", message); - - var provider = FIXMEGlobal.Provider.GetRequiredService(); - var token = provider.CreateAccessToken("agent", provider.TemporaryFamilyId, TimeSpan.FromMinutes(2)); - using var httpClient = FIXMEGlobal.Provider.GetRequiredService().CreateClient(); - httpClient.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token); - httpClient.BaseAddress = new System.Uri($"http://127.0.0.1:{agentConfig.WebservicePort}"); - - if (!string.IsNullOrWhiteSpace(WebserverCore.Middlewares.PreSharedKeyFilter.PreSharedKey)) - httpClient.DefaultRequestHeaders.Add(WebserverCore.Middlewares.PreSharedKeyFilter.HeaderName, PreSharedKey); - - await message.Handle(httpClient); - } - - private static Task ReKey(CommandLineArguments agentConfig, ClaimedClientData keydata) + private static Task> OnConnect(Dictionary metadata) + => FIXMEGlobal.Provider.GetRequiredService().OnConnect(metadata); + + private static Task OnControl(KeepRemoteConnection.ControlMessage message) + => FIXMEGlobal.Provider.GetRequiredService().OnControl(message); + + private static Task OnMessage(KeepRemoteConnection.CommandMessage message) + => FIXMEGlobal.Provider.GetRequiredService().OnMessage(message); + + + private static Task ReKey(ClaimedClientData keydata, CommandLineArguments agentConfig) { + // ReKey is handled here because we store the config outside of the database Log.WriteMessage(LogMessageType.Verbose, LogTag, "ReKey", "Rekeying the settings"); var settings = Settings.Load(agentConfig.AgentSettingsFile.FullName, agentConfig.AgentSettingsFilePassphrase); if (!string.IsNullOrWhiteSpace(keydata.JWT) && settings.JWT != keydata.JWT) diff --git a/Duplicati/Library/RemoteControl/ExchangeDataTypes.cs b/Duplicati/Library/RemoteControl/ExchangeDataTypes.cs index bb4003848..6b6cf7fa4 100644 --- a/Duplicati/Library/RemoteControl/ExchangeDataTypes.cs +++ b/Duplicati/Library/RemoteControl/ExchangeDataTypes.cs @@ -45,6 +45,10 @@ internal enum MessageType /// Command, /// + /// A control message + /// + Control, + /// /// An unknown command /// Unknown @@ -56,7 +60,9 @@ internal enum MessageType /// The client token /// The client public key /// The version of the client -public record AuthMessage(string Token, string PublicKey, string ClientVersion, int ProtocolVersion); +/// The protocol version of the client +/// The optional metadata to send +public record AuthMessage(string Token, string PublicKey, string ClientVersion, int ProtocolVersion, Dictionary? Metadata); /// /// A message authentication response @@ -75,6 +81,35 @@ internal sealed record AuthResultMessage(bool? Accepted, bool? WillReplaceToken, /// The version of the server public sealed record WelcomeMessage(string PublicKeyHash, string MachineName, string ServerVersion, IEnumerable SupportedProtocolVersions); +/// +/// A message with a control request +/// +/// The control command being requested +/// The parameters for the command +public sealed record ControlRequestMessage(string Command, Dictionary Parameters) +{ + /// + /// The configure report url command + /// + public const string ConfigureReportUrlSet = "configure-report-url:set"; + /// + /// A control message to return if the report url is configured + /// + public const string ConfigureReportUrlGet = "configure-report-url:get"; + /// + /// The url parameter for the configure report url command + /// + public const string ConfigureReportUrlParameter = "url"; + +} +/// +/// A message with a control response +/// +/// Whether the command was successful +/// The output of the command +/// The error message if the command failed +public sealed record ControlResponseMessage(bool Success, Dictionary? Output, string? ErrorMessage); + /// /// A message to send a command /// @@ -180,6 +215,7 @@ internal sealed record EnvelopedMessage "auth" => MessageType.Auth, "pong" => MessageType.Pong, "command" => MessageType.Command, + "control" => MessageType.Control, _ => MessageType.Unknown }; } diff --git a/Duplicati/Library/RemoteControl/KeepRemoteConnection.cs b/Duplicati/Library/RemoteControl/KeepRemoteConnection.cs index c2ad64e63..a2d25790f 100644 --- a/Duplicati/Library/RemoteControl/KeepRemoteConnection.cs +++ b/Duplicati/Library/RemoteControl/KeepRemoteConnection.cs @@ -19,10 +19,12 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +using System.Diagnostics; using System.Net; using System.Net.Http.Headers; using System.Security.Cryptography; using System.Text.Json; +using Duplicati.Library.AutoUpdater; using Duplicati.Library.Logging; namespace Duplicati.Library.RemoteControl; @@ -133,12 +135,20 @@ public class KeepRemoteConnection : IDisposable /// private RSA? _serverPublicKey; + /// + /// The callback to call when connecting + /// + private readonly Func, Task>> _onConnect; /// /// The callback to call when rekeying /// private readonly Func _onReKey; /// - /// The callback to call when a message is received + /// The callback to call when a control message is received + /// + private readonly Func _onControl; + /// + /// The callback to call when a command message is received /// private readonly Func _onMessage; @@ -168,16 +178,32 @@ public class KeepRemoteConnection : IDisposable /// /// The url to use /// The JWT token to use + /// The certificate url to use /// The server keys to use /// The token to cancel the connection - private KeepRemoteConnection(string serverUrl, string JWT, string certificateUrl, IEnumerable serverKeys, CancellationToken cancellationToken, Func onReKey, Func onMessage) + /// The callback to call when connecting + /// The callback to call when rekeying + /// The callback to call when a control message is received + /// The callback to call when a command message is received + private KeepRemoteConnection( + string serverUrl, + string JWT, + string certificateUrl, + IEnumerable serverKeys, + CancellationToken cancellationToken, + Func, Task>> onConnect, + Func onReKey, + Func onControl, + Func onMessage) { _serverUrl = serverUrl; _certificateUrl = certificateUrl; _token = JWT; _serverKeys = serverKeys; _cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + _onConnect = onConnect; _onReKey = onReKey; + _onControl = onControl; _onMessage = onMessage; _client = new Websocket.Client.WebsocketClient(new Uri(serverUrl)); @@ -300,13 +326,27 @@ public class KeepRemoteConnection : IDisposable } _state = ConnectionState.WelcomeReceived; + + // Prepare basic metadata and allow additional metadata to be added + var metadata = await _onConnect(new Dictionary() { + { "client-version", UpdaterManager.SelfVersion?.Version ?? "0.0.0.0" }, + { "client-id", ClientId }, + { "client-uptime", (DateTime.Now - Process.GetCurrentProcess().StartTime).ToString() }, + { "machine-name", DataFolderManager.MachineName }, + { "machine-id", DataFolderManager.MachineID }, + { "install-id", DataFolderManager.InstallID }, + { "machine-os", UpdaterManager.OperatingSystemName }, + { "package-id", UpdaterManager.PackageTypeId }, + }); + SendEnvelope( welcomeEnvelope.RespondWith( new AuthMessage( _token, ClientKey.ExportRSAPublicKeyPem(), - AutoUpdater.UpdaterManager.SelfVersion?.Version ?? "0.0.0", - PROTOCOL_VERSION + UpdaterManager.SelfVersion?.Version ?? "0.0.0.0", + PROTOCOL_VERSION, + metadata ), "auth" ), @@ -352,6 +392,13 @@ public class KeepRemoteConnection : IDisposable )); break; + case MessageType.Control: + await _onControl(new ControlMessage( + envelope.GetPayload(), + response => SendEnvelope(envelope.RespondWith(response)) + )); + break; + default: throw new ProtocolViolationException("Unexpected message"); } @@ -403,13 +450,24 @@ public class KeepRemoteConnection : IDisposable /// The certificate url to use /// The server keys to use /// The token to cancel the connection + /// The callback to call when connecting /// The callback to call when rekeying - /// The callback to call when a message is received + /// The callback to call when a control message is received + /// The callback to call when a command message is received /// - public static Task Start(string serverUrl, string JWT, string certificateUrl, IEnumerable serverKeys, CancellationToken cancellationToken, Func onReKey, Func onMessage) + public static Task Start( + string serverUrl, + string JWT, + string certificateUrl, + IEnumerable serverKeys, + CancellationToken cancellationToken, + Func, Task>> onConnect, + Func onReKey, + Func onControl, + Func onMessage) => Task.Run(async () => { - using var connection = new KeepRemoteConnection(serverUrl, JWT, certificateUrl, serverKeys, cancellationToken, onReKey, onMessage); + using var connection = new KeepRemoteConnection(serverUrl, JWT, certificateUrl, serverKeys, cancellationToken, onConnect, onReKey, onControl, onMessage); await connection._runnerTask; }); @@ -477,14 +535,24 @@ public class KeepRemoteConnection : IDisposable /// /// The url to use /// The JWT token to use - /// + /// The certificate url to use /// The server keys to use + /// The callback to call when connecting /// The callback to call when rekeying + /// The callback to call when a control message is received /// The callback to call when a message is received - /// The token to cancel the connection /// The connection object - public static KeepRemoteConnection CreateRemoteListener(string serverUrl, string JWT, string certificateUrl, IEnumerable serverKeys, CancellationToken cancellationToken, Func onReKey, Func onMessage) - => new KeepRemoteConnection(serverUrl, JWT, certificateUrl, serverKeys, cancellationToken, onReKey, onMessage); + public static KeepRemoteConnection CreateRemoteListener( + string serverUrl, + string JWT, + string certificateUrl, + IEnumerable serverKeys, + CancellationToken cancellationToken, + Func, Task>> onConnect, + Func onReKey, + Func onControl, + Func onMessage) + => new KeepRemoteConnection(serverUrl, JWT, certificateUrl, serverKeys, cancellationToken, onConnect, onReKey, onControl, onMessage); /// /// Requests a certificate refresh @@ -587,4 +655,38 @@ public class KeepRemoteConnection : IDisposable } } } + + /// + /// A wrapper for allowing external code to handle a control message + /// + public sealed class ControlMessage + { + /// + /// The callback method that will receive the response + /// + private readonly Func _respondCommand; + /// + /// The command request message + /// + public ControlRequestMessage ControlRequestMessage { get; } + + /// + /// Creates a new command message + /// + /// The command request message + /// The callback method that will receive the response + public ControlMessage(ControlRequestMessage controlRequestMessage, Func respondCommand) + { + ControlRequestMessage = controlRequestMessage; + _respondCommand = respondCommand; + } + + /// + /// Responds to the command message + /// + /// The response to send + /// True if the response was sent + public bool Respond(ControlResponseMessage response) + => _respondCommand(response); + } } diff --git a/Duplicati/Library/RestAPI/Database/Connection.cs b/Duplicati/Library/RestAPI/Database/Connection.cs index f9aadf1cd..ea15e5188 100644 --- a/Duplicati/Library/RestAPI/Database/Connection.cs +++ b/Duplicati/Library/RestAPI/Database/Connection.cs @@ -38,8 +38,8 @@ namespace Duplicati.Server.Database { public class Connection : IDisposable { - private readonly System.Data.IDbConnection m_connection; - private readonly System.Data.IDbCommand m_errorcmd; + private readonly IDbConnection m_connection; + private readonly IDbCommand m_errorcmd; public readonly object m_lock = new object(); public const int ANY_BACKUP_ID = -1; public const int SERVER_SETTINGS_ID = -2; @@ -65,7 +65,7 @@ namespace Duplicati.Server.Database ]) .ToHashSet(StringComparer.OrdinalIgnoreCase); - public Connection(System.Data.IDbConnection connection, bool disableFieldEncryption, EncryptedFieldHelper.KeyInstance? key) + public Connection(IDbConnection connection, bool disableFieldEncryption, EncryptedFieldHelper.KeyInstance? key) { m_encryptSensitiveFields = !disableFieldEncryption; m_key = key; @@ -141,7 +141,7 @@ namespace Duplicati.Server.Database } } - public void ExecuteWithCommand(Action f) + public void ExecuteWithCommand(Action f) { lock (m_lock) using (var cmd = m_connection.CreateCommand()) @@ -214,7 +214,7 @@ namespace Duplicati.Server.Database .ToDictionary((k) => k.Key, (k) => k.Value); } - internal void SetMetadata(IDictionary values, long id, System.Data.IDbTransaction? transaction) + internal void SetMetadata(IDictionary values, long id, IDbTransaction? transaction) { lock (m_lock) using (var tr = transaction == null ? m_connection.BeginTransaction() : null) @@ -246,7 +246,7 @@ namespace Duplicati.Server.Database .ToArray(); } - internal void SetFilters(IEnumerable values, long id, System.Data.IDbTransaction? transaction = null) + internal void SetFilters(IEnumerable values, long id, IDbTransaction? transaction = null) { lock (m_lock) using (var tr = transaction == null ? m_connection.BeginTransaction() : null) @@ -279,7 +279,7 @@ namespace Duplicati.Server.Database .ToArray(); } - internal void SetSettings(IEnumerable values, long id, System.Data.IDbTransaction? transaction = null) + internal void SetSettings(IEnumerable values, long id, IDbTransaction? transaction = null) { lock (m_lock) using (var tr = transaction == null ? m_connection.BeginTransaction() : null) @@ -319,7 +319,7 @@ namespace Duplicati.Server.Database .ToArray(); } - internal void SetSources(IEnumerable values, long id, System.Data.IDbTransaction transaction) + internal void SetSources(IEnumerable values, long id, IDbTransaction transaction) { lock (m_lock) using (var tr = transaction == null ? m_connection.BeginTransaction() : null) @@ -715,7 +715,7 @@ namespace Duplicati.Server.Database } } - private void AddOrUpdateSchedule(ISchedule item, System.Data.IDbTransaction tr) + private void AddOrUpdateSchedule(ISchedule item, IDbTransaction tr) { lock (m_lock) { @@ -981,7 +981,7 @@ namespace Duplicati.Server.Database .ToDictionary(x => x.Key, x => x.Last().Value); } - public void SetUISettings(string scheme, IDictionary values, System.Data.IDbTransaction? transaction = null) + public void SetUISettings(string scheme, IDictionary values, IDbTransaction? transaction = null) { lock (m_lock) using (var tr = transaction == null ? m_connection.BeginTransaction() : null) @@ -1002,7 +1002,7 @@ namespace Duplicati.Server.Database } } - public void UpdateUISettings(string scheme, IDictionary values, System.Data.IDbTransaction? transaction = null) + public void UpdateUISettings(string scheme, IDictionary values, IDbTransaction? transaction = null) { lock (m_lock) using (var tr = transaction == null ? m_connection.BeginTransaction() : null) @@ -1066,24 +1066,24 @@ namespace Duplicati.Server.Database } } - private static DateTime ConvertToDateTime(System.Data.IDataReader rd, int index) + private static DateTime ConvertToDateTime(IDataReader rd, int index) { var unixTime = ConvertToInt64(rd, index); return unixTime == 0 ? new DateTime(0) : Library.Utility.Utility.EPOCH.AddSeconds(unixTime); } - private static bool ConvertToBoolean(System.Data.IDataReader rd, int index) + private static bool ConvertToBoolean(IDataReader rd, int index) { return ConvertToInt64(rd, index) == 1; } - private static string? ConvertToString(System.Data.IDataReader rd, int index) + private static string? ConvertToString(IDataReader rd, int index) { var r = rd.GetValue(index); return r == null || r == DBNull.Value ? null : r.ToString(); } - private static long ConvertToInt64(System.Data.IDataReader rd, int index) + private static long ConvertToInt64(IDataReader rd, int index) { try { @@ -1097,20 +1097,20 @@ namespace Duplicati.Server.Database return -1; } - private static long ExecuteScalarInt64(System.Data.IDbCommand cmd, long defaultValue = -1) + private static long ExecuteScalarInt64(IDbCommand cmd, long defaultValue = -1) { using (var rd = cmd.ExecuteReader()) return rd.Read() ? ConvertToInt64(rd, 0) : defaultValue; } - private static string? ExecuteScalarString(System.Data.IDbCommand cmd) + private static string? ExecuteScalarString(IDbCommand cmd) { using (var rd = cmd.ExecuteReader()) return rd.Read() ? ConvertToString(rd, 0) : null; } - private object? ConvertToEnum(Type enumType, System.Data.IDataReader rd, int index, object? @default) + private object? ConvertToEnum(Type enumType, IDataReader rd, int index, object? @default) { try { @@ -1124,13 +1124,13 @@ namespace Duplicati.Server.Database } // Overloaded function for legacy functionality - private bool DeleteFromDb(string tablename, long id, System.Data.IDbTransaction? transaction = null) + private bool DeleteFromDb(string tablename, long id, IDbTransaction? transaction = null) { return DeleteFromDb(tablename, id, "ID", transaction); } // New function that allows to delete rows from tables with arbitrary identifier values (e.g. ID or BackupID) - private bool DeleteFromDb(string tablename, long id, string identifier, System.Data.IDbTransaction? transaction = null) + private bool DeleteFromDb(string tablename, long id, string identifier, IDbTransaction? transaction = null) { if (transaction == null) { @@ -1158,14 +1158,14 @@ namespace Duplicati.Server.Database } } - private static IEnumerable Read(System.Data.IDbCommand cmd, Func f) + private static IEnumerable Read(IDbCommand cmd, Func f) { using (var rd = cmd.ExecuteReader()) while (rd.Read()) yield return f(rd); } - private static IEnumerable Read(System.Data.IDataReader rd, Func f) + private static IEnumerable Read(IDataReader rd, Func f) { while (rd.Read()) yield return f(); @@ -1226,7 +1226,7 @@ namespace Duplicati.Server.Database }, sql, args); } - private void OverwriteAndUpdateDb(System.Data.IDbTransaction? transaction, string? deleteSql, object[]? deleteArgs, IEnumerable values, bool updateExisting) + private void OverwriteAndUpdateDb(IDbTransaction? transaction, string? deleteSql, object[]? deleteArgs, IEnumerable values, bool updateExisting) { var properties = GetORMFields(); var idfield = properties.FirstOrDefault(x => x.Name == "ID") @@ -1286,7 +1286,7 @@ namespace Duplicati.Server.Database } } - private IEnumerable ReadFromDb(Func f, string sql, params object?[]? args) + private IEnumerable ReadFromDb(Func f, string sql, params object?[]? args) { using (var cmd = m_connection.CreateCommand()) { @@ -1297,7 +1297,7 @@ namespace Duplicati.Server.Database } } - private void OverwriteAndUpdateDb(System.Data.IDbTransaction? transaction, string? deleteSql, object[]? deleteArgs, IEnumerable values, string insertSql, Func f) + private void OverwriteAndUpdateDb(IDbTransaction? transaction, string? deleteSql, object[]? deleteArgs, IEnumerable values, string insertSql, Func f) { using (var cmd = m_connection.CreateCommand()) { diff --git a/Duplicati/Library/RestAPI/Database/ServerSettings.cs b/Duplicati/Library/RestAPI/Database/ServerSettings.cs index 770c5dd39..d90d463b5 100644 --- a/Duplicati/Library/RestAPI/Database/ServerSettings.cs +++ b/Duplicati/Library/RestAPI/Database/ServerSettings.cs @@ -71,6 +71,7 @@ namespace Duplicati.Server.Database public const string TIMEZONE_OPTION = "server-timezone"; public const string PAUSED_UNTIL = "paused-until"; public const string LAST_UPDATE_CHECK_VERSION = "last-update-check-version"; + public const string ADDITIONAL_REPORT_URL = "additional-report-url"; } private readonly Dictionary settings; @@ -847,6 +848,20 @@ namespace Duplicati.Server.Database SaveSettings(); } } + + public string? AdditionalReportUrl + { + get + { + return settings[CONST.ADDITIONAL_REPORT_URL]; + } + set + { + lock (databaseConnection.m_lock) + settings[CONST.ADDITIONAL_REPORT_URL] = value; + SaveSettings(); + } + } } } diff --git a/Duplicati/Library/RestAPI/Runner.cs b/Duplicati/Library/RestAPI/Runner.cs index 2014358c8..663cba69f 100644 --- a/Duplicati/Library/RestAPI/Runner.cs +++ b/Duplicati/Library/RestAPI/Runner.cs @@ -919,6 +919,18 @@ namespace Duplicati.Server // The server hangs if the module is enabled as there is no console attached DisableModule("console-password-input", options); + // Patch in additional report urls + var additionalReportUrl = FIXMEGlobal.DataConnection.ApplicationSettings.AdditionalReportUrl; + if (!string.IsNullOrWhiteSpace(additionalReportUrl)) + { + options["send-http-json-urls"] = string.Join(";", + new[] { + options.GetValueOrDefault("send-http-json-urls"), + additionalReportUrl + }.Where(x => !string.IsNullOrWhiteSpace(x)) + ); + } + return options; } diff --git a/Duplicati/Library/RestAPI/Scheduler.cs b/Duplicati/Library/RestAPI/Scheduler.cs index 037af95cf..8940691fd 100644 --- a/Duplicati/Library/RestAPI/Scheduler.cs +++ b/Duplicati/Library/RestAPI/Scheduler.cs @@ -143,7 +143,7 @@ namespace Duplicati.Server get { lock (m_lock) - return m_schedule.ToList(); + return (m_schedule ?? []).ToList(); } } diff --git a/Duplicati/WebserverCore/Endpoints/V1/ServerSetting.cs b/Duplicati/WebserverCore/Endpoints/V1/ServerSetting.cs index 464652b09..b43f1d46c 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/ServerSetting.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/ServerSetting.cs @@ -19,7 +19,6 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using Duplicati.Server; using Duplicati.Server.Database; using Duplicati.WebserverCore.Abstractions; using Duplicati.WebserverCore.Exceptions; diff --git a/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs b/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs index df6c64b0c..7f3e80ad3 100644 --- a/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs +++ b/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs @@ -66,6 +66,7 @@ public static class ServiceCollectionsExtensions .AddTransient() .AddTransient() .AddSingleton() + .AddSingleton() .AddSingleton() .AddSingleton(); diff --git a/Duplicati/WebserverCore/Services/IRemoteControllerHandler.cs b/Duplicati/WebserverCore/Services/IRemoteControllerHandler.cs new file mode 100644 index 000000000..338b098cd --- /dev/null +++ b/Duplicati/WebserverCore/Services/IRemoteControllerHandler.cs @@ -0,0 +1,58 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using Duplicati.Library.RemoteControl; + +namespace Duplicati.WebserverCore.Services; + +/// +/// Interface for handling remote control messages. +/// +public interface IRemoteControllerHandler +{ + /// + /// Prepares metadata for a remote control connection. + /// + /// The initial metadata + /// The metadata to use + Task> OnConnect(Dictionary metadata); + + /// + /// Re-keys the remote control connection. + /// + /// The data to re-key with + /// An awaitable task + Task ReKey(ClaimedClientData data); + + /// + /// Handles a control message. + /// + /// The control message to handle + /// An awaitable task + Task OnControl(KeepRemoteConnection.ControlMessage message); + + /// + /// Handles a command message. + /// + /// The command message to handle + /// An awaitable task + Task OnMessage(KeepRemoteConnection.CommandMessage commandMessage); +} \ No newline at end of file diff --git a/Duplicati/WebserverCore/Services/RemoteControllerHandler.cs b/Duplicati/WebserverCore/Services/RemoteControllerHandler.cs new file mode 100644 index 000000000..6664e2918 --- /dev/null +++ b/Duplicati/WebserverCore/Services/RemoteControllerHandler.cs @@ -0,0 +1,120 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using System.Net.Http.Headers; +using Duplicati.Library.Logging; +using Duplicati.Library.RemoteControl; +using Duplicati.Library.RestAPI; +using Duplicati.Server.Database; +using Duplicati.WebserverCore.Abstractions; +using Newtonsoft.Json; + +namespace Duplicati.WebserverCore.Services; + +/// +/// Controller for handling remote control messages. +/// +/// The connection to the database +/// The HTTP client factory +/// The JWT token provider +public class RemoteControllerHandler(Connection connection, IHttpClientFactory httpClientFactory, IJWTTokenProvider jwtTokenProvider) : IRemoteControllerHandler +{ + /// + /// The log tag for this class. + /// + private static readonly string LOGTAG = Log.LogTagFromType(); + + /// + public Task> OnConnect(Dictionary metadata) + { + metadata["feature:additional-report-url"] = connection.ApplicationSettings.AdditionalReportUrl; + return Task.FromResult(metadata); + } + + /// + public Task ReKey(ClaimedClientData data) + { + var oldCerts = connection.ApplicationSettings.RemoteControlConfig == null + ? null + : JsonConvert.DeserializeObject(connection.ApplicationSettings.RemoteControlConfig)?.ServerCertificates; + + connection.ApplicationSettings.RemoteControlConfig = JsonConvert.SerializeObject(new RemoteControlConfig + { + Token = data.JWT, + ServerCertificates = data.ServerCertificates ?? oldCerts ?? [], + ServerUrl = data.ServerUrl, + CertificateUrl = data.CertificateUrl + }); + + if (!FIXMEGlobal.SettingsEncryptionKeyProvidedExternally) + { + // TODO: Implement changing the encryption key + // if (!string.IsNullOrWhiteSpace(data.LocalEncryptionKey) && data.LocalEncryptionKey != connection.ApplicationSettings.SettingsEncryptionKey) + // connection.ChangeDbKey(keydata.LocalEncryptionKey); + } + + return Task.CompletedTask; + } + + /// + public async Task OnControl(KeepRemoteConnection.ControlMessage message) + { + // Make method async + await Task.CompletedTask; + + Dictionary? result = null; + + Log.WriteMessage(LogMessageType.Verbose, LOGTAG, "OnControl", "Received control message: {0}", message); + try + { + if (string.Equals(message.ControlRequestMessage.Command, ControlRequestMessage.ConfigureReportUrlSet, StringComparison.OrdinalIgnoreCase)) + FIXMEGlobal.DataConnection.ApplicationSettings.AdditionalReportUrl = message.ControlRequestMessage.Parameters.GetValueOrDefault(ControlRequestMessage.ConfigureReportUrlParameter); + + if (string.Equals(message.ControlRequestMessage.Command, ControlRequestMessage.ConfigureReportUrlGet, StringComparison.OrdinalIgnoreCase)) + result = new Dictionary { { ControlRequestMessage.ConfigureReportUrlParameter, FIXMEGlobal.DataConnection.ApplicationSettings.AdditionalReportUrl } }; + } + catch (Exception ex) + { + Log.WriteMessage(LogMessageType.Error, LOGTAG, "OnControl", ex, "Failed to handle control message: {0}", message); + message.Respond(new ControlResponseMessage(false, null, ex.Message)); + return; + } + + message.Respond(new ControlResponseMessage(true, result, null)); + } + + + /// + public async Task OnMessage(KeepRemoteConnection.CommandMessage commandMessage) + { + using var httpClient = httpClientFactory.CreateClient(); + var token = jwtTokenProvider.CreateAccessToken("remote-control", jwtTokenProvider.TemporaryFamilyId, TimeSpan.FromMinutes(2)); + + httpClient.BaseAddress = new Uri($"{(connection.ApplicationSettings.UseHTTPS ? "https" : "http")}://127.0.0.1:{connection.ApplicationSettings.LastWebserverPort}"); + httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token); + + var psk = Middlewares.PreSharedKeyFilter.PreSharedKey; + if (!string.IsNullOrWhiteSpace(psk)) + httpClient.DefaultRequestHeaders.Add(Middlewares.PreSharedKeyFilter.HeaderName, psk); + + await commandMessage.Handle(httpClient); + } +} diff --git a/Duplicati/WebserverCore/Services/RemoteControllerService.cs b/Duplicati/WebserverCore/Services/RemoteControllerService.cs index decefb361..9286f87ff 100644 --- a/Duplicati/WebserverCore/Services/RemoteControllerService.cs +++ b/Duplicati/WebserverCore/Services/RemoteControllerService.cs @@ -19,9 +19,7 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System.Net.Http.Headers; using Duplicati.Library.RemoteControl; -using Duplicati.Library.RestAPI; using Duplicati.Server.Database; using Duplicati.WebserverCore.Abstractions; using Newtonsoft.Json; @@ -32,9 +30,8 @@ namespace Duplicati.WebserverCore.Services; /// Controller for toggling remote control. /// /// The connection to the database -/// The HTTP client factory -/// The JWT token provider -public class RemoteControllerService(Connection connection, IHttpClientFactory httpClientFactory, IJWTTokenProvider jwtTokenProvider) : IRemoteController +/// The remote controller handler +public class RemoteControllerService(Connection connection, IRemoteControllerHandler controllerHandler) : IRemoteController { /// /// Gets a value indicating whether remote control is enabled. @@ -83,58 +80,15 @@ public class RemoteControllerService(Connection connection, IHttpClientFactory h config.CertificateUrl, config.ServerCertificates, CancellationToken.None, - ReKey, - OnMessage + controllerHandler.OnConnect, + controllerHandler.ReKey, + controllerHandler.OnControl, + controllerHandler.OnMessage ); connection.ApplicationSettings.RemoteControlEnabled = true; } - /// - /// Re-keys the remote control connection. - /// - /// The data to re-key with - /// An awaitable task - private Task ReKey(ClaimedClientData data) - { - var oldCerts = connection.ApplicationSettings.RemoteControlConfig == null - ? null - : JsonConvert.DeserializeObject(connection.ApplicationSettings.RemoteControlConfig)?.ServerCertificates; - - connection.ApplicationSettings.RemoteControlConfig = JsonConvert.SerializeObject(new RemoteControlConfig - { - Token = data.JWT, - ServerCertificates = data.ServerCertificates ?? oldCerts ?? [], - ServerUrl = data.ServerUrl, - CertificateUrl = data.CertificateUrl - }); - - if (!FIXMEGlobal.SettingsEncryptionKeyProvidedExternally) - { - // TODO: Implement changing the encryption key - // if (!string.IsNullOrWhiteSpace(data.LocalEncryptionKey) && data.LocalEncryptionKey != connection.ApplicationSettings.SettingsEncryptionKey) - // connection.ChangeDbKey(keydata.LocalEncryptionKey); - } - - return Task.CompletedTask; - } - - /// - /// Handles a command message. - /// - /// The command message to handle - /// An awaitable task - private async Task OnMessage(KeepRemoteConnection.CommandMessage commandMessage) - { - using var httpClient = httpClientFactory.CreateClient(); - var token = jwtTokenProvider.CreateAccessToken("remote-control", jwtTokenProvider.TemporaryFamilyId, TimeSpan.FromMinutes(2)); - - httpClient.BaseAddress = new Uri($"{(connection.ApplicationSettings.UseHTTPS ? "https" : "http")}://127.0.0.1:{connection.ApplicationSettings.LastWebserverPort}"); - httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token); - - await commandMessage.Handle(httpClient); - } - /// public void Disable() { From 2118bd59171fcc0523c6ca4a8f1551ed8abb18fd Mon Sep 17 00:00:00 2001 From: "Marcelo C." Date: Tue, 4 Mar 2025 09:23:53 -0300 Subject: [PATCH 285/362] Adding ParseIntOption --- Duplicati/Library/Utility/Utility.cs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index 038d3adad..5843328db 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -774,6 +774,18 @@ namespace Duplicati.Library.Utility return (T)(object)flags; } + + /// + /// Parses an option with int value, returning the default value if the option is not found or cannot be parsed + /// + /// The set of options to look for the setting in + /// The value to look for in the settings + /// default value + /// + public static int ParseIntOption(IReadOnlyDictionary options, string value, int @default) + { + return options.TryGetValue(value, out var opt) && int.TryParse(opt ?? string.Empty, out var result) ? result : @default; + } /// /// Converts a sequence of bytes to a hex string From 1f7415cc8e8f4e96d2db04f6f5ca94de0c290079 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 4 Mar 2025 15:39:56 +0100 Subject: [PATCH 286/362] Simplified backend manager handler code --- .../Main/Backend/BackendManager.Handler.cs | 112 +++++++++--------- 1 file changed, 57 insertions(+), 55 deletions(-) diff --git a/Duplicati/Library/Main/Backend/BackendManager.Handler.cs b/Duplicati/Library/Main/Backend/BackendManager.Handler.cs index 62c5fd4eb..46724356d 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.Handler.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.Handler.cs @@ -191,65 +191,59 @@ partial class BackendManager } /// - /// Reclaims completed downloads + /// Reclaims completed tasks /// + /// The list of tasks to reclaim /// An awaitable task - private async Task ReclaimCompletedDownloads() + private static async Task ReclaimCompletedTasks(List tasks) { - for (int i = activeDownloads.Count - 1; i >= 0; i--) + for (int i = tasks.Count - 1; i >= 0; i--) { - if (activeDownloads[i].IsCompleted) + if (tasks[i].IsCompleted) { + var t = tasks[i]; + tasks.RemoveAt(i); // Make sure the task is awaited so we capture any exceptions - await activeDownloads[i].ConfigureAwait(false); - activeDownloads.RemoveAt(i); + await t.ConfigureAwait(false); } } } /// - /// Reclaims completed uploads + /// Reclaims completed tasks from uploads and downloads /// /// An awaitable task - private async Task ReclaimCompletedUploads() + private async Task ReclaimCompletedTasks() { - for (int i = activeUploads.Count - 1; i >= 0; i--) + await ReclaimCompletedTasks(activeUploads); + await ReclaimCompletedTasks(activeDownloads); + } + + /// + /// Ensures that there are at most N - 1 active tasks + /// + /// The maximum number of active tasks + /// The list of active tasks + /// An awaitable task + private static async Task EnsureAtMostNActiveTasks(int n, List tasks) + { + while (tasks.Count >= n) { - if (activeUploads[i].IsCompleted) - { - // Make sure the task is awaited so we capture any exceptions - await activeUploads[i].ConfigureAwait(false); - activeUploads.RemoveAt(i); - } + await Task.WhenAny(tasks).ConfigureAwait(false); + await ReclaimCompletedTasks(tasks).ConfigureAwait(false); } } /// - /// Ensures that there are at most N - 1 active downloads + /// Ensures that there are at most N - 1 active tasks /// - /// - /// - private async Task EnsureAtMostNActiveDownloads(int n) + /// The number of active uploads + /// The number of active downloads + /// An awaitable task + private async Task EnsureAtMostNActiveTasks(int uploads, int downloads) { - while (activeDownloads.Count >= n) - { - await Task.WhenAny(activeDownloads).ConfigureAwait(false); - await ReclaimCompletedDownloads().ConfigureAwait(false); - } - } - - /// - /// Ensures that there are at most N - 1 active uploads - /// - /// The maximum number of active uploads - /// An awaitable task - private async Task EnsureAtMostNActiveUploads(int n) - { - while (activeUploads.Count >= n) - { - await Task.WhenAny(activeUploads).ConfigureAwait(false); - await ReclaimCompletedUploads().ConfigureAwait(false); - } + await EnsureAtMostNActiveTasks(uploads, activeUploads).ConfigureAwait(false); + await EnsureAtMostNActiveTasks(downloads, activeDownloads).ConfigureAwait(false); } /// @@ -270,14 +264,13 @@ partial class BackendManager try { // Clean up completed uploads, if any - await ReclaimCompletedUploads().ConfigureAwait(false); + await ReclaimCompletedTasks().ConfigureAwait(false); // Allow PUT operations to be queued, if requested if (op is PutOperation putOp && !putOp.WaitForComplete) { // Wait for any active downloads to complete before starting an upload - await EnsureAtMostNActiveDownloads(1).ConfigureAwait(false); - await EnsureAtMostNActiveUploads(maxParallelUploads).ConfigureAwait(false); + await EnsureAtMostNActiveTasks(maxParallelUploads, 1).ConfigureAwait(false); // Operation is accepted into queue, so we can signal completion putOp.SetComplete(true); @@ -286,8 +279,7 @@ partial class BackendManager else if (op is GetOperation getOp) { // Wait for any active uploads to complete before starting a download - await EnsureAtMostNActiveUploads(1).ConfigureAwait(false); - await EnsureAtMostNActiveDownloads(maxParallelDownloads).ConfigureAwait(false); + await EnsureAtMostNActiveTasks(1, maxParallelDownloads).ConfigureAwait(false); // Operation is accepted into queue, so we can signal completion activeDownloads.Add(ExecuteWithRetry(getOp, tcs.Token)); @@ -295,8 +287,7 @@ partial class BackendManager else { // Wait for all of the active uploads and downloads to complete - await EnsureAtMostNActiveUploads(1).ConfigureAwait(false); - await EnsureAtMostNActiveDownloads(1).ConfigureAwait(false); + await EnsureAtMostNActiveTasks(1, 1).ConfigureAwait(false); // Execute the operation await ExecuteWithRetry(op, tcs.Token).ConfigureAwait(false); @@ -325,19 +316,30 @@ partial class BackendManager for (int i = activeUploads.Count - 1; i >= 0; i--) { var t = activeUploads[i]; - activeUploads.RemoveAt(i); - if (t.IsCompleted && !t.IsCanceled && t.Exception != null) - Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerDisposeError", t.Exception, "Error in active upload: {0}", t.Exception.Message); + if (t.IsCompleted) + { + activeUploads.RemoveAt(i); + if (t.IsCanceled) + Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerDisposeError", t.Exception, "Error in active upload: Cancelled"); + else if (t.IsFaulted) + Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerDisposeError", t.Exception, "Error in active upload: {0}", t.Exception?.Message ?? "null"); + else + Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerDisposeError", null, "Upload was active during termination, but completed successfully"); + } + else + { + Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerDisposeError", null, "Upload was active during termination, but had state: {0}", t.Status); + } + + if (activeUploads.Count > 0) + Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerDisposeError", null, "Terminating, but {0} active uploads are still active", activeUploads.Count); } - if (activeUploads.Count > 0) - Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerDisposeError", null, "Terminating, but {0} active uploads are still active", activeUploads.Count); + // Dispose of any remaining backends + while (backendPool.TryDequeue(out var backend)) + try { backend.Dispose(); } + catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerDisposeError", ex, "Failed to dispose backend instance: {0}", ex.Message); } } - - // Dispose of any remaining backends - while (backendPool.TryDequeue(out var backend)) - try { backend.Dispose(); } - catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerDisposeError", ex, "Failed to dispose backend instance: {0}", ex.Message); } } } From 78b45eb98373f4be880cdc96722b5dde2d68ed49 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 4 Mar 2025 11:48:40 +0100 Subject: [PATCH 287/362] Added additional tests for `OperationCancelledException` (cherry picked from commit 30568f0498e9469c27adab4b25a8d66cae69c3c0) --- Duplicati/UnitTest/DisruptionTests.cs | 314 ++++++++++++++++++++++++++ 1 file changed, 314 insertions(+) diff --git a/Duplicati/UnitTest/DisruptionTests.cs b/Duplicati/UnitTest/DisruptionTests.cs index 16fac2c5d..5345f677d 100644 --- a/Duplicati/UnitTest/DisruptionTests.cs +++ b/Duplicati/UnitTest/DisruptionTests.cs @@ -25,6 +25,7 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Interface; +using Duplicati.Library.Logging; using Duplicati.Library.Main; using Duplicati.Library.Main.Database; using Duplicati.Library.Main.Volumes; @@ -852,5 +853,318 @@ namespace Duplicati.UnitTest Assert.AreEqual(3, listResults.Filesets.Count()); } } + + private class LogSink : IMessageSink + { + public List StartedFiles { get; } = new List(); + public List CompletedFiles { get; } = new List(); + public void BackendEvent(BackendActionType action, BackendEventType type, string path, long size) + { + if (action == BackendActionType.Put && type == BackendEventType.Started) + { + this.StartedFiles.Add(path); + } + if (action == BackendActionType.Put && type == BackendEventType.Completed) + { + this.CompletedFiles.Add(path); + } + } + + public void SetBackendProgress(IBackendProgress progress) + { + } + + public void SetOperationProgress(IOperationProgress progress) + { + } + + public void WriteMessage(LogEntry entry) + { + } + } + + [Test] + [Category("Disruption")] + public void TestFailedDblockUploadWithOperationCancellation() + { + var testopts = TestOptions; + testopts["number-of-retries"] = "1"; + testopts["dblock-size"] = "10mb"; + + // Make a base backup + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER }); + Assert.AreEqual(0, backupResults.Errors.Count()); + Assert.AreEqual(0, backupResults.Warnings.Count()); + } + + // Make a new backup that fails uploading a dblock file + ModifySourceFiles(); + + // Deterministic error backend + Library.DynamicLoader.BackendLoader.AddBackend(new DeterministicErrorBackend()); + var failtarget = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER; + var uploadCount = 0; + var failedFile = string.Empty; + + // Fail the dlist upload + DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) => + { + if (action == DeterministicErrorBackend.BackendAction.PutBefore && remotename.Contains(".dblock.")) + { + // Fail once with a cancellation + if (Interlocked.Increment(ref uploadCount) == 1) + { + failedFile = remotename; + throw new OperationCanceledException(); + } + } + + return false; + }; + + using (var c = new Library.Main.Controller(failtarget, testopts, null)) + { + var sink = new LogSink(); + c.AppendSink(sink); + var res = c.Backup(new string[] { DATAFOLDER }); + if (uploadCount == 0) + Assert.Fail("Upload count was not incremented"); + + Assert.AreEqual(2, c.List().Filesets.Count()); + Assert.AreEqual(7 * 2 + 1, sink.CompletedFiles.Count); // 7 completed (dblock + dindex) + 1 dlist + Assert.AreEqual(7 * 2 + 2, sink.StartedFiles.Count); // 1 retry + Assert.AreEqual(1, res.BackendStatistics.RetryAttempts); + } + + ModifySourceFiles(); + // Create a regular backup + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER }); + Assert.AreEqual(0, backupResults.Errors.Count()); + Assert.AreEqual(0, backupResults.Warnings.Count()); + } + + // Verify that all is in order + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null)) + { + var r = c.Test(long.MaxValue); + Assert.AreEqual(0, r.Errors.Count()); + Assert.AreEqual(0, r.Warnings.Count()); + Assert.IsFalse(r.Verifications.Any(p => p.Value.Any())); + } + + // Test that we can recreate + var recreatedDatabaseFile = Path.Combine(BASEFOLDER, "recreated-database.sqlite"); + if (File.Exists(recreatedDatabaseFile)) + File.Delete(recreatedDatabaseFile); + + testopts["dbpath"] = recreatedDatabaseFile; + + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IRepairResults repairResults = c.Repair(); + Assert.AreEqual(0, repairResults.Errors.Count()); + Assert.AreEqual(0, repairResults.Warnings.Count()); + } + + // Check that we have 3 versions + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IListResults listResults = c.List(); + Assert.AreEqual(0, listResults.Errors.Count()); + Assert.AreEqual(0, listResults.Warnings.Count()); + Assert.AreEqual(3, listResults.Filesets.Count()); + } + } + + [Test] + [Category("Disruption")] + public void TestFailedDindexUploadWithOperationCancellation() + { + var testopts = TestOptions; + testopts["number-of-retries"] = "1"; + testopts["dblock-size"] = "10mb"; + + // Make a base backup + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER }); + Assert.AreEqual(0, backupResults.Errors.Count()); + Assert.AreEqual(0, backupResults.Warnings.Count()); + } + + // Make a new backup that fails uploading a dblock file + ModifySourceFiles(); + + // Deterministic error backend + Library.DynamicLoader.BackendLoader.AddBackend(new DeterministicErrorBackend()); + var failtarget = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER; + var uploadCount = 0; + var failedFile = string.Empty; + + // Fail the dlist upload + DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) => + { + if (action == DeterministicErrorBackend.BackendAction.PutBefore && remotename.Contains(".dindex.")) + { + // Fail once with a cancellation + if (Interlocked.Increment(ref uploadCount) == 1) + { + failedFile = remotename; + throw new OperationCanceledException(); + } + } + + return false; + }; + + using (var c = new Library.Main.Controller(failtarget, testopts, null)) + { + var sink = new LogSink(); + c.AppendSink(sink); + var res = c.Backup(new string[] { DATAFOLDER }); + if (uploadCount == 0) + Assert.Fail("Upload count was not incremented"); + + Assert.AreEqual(2, c.List().Filesets.Count()); + Assert.AreEqual(7 * 2 + 1, sink.CompletedFiles.Count); // 7 completed (dblock + dindex) + 1 dlist + Assert.AreEqual(7 * 2 + 2, sink.StartedFiles.Count); // 1 retry + Assert.AreEqual(sink.CompletedFiles.Count(x => x.Contains(".dblock.")), sink.CompletedFiles.Count(x => x.Contains(".dindex."))); + Assert.AreEqual(1, res.BackendStatistics.RetryAttempts); + } + + ModifySourceFiles(); + // Create a regular backup + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER }); + Assert.AreEqual(0, backupResults.Errors.Count()); + Assert.AreEqual(0, backupResults.Warnings.Count()); + } + + // Verify that all is in order + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null)) + { + var r = c.Test(long.MaxValue); + Assert.AreEqual(0, r.Errors.Count()); + Assert.AreEqual(0, r.Warnings.Count()); + Assert.IsFalse(r.Verifications.Any(p => p.Value.Any())); + } + + // Test that we can recreate + var recreatedDatabaseFile = Path.Combine(BASEFOLDER, "recreated-database.sqlite"); + if (File.Exists(recreatedDatabaseFile)) + File.Delete(recreatedDatabaseFile); + + testopts["dbpath"] = recreatedDatabaseFile; + + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IRepairResults repairResults = c.Repair(); + Assert.AreEqual(0, repairResults.Errors.Count()); + Assert.AreEqual(0, repairResults.Warnings.Count()); + } + + // Check that we have 3 versions + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IListResults listResults = c.List(); + Assert.AreEqual(0, listResults.Errors.Count()); + Assert.AreEqual(0, listResults.Warnings.Count()); + Assert.AreEqual(3, listResults.Filesets.Count()); + } + } + + [Test] + [Category("Disruption")] + public void TestFailedDlistUploadWithOperationCancellation() + { + var testopts = TestOptions; + testopts["number-of-retries"] = "1"; + testopts["dblock-size"] = "10mb"; + + // Make a base backup + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER }); + Assert.AreEqual(0, backupResults.Errors.Count()); + Assert.AreEqual(0, backupResults.Warnings.Count()); + } + + // Make a new backup that fails uploading a dblock file + ModifySourceFiles(); + + // Deterministic error backend + Library.DynamicLoader.BackendLoader.AddBackend(new DeterministicErrorBackend()); + var failtarget = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER; + var uploadCount = 0; + + // Fail the dlist upload + DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) => + { + if (action == DeterministicErrorBackend.BackendAction.PutBefore && remotename.Contains(".dlist.")) + { + if (Interlocked.Increment(ref uploadCount) == 1) + throw new OperationCanceledException(); + } + + return false; + }; + + using (var c = new Library.Main.Controller(failtarget, testopts, null)) + { + var res = c.Backup(new string[] { DATAFOLDER }); + var dlistCount = Directory.GetFiles(TARGETFOLDER, "*.dlist.*", SearchOption.TopDirectoryOnly).Count(); + if (uploadCount == 0) + Assert.Fail("Upload count was not incremented"); + Assert.AreEqual(2, c.List().Filesets.Count()); + Assert.AreEqual(2, dlistCount); + Assert.AreEqual(1, res.BackendStatistics.RetryAttempts); + } + + ModifySourceFiles(); + // Create a regular backup + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER }); + Assert.AreEqual(0, backupResults.Errors.Count()); + Assert.AreEqual(0, backupResults.Warnings.Count()); + } + + // Verify that all is in order + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null)) + { + var r = c.Test(long.MaxValue); + Assert.AreEqual(0, r.Errors.Count()); + Assert.AreEqual(0, r.Warnings.Count()); + Assert.IsFalse(r.Verifications.Any(p => p.Value.Any())); + } + + // Test that we can recreate + var recreatedDatabaseFile = Path.Combine(BASEFOLDER, "recreated-database.sqlite"); + if (File.Exists(recreatedDatabaseFile)) + File.Delete(recreatedDatabaseFile); + + testopts["dbpath"] = recreatedDatabaseFile; + + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IRepairResults repairResults = c.Repair(); + Assert.AreEqual(0, repairResults.Errors.Count()); + Assert.AreEqual(0, repairResults.Warnings.Count()); + } + + // Check that we have 3 versions + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IListResults listResults = c.List(); + Assert.AreEqual(0, listResults.Errors.Count()); + Assert.AreEqual(0, listResults.Warnings.Count()); + Assert.AreEqual(3, listResults.Filesets.Count()); + } + } } } \ No newline at end of file From 3596e0ebd3c290e05199da7231935c630c74d950 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 4 Mar 2025 19:54:40 +0100 Subject: [PATCH 288/362] When profiling a get, it's worthwhile knowing where the remote file is being stored locally --- Duplicati/Library/Main/Backend/BackendManager.GetOperation.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Backend/BackendManager.GetOperation.cs b/Duplicati/Library/Main/Backend/BackendManager.GetOperation.cs index 1d64b2ccd..d159ef046 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.GetOperation.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.GetOperation.cs @@ -77,7 +77,8 @@ partial class BackendManager (tmpfile, var dataSizeDownloaded, var fileHash) = await DoGetFileAsync(backend, cancelToken).ConfigureAwait(false); var duration = DateTime.Now - begin; - Logging.Log.WriteProfilingMessage(LOGTAG, "DownloadSpeed", "Downloaded {0} in {1}, {2}/s", Library.Utility.Utility.FormatSizeString(dataSizeDownloaded), + Logging.Log.WriteProfilingMessage(LOGTAG, "DownloadSpeed", "Downloaded {0} to {1} ({2}) in {3}, {4}/s", + RemoteFilename, tmpfile.Name, Library.Utility.Utility.FormatSizeString(dataSizeDownloaded), duration, Library.Utility.Utility.FormatSizeString((long)(dataSizeDownloaded / duration.TotalSeconds))); Context.Database.LogRemoteOperation("get", RemoteFilename, System.Text.Json.JsonSerializer.Serialize(new { Size = dataSizeDownloaded, Hash = fileHash })); From 879af349704a9a0da58fc64850a101c00598aab5 Mon Sep 17 00:00:00 2001 From: "Marcelo C." Date: Tue, 4 Mar 2025 19:07:28 -0300 Subject: [PATCH 289/362] Adding Box.com to CI --- .../Utility/Duplicati.Library.Utility.csproj | 2 +- .../Duplicati.Backend.Tests/Box/BoxTests.cs | 46 +++++++++++++++++++ LiveTests/Duplicati.Backend.Tests/README.md | 9 +++- 3 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 LiveTests/Duplicati.Backend.Tests/Box/BoxTests.cs diff --git a/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj b/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj index d53703e2f..bf847d415 100644 --- a/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj +++ b/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj @@ -8,7 +8,7 @@ - + diff --git a/LiveTests/Duplicati.Backend.Tests/Box/BoxTests.cs b/LiveTests/Duplicati.Backend.Tests/Box/BoxTests.cs new file mode 100644 index 000000000..e4bd6e6b0 --- /dev/null +++ b/LiveTests/Duplicati.Backend.Tests/Box/BoxTests.cs @@ -0,0 +1,46 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +namespace Duplicati.Backend.Tests.Box; + +/// +/// Box.com Tests +/// +[TestClass] +public sealed class BoxTests : BaseTest +{ + [TestMethod] + public Task TestBox() + { + CheckRequiredEnvironment(["TESTCREDENTIAL_BOX_FOLDER", "TESTCREDENTIAL_BOX_AUTHID"]); + + var exitCode = CommandLine.BackendTester.Program.Main( + new[] + { + $"box://{Environment.GetEnvironmentVariable("TESTCREDENTIAL_BOX_FOLDER")}/?auth-id={Uri.EscapeDataString(Environment.GetEnvironmentVariable("TESTCREDENTIAL_BOX_AUTHID")!)}", + + }.Concat(Parameters.GlobalTestParameters).ToArray()); + + if (exitCode != 0) Assert.Fail("BackendTester is returning non-zero exit code, check logs for details"); + + return Task.CompletedTask; + } +} \ No newline at end of file diff --git a/LiveTests/Duplicati.Backend.Tests/README.md b/LiveTests/Duplicati.Backend.Tests/README.md index 8427d1ffd..c15ba6fcb 100644 --- a/LiveTests/Duplicati.Backend.Tests/README.md +++ b/LiveTests/Duplicati.Backend.Tests/README.md @@ -27,6 +27,7 @@ NUMBER_OF_FILES default is 20 | AzureBlob | Planned | Live | | CIFS | Planned | Live | | Google Drive | Planned | Live | +| Box | Planned | Live | | TahoeLAFS | Pending Decision | | | TescentCOS | Pending Decision | | | Storj | Pending Decision | | @@ -39,7 +40,6 @@ NUMBER_OF_FILES default is 20 | Jottacloud | Pending Decision | | | IDrive2 | Planned | | | File | Planned | | -| Box | Pending Decision | | | CloudFiles | Pending Decision | | | backBlaze | Planned | Live | | AliyunOSS | Pending Decision | | @@ -56,6 +56,13 @@ Please note that TestContainers token has to be configured in secrets/Github act ## Backends that require Environment variables +## Box.com + +``` +TESTCREDENTIAL_BOX_FOLDER +TESTCREDENTIAL_BOX_AUTHID +``` + ## Backblaze B2 Backblaze B2 credentials are mapped to the following environment variables: From ea2c69560da1628b17ec0a755d57dd6f0527f84c Mon Sep 17 00:00:00 2001 From: "Marcelo C." Date: Tue, 4 Mar 2025 19:12:57 -0300 Subject: [PATCH 290/362] Removing library upgrade from CI PR --- Duplicati/Library/Utility/Duplicati.Library.Utility.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj b/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj index bf847d415..d53703e2f 100644 --- a/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj +++ b/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj @@ -8,7 +8,7 @@ - + From 3da510f5dc88c8fd40d1c18a31b310a1b8d76f55 Mon Sep 17 00:00:00 2001 From: "Marcelo C." Date: Tue, 4 Mar 2025 19:20:32 -0300 Subject: [PATCH 291/362] Adding Box to yml --- .github/workflows/backendtests.yml | 52 ++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/.github/workflows/backendtests.yml b/.github/workflows/backendtests.yml index 9b3148002..bb5b0765f 100644 --- a/.github/workflows/backendtests.yml +++ b/.github/workflows/backendtests.yml @@ -14,6 +14,7 @@ jobs: google_secrets_available: "${{ steps.check_google.outputs.available }}" pcloud_secrets_available: "${{ steps.check_pcloud.outputs.available }}" b2_secrets_available: "${{ steps.check_b2.outputs.available }}" + box_secrets_available: "${{ steps.check_box.outputs.available }}" steps: - id: check_pcloud name: Check pCloud secrets @@ -120,6 +121,20 @@ jobs: echo "available=false" >> $GITHUB_OUTPUT fi echo "B2 check completed" + - id: check_box + name: Check Box.com secrets + shell: bash + run: | + echo "Starting Box.com secrets check..." + if [[ -n "${{ secrets.TESTCREDENTIAL_BOX_FOLDER }}" ]] && \ + [[ -n "${{ secrets.TESTCREDENTIAL_BOX_AUTHID }}" ]]; then + echo "All Box.com secrets found" + echo "available=true" >> $GITHUB_OUTPUT + else + echo "Missing some Box.com secrets" + echo "available=false" >> $GITHUB_OUTPUT + fi + echo "Box.com check completed" - name: Debug Output shell: bash run: | @@ -130,6 +145,7 @@ jobs: echo "pCloud Available: ${{ steps.check_pcloud.outputs.available }}" echo "Azure Available: ${{ steps.check_azure.outputs.available }}" echo "B2 Available: ${{ steps.check_b2.outputs.available }}" + echo "Box.com Available: ${{ steps.check_box.outputs.available }}" test_ftp: needs: check_secrets if: needs.check_secrets.outputs.testcontainers_secrets_available == 'true' @@ -504,4 +520,40 @@ jobs: run: >- dotnet test --no-build --filter="ClassName~B2Tests" --logger:"console;verbosity=detailed" + LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln + test_box: + needs: check_secrets + if: needs.check_secrets.outputs.box_secrets_available == 'true' + name: Box.com Tests + runs-on: "${{ matrix.os }}" + strategy: + max-parallel: 1 + fail-fast: false + matrix: + os: + - ubuntu-latest + - windows-latest + - macos-latest + steps: + - name: Set up .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 8.x + - name: Checkout source + uses: actions/checkout@v4 + - name: Restore NuGet dependencies + run: >- + dotnet restore + LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln + - name: Build project + run: >- + dotnet build --no-restore + LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln + - name: Run Box.com tests + env: + TESTCREDENTIAL_BOX_FOLDER: "${{ secrets.TESTCREDENTIAL_BOX_FOLDER }}" + TESTCREDENTIAL_BOX_AUTHID: "${{ secrets.TESTCREDENTIAL_BOX_AUTHID }}" + run: >- + dotnet test --no-build --filter="ClassName~BoxTests" + --logger:"console;verbosity=detailed" LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln \ No newline at end of file From 4f62a8a413b6e471ff69ac143660675c6de11451 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 25 Feb 2025 15:07:17 +0100 Subject: [PATCH 292/362] Fixed off-by-one for retry count --- Duplicati/Library/Main/Backend/BackendManager.Handler.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Main/Backend/BackendManager.Handler.cs b/Duplicati/Library/Main/Backend/BackendManager.Handler.cs index 62c5fd4eb..c43895250 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.Handler.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.Handler.cs @@ -415,7 +415,7 @@ partial class BackendManager } } - context.Statwriter.SendEvent(op.Operation, retries < maxRetries ? BackendEventType.Retrying : BackendEventType.Failed, op.RemoteFilename, op.Size); + context.Statwriter.SendEvent(op.Operation, retries <= maxRetries ? BackendEventType.Retrying : BackendEventType.Failed, op.RemoteFilename, op.Size); // Check if we can recover from the error var recovered = false; @@ -428,14 +428,14 @@ partial class BackendManager } // We did not recover, so wait or give up - if (!recovered && retries < maxRetries && retryDelay.Ticks != 0) + if (!recovered && retries <= maxRetries && retryDelay.Ticks != 0) { var delay = Library.Utility.Utility.GetRetryDelay(retryDelay, retries, retryWithExponentialBackoff); await Task.Delay(delay, context.TaskReader.ProgressToken).ConfigureAwait(false); } } - } while (retries < maxRetries); + } while (retries <= maxRetries); // If we have a last exception, we failed if (lastException != null) From be0d74fe2e6524915eb135cc2076245752d486b2 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 5 Mar 2025 10:01:08 +0100 Subject: [PATCH 293/362] Attempt to make test more stable --- Duplicati/UnitTest/DisruptionTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/UnitTest/DisruptionTests.cs b/Duplicati/UnitTest/DisruptionTests.cs index 5345f677d..32c053fe6 100644 --- a/Duplicati/UnitTest/DisruptionTests.cs +++ b/Duplicati/UnitTest/DisruptionTests.cs @@ -66,7 +66,7 @@ namespace Duplicati.UnitTest { if (path.EndsWith(this.fileSizes[2] + "MB")) { - Thread.Sleep(500); + Thread.Sleep(2000); controller.Stop(); stopped.TrySetResult(true); } From ddd34ca9c6bd8706234ba300626fba46d25d1b76 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 5 Mar 2025 14:53:51 +0100 Subject: [PATCH 294/362] Second instance of TrayIcon opens logged in view. This fixes #6003 --- Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs | 2 +- Duplicati/GUI/Duplicati.GUI.TrayIcon/TrayIconBase.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs index d9e924630..2529f79f4 100644 --- a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs +++ b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs @@ -224,7 +224,7 @@ No password provided, unable to connect to server, exiting"); if (hosted != null && Server.Program.ApplicationInstance != null) Server.Program.ApplicationInstance.SecondInstanceDetected += new Server.SingleInstance.SecondInstanceDelegate( - x => { tk.ShowUrlInWindow(serverURL.ToString()); }); + x => tk.ShowStatusWindow()); // TODO: If we change to hosted browser this should be a callback if (openui) diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/TrayIconBase.cs b/Duplicati/GUI/Duplicati.GUI.TrayIcon/TrayIconBase.cs index 1b769718f..d6815b21f 100644 --- a/Duplicati/GUI/Duplicati.GUI.TrayIcon/TrayIconBase.cs +++ b/Duplicati/GUI/Duplicati.GUI.TrayIcon/TrayIconBase.cs @@ -180,7 +180,7 @@ namespace Duplicati.GUI.TrayIcon }); } - protected void ShowStatusWindow() + public void ShowStatusWindow() { Program.Connection.GetStatusWindowURLAsync().ContinueWith(t => { From a9c05c1883c420ebe8f61a13f30231449e3e1475 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 5 Mar 2025 16:03:25 +0100 Subject: [PATCH 295/362] Fixed locale aware systeminfo --- .../Localization/ILocalizationService.cs | 46 ++++---- .../Localization/LocalizationService.cs | 41 ++++--- .../Localization/MoLocalizationService.cs | 6 + .../Localization/MockLocalizationService.cs | 46 ++++---- .../Services/SystemInfoProvider.cs | 104 +++++++++++------- 5 files changed, 141 insertions(+), 102 deletions(-) diff --git a/Duplicati/Library/Localization/ILocalizationService.cs b/Duplicati/Library/Localization/ILocalizationService.cs index c0c35f262..f9eeb825f 100644 --- a/Duplicati/Library/Localization/ILocalizationService.cs +++ b/Duplicati/Library/Localization/ILocalizationService.cs @@ -1,31 +1,35 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System; -using System.Collections.Generic; +using System.Globalization; namespace Duplicati.Library.Localization { public interface ILocalizationService { + /// + /// The culture of the localization service + /// + public CultureInfo Culture { get; } + /// /// Localizes the string similar to how string.Format works /// diff --git a/Duplicati/Library/Localization/LocalizationService.cs b/Duplicati/Library/Localization/LocalizationService.cs index b86b67182..adc53e327 100644 --- a/Duplicati/Library/Localization/LocalizationService.cs +++ b/Duplicati/Library/Localization/LocalizationService.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -24,7 +24,6 @@ using System.Linq; using System.Globalization; using System.Collections.Generic; using System.Text.RegularExpressions; -using Duplicati.Library.Localization.Short; using Duplicati.Library.Logging; namespace Duplicati.Library.Localization @@ -147,7 +146,7 @@ namespace Duplicati.Library.Localization /// /// Returns true if the culture has localization support /// - public static Boolean isCultureSupported(CultureInfo culture) + public static bool IsCultureSupported(CultureInfo culture) { return MoLocalizationService.isCultureSupported(culture); } diff --git a/Duplicati/Library/Localization/MoLocalizationService.cs b/Duplicati/Library/Localization/MoLocalizationService.cs index e0a81fbbb..a5983cf77 100644 --- a/Duplicati/Library/Localization/MoLocalizationService.cs +++ b/Duplicati/Library/Localization/MoLocalizationService.cs @@ -75,6 +75,7 @@ namespace Duplicati.Library.Localization /// The culture to find. public MoLocalizationService(CultureInfo ci) { + Culture = ci; var filenames = new string[] { // Load the specialized version first string.Format("localization-{0}.mo", ci.Name.Replace('-', '_')), @@ -117,6 +118,11 @@ namespace Duplicati.Library.Localization } } + /// + /// Gets the culture of the localization + /// + public CultureInfo Culture { get; } + /// /// Pre-processes the message to use Linux line endings /// diff --git a/Duplicati/Library/Localization/MockLocalizationService.cs b/Duplicati/Library/Localization/MockLocalizationService.cs index d0e894cf9..1e2ee86e9 100644 --- a/Duplicati/Library/Localization/MockLocalizationService.cs +++ b/Duplicati/Library/Localization/MockLocalizationService.cs @@ -1,31 +1,35 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System; -using System.Collections.Generic; +using System.Globalization; namespace Duplicati.Library.Localization { public class MockLocalizationService : ILocalizationService { + /// + /// The culture of the localization service + /// + public CultureInfo Culture => CultureInfo.InvariantCulture; + /// /// Localizes the string similar to how string.Format works /// diff --git a/Duplicati/WebserverCore/Services/SystemInfoProvider.cs b/Duplicati/WebserverCore/Services/SystemInfoProvider.cs index 1662b3352..8d9679c64 100644 --- a/Duplicati/WebserverCore/Services/SystemInfoProvider.cs +++ b/Duplicati/WebserverCore/Services/SystemInfoProvider.cs @@ -20,6 +20,7 @@ // DEALINGS IN THE SOFTWARE. using System.Globalization; using Duplicati.Library.AutoUpdater; +using Duplicati.Library.Localization; using Duplicati.Library.RestAPI; using Duplicati.Server; using Duplicati.Server.Serialization.Interface; @@ -205,62 +206,87 @@ public class SystemInfoProvider : ISystemInfoProvider } /// - /// Cached static system information. + /// Creates a new instance of the class. /// - private Lazy _systemInfoBase = new(() => new StaticSystemInformation - { - APIVersion = 1, - PasswordPlaceholder = FIXMEGlobal.PASSWORD_PLACEHOLDER, - ServerVersion = UpdaterManager.SelfVersion.Version, - ServerVersionName = License.VersionNumbers.VERSION_NAME, - ServerVersionType = UpdaterManager.SelfVersion.ReleaseType, - RemoteControlRegistrationUrl = Library.RemoteControl.RegisterForRemote.DefaultRegisterationUrl, - StartedBy = FIXMEGlobal.Origin, - DefaultUpdateChannel = AutoUpdateSettings.DefaultUpdateChannel.ToString(), - DefaultUsageReportLevel = Library.UsageReporter.Reporter.DefaultReportLevel, - OSType = UpdaterManager.OperatingSystemName, - OSVersion = Library.UsageReporter.OSInfoHelper.PlatformString, - DirectorySeparator = Path.DirectorySeparatorChar, - PathSeparator = Path.PathSeparator, - CaseSensitiveFilesystem = Library.Utility.Utility.IsFSCaseSensitive, - MachineName = Environment.MachineName, - PackageTypeId = UpdaterManager.PackageTypeId, - UserName = OperatingSystem.IsWindows() ? System.Security.Principal.WindowsIdentity.GetCurrent().Name : Environment.UserName, - NewLine = Environment.NewLine, - CLRVersion = Environment.Version.ToString(), - Options = Server.Serializable.ServerSettings.Options, - CompressionModules = Server.Serializable.ServerSettings.CompressionModules, - EncryptionModules = Server.Serializable.ServerSettings.EncryptionModules, - BackendModules = Server.Serializable.ServerSettings.BackendModules, - GenericModules = Server.Serializable.ServerSettings.GenericModules, - WebModules = Server.Serializable.ServerSettings.WebModules, - ConnectionModules = Server.Serializable.ServerSettings.ConnectionModules, - ServerModules = Server.Serializable.ServerSettings.ServerModules, - SecretProviderModules = Server.Serializable.ServerSettings.SecretProviderModules, - UsingAlternateUpdateURLs = AutoUpdateSettings.UsesAlternateURLs, - LogLevels = Enum.GetNames(typeof(Library.Logging.LogMessageType)), - SpecialFolders = SpecialFolders.Nodes.Select(n => new Dto.SystemInfoDto.SpecialFolderDto { ID = n.id, Path = n.resolvedpath }).ToArray(), - SupportedLocales = Library.Localization.LocalizationService.SupportedCultures + /// The new instance. + private static StaticSystemInformation CreateStaticSystemInformation() + => new StaticSystemInformation + { + APIVersion = 1, + PasswordPlaceholder = FIXMEGlobal.PASSWORD_PLACEHOLDER, + ServerVersion = UpdaterManager.SelfVersion.Version, + ServerVersionName = License.VersionNumbers.VERSION_NAME, + ServerVersionType = UpdaterManager.SelfVersion.ReleaseType, + RemoteControlRegistrationUrl = Library.RemoteControl.RegisterForRemote.DefaultRegisterationUrl, + StartedBy = FIXMEGlobal.Origin, + DefaultUpdateChannel = AutoUpdateSettings.DefaultUpdateChannel.ToString(), + DefaultUsageReportLevel = Library.UsageReporter.Reporter.DefaultReportLevel, + OSType = UpdaterManager.OperatingSystemName, + OSVersion = Library.UsageReporter.OSInfoHelper.PlatformString, + DirectorySeparator = Path.DirectorySeparatorChar, + PathSeparator = Path.PathSeparator, + CaseSensitiveFilesystem = Library.Utility.Utility.IsFSCaseSensitive, + MachineName = Environment.MachineName, + PackageTypeId = UpdaterManager.PackageTypeId, + UserName = OperatingSystem.IsWindows() ? System.Security.Principal.WindowsIdentity.GetCurrent().Name : Environment.UserName, + NewLine = Environment.NewLine, + CLRVersion = Environment.Version.ToString(), + Options = Server.Serializable.ServerSettings.Options, + CompressionModules = Server.Serializable.ServerSettings.CompressionModules, + EncryptionModules = Server.Serializable.ServerSettings.EncryptionModules, + BackendModules = Server.Serializable.ServerSettings.BackendModules, + GenericModules = Server.Serializable.ServerSettings.GenericModules, + WebModules = Server.Serializable.ServerSettings.WebModules, + ConnectionModules = Server.Serializable.ServerSettings.ConnectionModules, + ServerModules = Server.Serializable.ServerSettings.ServerModules, + SecretProviderModules = Server.Serializable.ServerSettings.SecretProviderModules, + UsingAlternateUpdateURLs = AutoUpdateSettings.UsesAlternateURLs, + LogLevels = Enum.GetNames(typeof(Library.Logging.LogMessageType)), + SpecialFolders = SpecialFolders.Nodes.Select(n => new Dto.SystemInfoDto.SpecialFolderDto { ID = n.id, Path = n.resolvedpath }).ToArray(), + SupportedLocales = Library.Localization.LocalizationService.SupportedCultures .Select(x => new Dto.SystemInfoDto.LocaleDto { Code = x, EnglishName = new CultureInfo(x).EnglishName, DisplayName = new CultureInfo(x).NativeName }).ToArray(), - TimeZones = Library.Utility.TimeZoneHelper.GetTimeZones() + TimeZones = Library.Utility.TimeZoneHelper.GetTimeZones() .Select(x => new Dto.SystemInfoDto.TimeZoneDto { ID = x.Id, DisplayName = x.DisplayName, CurrentUTCOffset = x.CurrentUtcOffset.ToString() }), - }); + }; + + /// + /// The lock guarding the _systemInfoBases cache. + /// + private readonly object _lock = new(); + + /// + /// Cached static system information, the key is the language code. + /// + private Dictionary _systemInfoBases = new(StringComparer.OrdinalIgnoreCase); /// public SystemInfoDto GetSystemInfo(CultureInfo? browserlanguage) { browserlanguage ??= CultureInfo.InvariantCulture; - var systeminfo = _systemInfoBase.Value; + var currentLanguage = LocalizationService.Current?.Culture.Name; + if (string.IsNullOrWhiteSpace(currentLanguage)) + currentLanguage = CultureInfo.CurrentUICulture.Name; + + StaticSystemInformation? systeminfo; + + lock (_lock) + { + if (!_systemInfoBases.TryGetValue(currentLanguage, out systeminfo)) + { + systeminfo = CreateStaticSystemInformation(); + _systemInfoBases[currentLanguage] = systeminfo; + } + } // Return the system information, patch in dynamic values return new SystemInfoDto() @@ -305,7 +331,7 @@ public class SystemInfoProvider : ISystemInfoProvider DisplayName = browserlanguage.NativeName }, SupportedLocales = systeminfo.SupportedLocales, - BrowserLocaleSupported = Library.Localization.LocalizationService.isCultureSupported(browserlanguage), + BrowserLocaleSupported = Library.Localization.LocalizationService.IsCultureSupported(browserlanguage), TimeZones = systeminfo.TimeZones, }; } From e74ae5555c10c3ba0ab92356d252f783a3131b91 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 5 Mar 2025 16:18:19 +0100 Subject: [PATCH 296/362] Removed used of callcontext in favor of direct `AsyncLocal` usage --- .../Localization/LocalizationContext.cs | 57 +++++++++++-------- .../Localization/LocalizationService.cs | 7 +-- 2 files changed, 34 insertions(+), 30 deletions(-) diff --git a/Duplicati/Library/Localization/LocalizationContext.cs b/Duplicati/Library/Localization/LocalizationContext.cs index d2162bd05..9fed23cbb 100644 --- a/Duplicati/Library/Localization/LocalizationContext.cs +++ b/Duplicati/Library/Localization/LocalizationContext.cs @@ -1,26 +1,26 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using Duplicati.Library.Logging; using System; +using System.Threading; namespace Duplicati.Library.Localization { @@ -29,10 +29,14 @@ namespace Duplicati.Library.Localization /// internal class LocalizationContext : IDisposable { + /// + /// The current culture, using an async local to keep the call context + /// + private static readonly AsyncLocal _currentCulture = new(); /// /// The previous context /// - private readonly object m_prev; + private readonly string m_prev; /// /// Flag to prevent double dispose @@ -45,12 +49,17 @@ namespace Duplicati.Library.Localization /// The localization to use. public LocalizationContext(System.Globalization.CultureInfo ci) { - m_prev = CallContext.GetData(LocalizationService.LOGICAL_CONTEXT_KEY) as string; - CallContext.SetData(LocalizationService.LOGICAL_CONTEXT_KEY, ci.Name); + m_prev = _currentCulture.Value; + _currentCulture.Value = ci.Name; m_isDisposed = false; } + /// + /// Returns the current culture value for the call context + /// + public static string Current => _currentCulture.Value; + /// /// Releases all resource used by the object. /// @@ -64,7 +73,7 @@ namespace Duplicati.Library.Localization { if (!m_isDisposed) { - CallContext.SetData(LocalizationService.LOGICAL_CONTEXT_KEY, m_prev); + _currentCulture.Value = m_prev; m_isDisposed = true; } } diff --git a/Duplicati/Library/Localization/LocalizationService.cs b/Duplicati/Library/Localization/LocalizationService.cs index adc53e327..8d62b88fd 100644 --- a/Duplicati/Library/Localization/LocalizationService.cs +++ b/Duplicati/Library/Localization/LocalizationService.cs @@ -38,11 +38,6 @@ namespace Duplicati.Library.Localization /// private static readonly Dictionary Services = new Dictionary(); - /// - /// The key for accessing logical context - /// - internal const string LOGICAL_CONTEXT_KEY = "DUPLICATI_LOCALIZATION_CULTURE_CONTEXT"; - /// /// Regular expression to match a locale /// @@ -96,7 +91,7 @@ namespace Duplicati.Library.Localization { get { - var lc = CallContext.GetData(LOGICAL_CONTEXT_KEY) as string; + var lc = LocalizationContext.Current; if (!string.IsNullOrWhiteSpace(lc)) return Get(new CultureInfo(lc)); return Get(CultureInfo.CurrentCulture); From 5177399bc20fb14aa6709399f79a27f75d030f7d Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 5 Mar 2025 16:31:13 +0100 Subject: [PATCH 297/362] Changed to UI culture by default for web output --- .../Library/Localization/LocalizationService.cs | 14 ++++++++++++++ .../WebserverCore/Middlewares/LanguageFilter.cs | 3 ++- .../WebserverCore/Services/SystemInfoProvider.cs | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Localization/LocalizationService.cs b/Duplicati/Library/Localization/LocalizationService.cs index 8d62b88fd..636a75217 100644 --- a/Duplicati/Library/Localization/LocalizationService.cs +++ b/Duplicati/Library/Localization/LocalizationService.cs @@ -98,6 +98,20 @@ namespace Duplicati.Library.Localization } } + /// + /// Gets a localization provider with the current UI language + /// + public static ILocalizationService CurrentUI + { + get + { + var lc = LocalizationContext.Current; + if (!string.IsNullOrWhiteSpace(lc)) + return Get(new CultureInfo(lc)); + return Get(CultureInfo.CurrentUICulture); + } + } + /// /// Gets a localization provider with the OS install language /// diff --git a/Duplicati/WebserverCore/Middlewares/LanguageFilter.cs b/Duplicati/WebserverCore/Middlewares/LanguageFilter.cs index 353720ebf..a376baee5 100644 --- a/Duplicati/WebserverCore/Middlewares/LanguageFilter.cs +++ b/Duplicati/WebserverCore/Middlewares/LanguageFilter.cs @@ -18,6 +18,7 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +using System.Globalization; using Duplicati.Library.Localization; namespace Duplicati.WebserverCore.Middlewares; @@ -30,7 +31,7 @@ public class LanguageFilter : IEndpointFilter ? LocalizationService.ParseCulture(locale) : null; - using (ci == null ? null : LocalizationService.TemporaryContext(ci)) + using (LocalizationService.TemporaryContext(ci ?? CultureInfo.CurrentUICulture)) return await next(context); } diff --git a/Duplicati/WebserverCore/Services/SystemInfoProvider.cs b/Duplicati/WebserverCore/Services/SystemInfoProvider.cs index 8d9679c64..724732457 100644 --- a/Duplicati/WebserverCore/Services/SystemInfoProvider.cs +++ b/Duplicati/WebserverCore/Services/SystemInfoProvider.cs @@ -273,7 +273,7 @@ public class SystemInfoProvider : ISystemInfoProvider public SystemInfoDto GetSystemInfo(CultureInfo? browserlanguage) { browserlanguage ??= CultureInfo.InvariantCulture; - var currentLanguage = LocalizationService.Current?.Culture.Name; + var currentLanguage = LocalizationService.CurrentUI?.Culture.Name; if (string.IsNullOrWhiteSpace(currentLanguage)) currentLanguage = CultureInfo.CurrentUICulture.Name; From fe5c536e536b75400d7082a63b7587d54b5a76d3 Mon Sep 17 00:00:00 2001 From: "Marcelo C." Date: Wed, 5 Mar 2025 16:38:25 -0300 Subject: [PATCH 298/362] Azure Library Upgrade and Fixes - Removed obsolete Microsoft.Azure.* references which were deprecated and replaced with Azure.Storage.* - Cleaned up unused references - Changes to code due to obsolete classes - Patched some missing ConfigureAwait(false) on calls that could freeze the UI - Merged in new async List changes. --- .../Backend/AzureBlob/AzureBlobBackend.cs | 64 +++--- .../Backend/AzureBlob/AzureBlobWrapper.cs | 184 ++++++++---------- ...Duplicati.Library.Backend.AzureBlob.csproj | 13 +- .../Library/Backend/AzureBlob/Strings.cs | 41 ++-- 4 files changed, 130 insertions(+), 172 deletions(-) diff --git a/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs b/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs index 2c6b6fcd8..d82962eed 100644 --- a/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs +++ b/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs @@ -19,12 +19,9 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +using Azure; +using Azure.Storage.Blobs.Models; using Duplicati.Library.Interface; -using Microsoft.Azure.Storage.Shared.Protocol; -using System.Collections.Generic; -using System.IO; -using System.Threading; -using System.Threading.Tasks; namespace Duplicati.Library.Backend.AzureBlob { @@ -77,31 +74,22 @@ namespace Duplicati.Library.Backend.AzureBlob { throw new UserInformationException(Strings.AzureBlobBackend.NoAccessKeyOrSasToken, "AzureNoAccessKeyOrSasToken"); } - - + _azureBlob = new AzureBlobWrapper(storageAccountName, accessKey, sasToken, containerName); } - public string DisplayName - { - get { return Strings.AzureBlobBackend.DisplayName; } - } + public string DisplayName => Strings.AzureBlobBackend.DisplayName; - public string ProtocolKey - { - get { return "azure"; } - } + public string ProtocolKey => "azure"; public IAsyncEnumerable ListAsync(CancellationToken cancelToken) => _azureBlob.ListContainerEntriesAsync(cancelToken); public async Task PutAsync(string remotename, string localname, CancellationToken cancelToken) { - using (var fs = File.Open(localname, - FileMode.Open, FileAccess.Read, FileShare.Read)) - { - await PutAsync(remotename, fs, cancelToken); - } + await using var fs = File.Open(localname, + FileMode.Open, FileAccess.Read, FileShare.Read); + await PutAsync(remotename, fs, cancelToken).ConfigureAwait(false); } public Task PutAsync(string remotename, Stream input, CancellationToken cancelToken) @@ -111,12 +99,10 @@ namespace Duplicati.Library.Backend.AzureBlob public async Task GetAsync(string remotename, string localname, CancellationToken cancellationToken) { - using (var fs = File.Open(localname, + await using var fs = File.Open(localname, FileMode.Create, FileAccess.Write, - FileShare.None)) - { - await GetAsync(remotename, fs, cancellationToken).ConfigureAwait(false); - } + FileShare.None); + await GetAsync(remotename, fs, cancellationToken).ConfigureAwait(false); } public Task GetAsync(string remotename, Stream output, CancellationToken cancellationToken) @@ -159,13 +145,7 @@ namespace Duplicati.Library.Backend.AzureBlob } } - public string Description - { - get - { - return Strings.AzureBlobBackend.Description_v2; - } - } + public string Description => Strings.AzureBlobBackend.DescriptionV2; public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(_azureBlob.DnsNames); @@ -176,23 +156,27 @@ namespace Duplicati.Library.Backend.AzureBlob { return WrapWithExceptionHandler(_azureBlob.AddContainerAsync(cancellationToken)); } - + + /// + /// Wraps the task with exception handling + /// private async Task WrapWithExceptionHandler(Task task) { try { await task.ConfigureAwait(false); } - catch (Microsoft.WindowsAzure.Storage.StorageException e) - when (e.RequestInformation.HttpStatusCode == 404 - || e.RequestInformation.HttpStatusMessage == StorageErrorCodeStrings.ResourceNotFound) + catch (RequestFailedException e) + when (e.Status == 404 + || e.ErrorCode == BlobErrorCode.BlobNotFound + || e.ErrorCode == BlobErrorCode.ResourceNotFound) { throw new FileMissingException(e.Message, e); } - catch (Microsoft.WindowsAzure.Storage.StorageException e) - when (e.RequestInformation.HttpStatusMessage == StorageErrorCodeStrings.ContainerNotFound - || e.RequestInformation.HttpStatusMessage == StorageErrorCodeStrings.ContainerBeingDeleted - || e.RequestInformation.HttpStatusMessage == StorageErrorCodeStrings.ContainerDisabled) + catch (RequestFailedException e) + when (e.ErrorCode == BlobErrorCode.ContainerNotFound + || e.ErrorCode == BlobErrorCode.ContainerBeingDeleted + || e.ErrorCode == BlobErrorCode.ContainerDisabled) { throw new FolderMissingException(e.Message, e); } diff --git a/Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs b/Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs index 7d5b00b90..b4e9b8830 100644 --- a/Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs +++ b/Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs @@ -19,16 +19,12 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System.Collections.Generic; -using System.IO; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; using System.Runtime.CompilerServices; -using System.Threading; -using System.Threading.Tasks; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; -using Duplicati.Library.Utility; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Blob; +using Uri = System.Uri; namespace Duplicati.Library.Backend.AzureBlob { @@ -37,141 +33,125 @@ namespace Duplicati.Library.Backend.AzureBlob /// public class AzureBlobWrapper { - private readonly string _containerName; - private readonly CloudBlobContainer _container; - private readonly OperationContext _operationContext; - - // Note: May need metadata; need to test with Azure blobs - private const BlobListingDetails ListDetails = BlobListingDetails.None; + private readonly BlobContainerClient _container; + /// + /// Gets an array of DNS names associated with the blob container. + /// + /// An array of DNS hostnames for the primary and secondary URIs of the container. public string[] DnsNames { get { var lst = new List(); - if (_container != null) - { - if (_container.Uri != null) - lst.Add(_container.Uri.Host); - - if (_container.StorageUri != null) - { - if (_container.StorageUri.PrimaryUri != null) - lst.Add(_container.StorageUri.PrimaryUri.Host); - if (_container.StorageUri.SecondaryUri != null) - lst.Add(_container.StorageUri.SecondaryUri.Host); - } - } - + if (_container != null && _container.Uri != null) lst.Add(_container.Uri.Host); return lst.ToArray(); } } + /// + /// Initializes a new instance of the AzureBlobWrapper class. + /// + /// The Azure storage account name. + /// The access key for the storage account. + /// The Shared Access Signature (SAS) token for authentication. + /// The name of the blob container. public AzureBlobWrapper(string accountName, string accessKey, string sasToken, string containerName) { - _operationContext = new() - { - CustomUserAgent = string.Format( - "APN/1.0 Duplicati/{0} AzureBlob/2.0 {1}", - System.Reflection.Assembly.GetExecutingAssembly().GetName().Version, - Microsoft.WindowsAzure.Storage.Shared.Protocol.Constants.HeaderConstants.UserAgent - ) - }; - - string connectionString; + BlobServiceClient blobServiceClient; if (sasToken != null) { - connectionString = string.Format("DefaultEndpointsProtocol=https;AccountName={0};SharedAccessSignature={1}", - accountName, sasToken); + var sasUri = new Uri($"https://{accountName}.blob.core.windows.net/?{sasToken}"); + blobServiceClient = new BlobServiceClient(sasUri); } else { - connectionString = string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", - accountName, accessKey); + var connectionString = $"DefaultEndpointsProtocol=https;AccountName={accountName};AccountKey={accessKey};EndpointSuffix=core.windows.net"; + blobServiceClient = new BlobServiceClient(connectionString); } - var storageAccount = CloudStorageAccount.Parse(connectionString); - var blobClient = storageAccount.CreateCloudBlobClient(); - - _containerName = containerName; - _container = blobClient.GetContainerReference(containerName); + _container = blobServiceClient.GetBlobContainerClient(containerName); } + /// + /// Creates a new blob container asynchronously and sets its access permissions to private. + /// + /// A token to monitor for cancellation requests. + /// A task that represents the asynchronous operation. public async Task AddContainerAsync(CancellationToken cancellationToken) { - await _container.CreateAsync(default, default, _operationContext, cancellationToken).ConfigureAwait(false); - await _container.SetPermissionsAsync(new BlobContainerPermissions { PublicAccess = BlobContainerPublicAccessType.Off }, default, default, _operationContext, cancellationToken).ConfigureAwait(false); + await _container.CreateAsync(cancellationToken: cancellationToken).ConfigureAwait(false); } - public virtual Task GetFileStreamAsync(string keyName, Stream target, CancellationToken cancellationToken) + /// + /// Downloads a blob to a stream asynchronously. + /// + /// The name of the blob to download. + /// The stream to download the blob to. + /// A token to monitor for cancellation requests. + /// A task that represents the asynchronous download operation. + public async Task GetFileStreamAsync(string keyName, Stream target, CancellationToken cancellationToken) { - return _container.GetBlockBlobReference(keyName).DownloadToStreamAsync(target, default, default, _operationContext, cancellationToken); + var blobClient = _container.GetBlobClient(keyName); + await blobClient.DownloadToAsync(target, cancellationToken).ConfigureAwait(false); } - public virtual Task AddFileStream(string keyName, Stream source, CancellationToken cancelToken) + /// + /// Uploads a stream to a blob asynchronously. + /// + /// The name to give the uploaded blob. + /// The stream containing the data to upload. + /// A token to monitor for cancellation requests. + /// A task that represents the asynchronous upload operation. + public async Task AddFileStream(string keyName, Stream source, CancellationToken cancelToken) { - return _container.GetBlockBlobReference(keyName).UploadFromStreamAsync(source, source.Length, default, default, _operationContext, cancelToken); + var blobClient = _container.GetBlobClient(keyName); + await blobClient.UploadAsync(source, cancelToken).ConfigureAwait(false); } - public Task DeleteObjectAsync(string keyName, CancellationToken cancelToken) + /// + /// Deletes a blob if it exists asynchronously. + /// + /// The name of the blob to delete. + /// A token to monitor for cancellation requests. + public async Task DeleteObjectAsync(string keyName, CancellationToken cancelToken) { - return _container.GetBlockBlobReference(keyName).DeleteIfExistsAsync(default, default, default, _operationContext, cancelToken); + var blobClient = _container.GetBlobClient(keyName); + await blobClient.DeleteIfExistsAsync(cancellationToken: cancelToken).ConfigureAwait(false); } - - private async IAsyncEnumerable ListContainerBlobEntriesAsync([EnumeratorCancellation] CancellationToken cancelToken) + + /// + /// List container files. + /// + /// A token to monitor for cancellation requests. + /// + /// Thrown when the container is not found + public virtual async IAsyncEnumerable ListContainerEntriesAsync([EnumeratorCancellation] CancellationToken cancelToken) { - BlobResultSegment segment; - try + IAsyncEnumerable blobs = GetBlobsWithExceptionHandling(); + + await foreach (var blobItem in blobs.WithCancellation(cancelToken)) { - segment = await _container.ListBlobsSegmentedAsync(null, false, ListDetails, null, null, null, _operationContext, cancelToken).ConfigureAwait(false); + var blobName = Uri.UnescapeDataString(blobItem.Name.Replace("+", "%2B")); + + if (blobItem is { } bi) + { + var lastModified = bi.Properties.LastModified?.UtcDateTime ?? DateTime.UtcNow; + yield return new FileEntry(blobName, bi.Properties.ContentLength ?? 0, lastModified, lastModified); + } } - catch (StorageException ex) + + IAsyncEnumerable GetBlobsWithExceptionHandling() { - if (ex.RequestInformation.HttpStatusCode == 404) + try + { + // Note: AsyncPageable return internally uses .ConfigureAwait(false). + return _container.GetBlobsAsync(cancellationToken: cancelToken); + } + catch (Azure.RequestFailedException ex) when (ex.Status == 404) { throw new FolderMissingException(ex); } - throw; - } - - foreach (var item in segment.Results) - yield return item; - - while (segment.ContinuationToken != null) - { - segment = await _container.ListBlobsSegmentedAsync(null, false, ListDetails, null, segment.ContinuationToken, null, _operationContext, cancelToken).ConfigureAwait(false); - - foreach (var item in segment.Results) - yield return item; - } - } - - public virtual async IAsyncEnumerable ListContainerEntriesAsync([EnumeratorCancellation] CancellationToken cancelToken) - { - await foreach (var x in ListContainerBlobEntriesAsync(cancelToken).ConfigureAwait(false)) - { - var absolutePath = x.StorageUri.PrimaryUri.AbsolutePath; - var containerSegment = string.Concat("/", _containerName, "/"); - var blobName = absolutePath.Substring(absolutePath.IndexOf( - containerSegment, System.StringComparison.Ordinal) + containerSegment.Length); - - var res = new FileEntry(Uri.UrlDecode(blobName.Replace("+", "%2B"))); - try - { - if (x is CloudBlockBlob cb) - { - var lastModified = new System.DateTime(); - if (cb.Properties.LastModified != null) - lastModified = new System.DateTime(cb.Properties.LastModified.Value.Ticks, System.DateTimeKind.Utc); - res = new FileEntry(res.Name, cb.Properties.Length, lastModified, lastModified); - } - } - catch - { - // If the metadata fails to parse, return the basic entry - } - - yield return res; } } } diff --git a/Duplicati/Library/Backend/AzureBlob/Duplicati.Library.Backend.AzureBlob.csproj b/Duplicati/Library/Backend/AzureBlob/Duplicati.Library.Backend.AzureBlob.csproj index 0b52f52bb..f3de97e26 100644 --- a/Duplicati/Library/Backend/AzureBlob/Duplicati.Library.Backend.AzureBlob.csproj +++ b/Duplicati/Library/Backend/AzureBlob/Duplicati.Library.Backend.AzureBlob.csproj @@ -4,18 +4,11 @@ net8.0 Library Copyright © 2025 Team Duplicati, MIT license + enable - - - - - - - - - - + + diff --git a/Duplicati/Library/Backend/AzureBlob/Strings.cs b/Duplicati/Library/Backend/AzureBlob/Strings.cs index d877ff22b..e0693f8d2 100644 --- a/Duplicati/Library/Backend/AzureBlob/Strings.cs +++ b/Duplicati/Library/Backend/AzureBlob/Strings.cs @@ -20,23 +20,24 @@ // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Localization.Short; -namespace Duplicati.Library.Backend.AzureBlob.Strings { - internal static class AzureBlobBackend { - public static string Description_v2 { get { return LC.L(@"This backend can read and write data to Azure blob storage. Allowed format is ""azure://bucketname""."); } } - public static string DisplayName { get { return LC.L(@"Azure blob"); } } - public static string ContainerNameDescriptionLong { get { return LC.L(@"All files will be written to the container specified."); } } - public static string ContainerNameDescriptionShort { get { return LC.L(@"The name of the storage container"); } } - public static string NoStorageAccountName { get { return LC.L(@"No Azure storage account name given"); } } - public static string StorageAccountNameDescriptionLong { get { return LC.L(@"The Azure storage account name which can be obtained by clicking the ""Manage Access Keys"" button on the storage account dashboard."); } } - public static string StorageAccountNameDescriptionShort { get { return LC.L(@"The storage account name"); } } - public static string AccessKeyDescriptionLong { get { return LC.L(@"The Azure access key which can be obtained by clicking the ""Manage Access Keys"" button on the storage account dashboard."); } } - public static string AccessKeyDescriptionShort { get { return LC.L(@"The access key"); } } - public static string SasTokenDescriptionLong { get { return LC.L(@"The Azure shared access signature (SAS) token which can be obtained by selecting the ""Shared access signature"" blade on the storage account dashboard, or inside a container blade."); } } - public static string SasTokenDescriptionShort { get { return LC.L(@"The SAS token"); } } - public static string NoAccessKeyOrSasToken { get { return LC.L(@"No Azure access key or SAS token given"); } } - public static string AuthPasswordDescriptionLong { get { return LC.L(@"The password used to connect to the server. This may also be supplied as the environment variable ""AUTH_PASSWORD""."); } } - public static string AuthPasswordDescriptionShort { get { return LC.L(@"Supply the password used to connect to the server"); } } - public static string AuthUsernameDescriptionLong { get { return LC.L(@"The username used to connect to the server. This may also be supplied as the environment variable ""AUTH_USERNAME""."); } } - public static string AuthUsernameDescriptionShort { get { return LC.L(@"Supply the username used to connect to the server"); } } - } -} + +namespace Duplicati.Library.Backend.AzureBlob.Strings; + +internal static class AzureBlobBackend { + public static string DescriptionV2 => LC.L(@"This backend can read and write data to Azure blob storage. Allowed format is ""azure://bucketname""."); + public static string DisplayName => LC.L(@"Azure blob"); + public static string ContainerNameDescriptionLong => LC.L(@"All files will be written to the container specified."); + public static string ContainerNameDescriptionShort => LC.L(@"The name of the storage container"); + public static string NoStorageAccountName => LC.L(@"No Azure storage account name given"); + public static string StorageAccountNameDescriptionLong => LC.L(@"The Azure storage account name which can be obtained by clicking the ""Manage Access Keys"" button on the storage account dashboard."); + public static string StorageAccountNameDescriptionShort => LC.L(@"The storage account name"); + public static string AccessKeyDescriptionLong => LC.L(@"The Azure access key which can be obtained by clicking the ""Manage Access Keys"" button on the storage account dashboard."); + public static string AccessKeyDescriptionShort => LC.L(@"The access key"); + public static string SasTokenDescriptionLong => LC.L(@"The Azure shared access signature (SAS) token which can be obtained by selecting the ""Shared access signature"" blade on the storage account dashboard, or inside a container blade."); + public static string SasTokenDescriptionShort => LC.L(@"The SAS token"); + public static string NoAccessKeyOrSasToken => LC.L(@"No Azure access key or SAS token given"); + public static string AuthPasswordDescriptionLong => LC.L(@"The password used to connect to the server. This may also be supplied as the environment variable ""AUTH_PASSWORD""."); + public static string AuthPasswordDescriptionShort => LC.L(@"Supply the password used to connect to the server"); + public static string AuthUsernameDescriptionLong => LC.L(@"The username used to connect to the server. This may also be supplied as the environment variable ""AUTH_USERNAME""."); + public static string AuthUsernameDescriptionShort => LC.L(@"Supply the username used to connect to the server"); +} \ No newline at end of file From d3bc413b949e88eb11ac339932dc1d98caeb657e Mon Sep 17 00:00:00 2001 From: "Marcelo C." Date: Wed, 5 Mar 2025 23:13:02 -0300 Subject: [PATCH 299/362] Explicit permission + force overwrite --- Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs b/Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs index b4e9b8830..2cdfbbe38 100644 --- a/Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs +++ b/Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs @@ -80,7 +80,8 @@ namespace Duplicati.Library.Backend.AzureBlob /// A task that represents the asynchronous operation. public async Task AddContainerAsync(CancellationToken cancellationToken) { - await _container.CreateAsync(cancellationToken: cancellationToken).ConfigureAwait(false); + // Even though PublicAccessType.None is by default, we set it explicitly to highlight it. + await _container.CreateAsync(PublicAccessType.None, cancellationToken: cancellationToken).ConfigureAwait(false); } /// @@ -106,7 +107,7 @@ namespace Duplicati.Library.Backend.AzureBlob public async Task AddFileStream(string keyName, Stream source, CancellationToken cancelToken) { var blobClient = _container.GetBlobClient(keyName); - await blobClient.UploadAsync(source, cancelToken).ConfigureAwait(false); + await blobClient.UploadAsync(source, true, cancelToken).ConfigureAwait(false); } /// @@ -130,7 +131,7 @@ namespace Duplicati.Library.Backend.AzureBlob { IAsyncEnumerable blobs = GetBlobsWithExceptionHandling(); - await foreach (var blobItem in blobs.WithCancellation(cancelToken)) + await foreach (var blobItem in blobs.WithCancellation(cancelToken).ConfigureAwait(false)) { var blobName = Uri.UnescapeDataString(blobItem.Name.Replace("+", "%2B")); From 277362ce64aae7b84147383e8b5be98781f1c625 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 6 Mar 2025 10:48:52 +0100 Subject: [PATCH 300/362] Another attempt at fixing the failing tests --- Duplicati/UnitTest/DisruptionTests.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Duplicati/UnitTest/DisruptionTests.cs b/Duplicati/UnitTest/DisruptionTests.cs index 32c053fe6..083a6bab9 100644 --- a/Duplicati/UnitTest/DisruptionTests.cs +++ b/Duplicati/UnitTest/DisruptionTests.cs @@ -66,7 +66,7 @@ namespace Duplicati.UnitTest { if (path.EndsWith(this.fileSizes[2] + "MB")) { - Thread.Sleep(2000); + Thread.Sleep(500); controller.Stop(); stopped.TrySetResult(true); } @@ -264,6 +264,8 @@ namespace Duplicati.UnitTest Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[0].IsFullBackup); } + ModifySourceFiles(); + // Run another partial backup. We will then verify that a full backup is retained // even when all the "recent" backups are partial. using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) @@ -594,6 +596,8 @@ namespace Duplicati.UnitTest Assert.AreEqual(BackupType.FULL_BACKUP, c.List().Filesets.Single(x => x.Version == 0).IsFullBackup); } + ModifySourceFiles(); + // Run a partial backup. using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { From 13214885270ed9082c9cdbfce4978d787269265e Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 6 Mar 2025 12:02:11 +0100 Subject: [PATCH 301/362] Improve failure detection This commit adds extra logic to prevent cases where the backup reports success but leaves behind partial files. A new flag is recorded to track if partial files are possible. If no partial files are possible, the verification will fail if any are found. --- .../Library/Main/Backend/BackendManager.cs | 1 - .../Library/Main/Database/LocalDatabase.cs | 50 ++++++------- .../Library/Main/Operation/BackupHandler.cs | 14 +++- .../Library/Main/Operation/CompactHandler.cs | 10 ++- .../Library/Main/Operation/DeleteHandler.cs | 2 +- .../Main/Operation/FilelistProcessor.cs | 75 +++++++++++++++++-- .../Main/Operation/ListBrokenFilesHandler.cs | 2 +- .../Main/Operation/PurgeFilesHandler.cs | 8 +- .../Library/Main/Operation/RepairHandler.cs | 7 +- .../Library/Main/Operation/RestoreHandler.cs | 4 +- .../Library/Main/Operation/TestHandler.cs | 2 +- 11 files changed, 126 insertions(+), 49 deletions(-) diff --git a/Duplicati/Library/Main/Backend/BackendManager.cs b/Duplicati/Library/Main/Backend/BackendManager.cs index 2e5d6d4b7..2c948edbe 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.cs @@ -6,7 +6,6 @@ using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using CoCoL; -using Duplicati.Library.DynamicLoader; using Duplicati.Library.Interface; using Duplicati.Library.Main.Database; using Duplicati.Library.Main.Operation.Common; diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 84effc3c0..bca0d32af 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -737,42 +737,34 @@ AND Fileset.ID NOT IN return GetDbOptionList(transaction).ToDictionary(x => x.Key, x => x.Value); } + private void UpdateDbOption(string key, bool value) + { + var opts = GetDbOptions(); + + if (value) + opts["repair-in-progress"] = "true"; + else + opts.Remove("repair-in-progress"); + + SetDbOptions(opts); + } + public bool RepairInProgress { - get - { - return GetDbOptions().ContainsKey("repair-in-progress"); - } - set - { - var opts = GetDbOptions(); - - if (value) - opts["repair-in-progress"] = "true"; - else - opts.Remove("repair-in-progress"); - - SetDbOptions(opts); - } + get => GetDbOptions().ContainsKey("repair-in-progress"); + set => UpdateDbOption("repair-in-progress", value); } public bool PartiallyRecreated { - get - { - return GetDbOptions().ContainsKey("partially-recreated"); - } - set - { - var opts = GetDbOptions(); + get => GetDbOptions().ContainsKey("partially-recreated"); + set => UpdateDbOption("partially-recreated", value); + } - if (value) - opts["partially-recreated"] = "true"; - else - opts.Remove("partially-recreated"); - - SetDbOptions(opts); - } + public bool UncleanShutdown + { + get => GetDbOptions().ContainsKey("unclean-shutdown"); + set => UpdateDbOption("unclean-shutdown", value); } public void SetDbOptions(IDictionary options, System.Data.IDbTransaction transaction = null) diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 93a8e3222..e69854f7d 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -343,7 +343,9 @@ namespace Duplicati.Library.Main.Operation await UpdateStorageStatsFromDatabase(result, database, options, backendManager, result.TaskControl.ProgressToken).ConfigureAwait(false); } else - await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, new string[] { lastTempFilelist }, logErrors: false).ConfigureAwait(false); + { + await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, new string[] { lastTempFilelist }, logErrors: false, verifyMode: FilelistProcessor.VerifyMode.VerifyAndClean).ConfigureAwait(false); + } } catch (RemoteListVerificationException ex) { @@ -362,7 +364,7 @@ namespace Duplicati.Library.Main.Operation result.SetDatabase(database); Log.WriteInformationMessage(LOGTAG, "BackendCleanupFinished", "Backend cleanup finished, retrying verification"); - await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, new string[] { lastTempFilelist }).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, new string[] { lastTempFilelist }, logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); } else throw; @@ -506,7 +508,7 @@ namespace Duplicati.Library.Main.Operation { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PostBackupVerify); using (new Logging.Timer(LOGTAG, "AfterBackupVerify", "AfterBackupVerify")) - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, m_database, m_result.BackendWriter, new string[] { currentFilelistVolume }).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, m_database, m_result.BackendWriter, new string[] { currentFilelistVolume }, logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); await backendManager.WaitForEmptyAsync(m_database, m_transaction, m_taskReader.ProgressToken); long remoteVolumeCount = m_database.GetRemoteVolumes().LongCount(x => x.State == RemoteVolumeState.Verified); @@ -613,6 +615,10 @@ namespace Duplicati.Library.Main.Operation Task parallelScanner = null; try { + // If we crash now, there is a chance we leave behind partial remote files + if (!m_options.Dryrun) + database.UncleanShutdown = true; + using (m_database = database) using (var db = new Backup.BackupDatabase(m_database, m_options)) // Setup runners and instances here @@ -691,6 +697,8 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_WaitForUpload); var lastVolumeSize = await FlushBackend(m_database, m_transaction, m_result, backendManager).ConfigureAwait(false); + if (!m_options.Dryrun) + database.UncleanShutdown = false; // Make sure we have the database up-to-date await db.CommitTransactionAsync("CommitAfterUpload", false); diff --git a/Duplicati/Library/Main/Operation/CompactHandler.cs b/Duplicati/Library/Main/Operation/CompactHandler.cs index b691e9ffe..abd279869 100644 --- a/Duplicati/Library/Main/Operation/CompactHandler.cs +++ b/Duplicati/Library/Main/Operation/CompactHandler.cs @@ -104,7 +104,7 @@ namespace Duplicati.Library.Main.Operation { // Workaround where we allow a running backendmanager to be used if (!hasVerifiedBackend) - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, true, transaction).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, true, FilelistProcessor.VerifyMode.VerifyAndClean, transaction).ConfigureAwait(false); var newvol = new BlockVolumeWriter(m_options); newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, transaction); @@ -146,6 +146,10 @@ namespace Duplicati.Library.Main.Operation if (report.ShouldCompact) { + // If we crash now, we may leave partial files + if (!m_options.Dryrun) + db.UncleanShutdown = true; + newvolindex?.StartVolume(newvol.RemoteFilename); List volumesToDownload = []; if (report.CompactableVolumes.Any()) @@ -240,6 +244,10 @@ namespace Duplicati.Library.Main.Operation } } } + + // The remainder of the operation cannot leave partial files + if (!m_options.Dryrun) + db.UncleanShutdown = false; } else { diff --git a/Duplicati/Library/Main/Operation/DeleteHandler.cs b/Duplicati/Library/Main/Operation/DeleteHandler.cs index fdd233154..477574311 100644 --- a/Duplicati/Library/Main/Operation/DeleteHandler.cs +++ b/Duplicati/Library/Main/Operation/DeleteHandler.cs @@ -86,7 +86,7 @@ namespace Duplicati.Library.Main.Operation { CancellationToken cancellationToken = CancellationToken.None; if (!hasVerifiedBackend) - FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, true, transaction).Await(); + FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyAndClean, transaction).Await(); IListResultFileset[] filesets = db.FilesetsWithBackupVersion.ToArray(); List versionsToDelete = diff --git a/Duplicati/Library/Main/Operation/FilelistProcessor.cs b/Duplicati/Library/Main/Operation/FilelistProcessor.cs index f1d10fdfb..3a7bf334c 100644 --- a/Duplicati/Library/Main/Operation/FilelistProcessor.cs +++ b/Duplicati/Library/Main/Operation/FilelistProcessor.cs @@ -32,6 +32,30 @@ namespace Duplicati.Library.Main.Operation { internal static class FilelistProcessor { + /// + /// The approach to take when verifying the filelist + /// + public enum VerifyMode + { + /// + /// Just verify the list, do not repair partials + /// + VerifyOnly, + /// + /// Verify the list and fail if there are partials + /// + VerifyStrict, + /// + /// Verify the list and repair partials + /// + VerifyAndClean, + /// + /// Force a verification and repair, even if the last operation was successful + /// + VerifyAndCleanForced + } + + /// /// The tag used for logging /// @@ -78,13 +102,13 @@ namespace Duplicati.Library.Main.Operation } } - public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter backendWriter, bool latestVolumesOnly, IDbTransaction transaction) + public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter backendWriter, bool latestVolumesOnly, VerifyMode verifyMode, IDbTransaction transaction) { if (!options.NoBackendverification) { LocalBackupDatabase backupDatabase = new LocalBackupDatabase(database, options); IEnumerable protectedFiles = backupDatabase.GetTemporaryFilelistVolumeNames(latestVolumesOnly, transaction); - await VerifyRemoteList(backend, options, database, backendWriter, protectedFiles).ConfigureAwait(false); + await VerifyRemoteList(backend, options, database, backendWriter, protectedFiles, logErrors: true, verifyMode: verifyMode).ConfigureAwait(false); } } @@ -97,10 +121,11 @@ namespace Duplicati.Library.Main.Operation /// The database to compare with /// The log instance to use /// Filenames that should be exempted from deletion + /// The mode to use for verification /// Disable the logging of errors to prevent spamming the log; exceptions will be thrown regardless - public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter log, IEnumerable protectedFiles = null, bool logErrors = true) + public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter log, IEnumerable protectedFiles, bool logErrors, VerifyMode verifyMode) { - var tp = await RemoteListAnalysis(backend, options, database, log, protectedFiles).ConfigureAwait(false); + var tp = await RemoteListAnalysis(backend, options, database, log, protectedFiles, verifyMode).ConfigureAwait(false); long extraCount = 0; long missingCount = 0; @@ -207,8 +232,18 @@ namespace Duplicati.Library.Main.Operation /// The options used /// The database to compare with /// Filenames that should be exempted from deletion - public static async Task RemoteListAnalysis(IBackendManager backendManager, Options options, LocalDatabase database, IBackendWriter log, IEnumerable protectedFiles) + /// The mode to use for verification + /// The result of the analysis + public static async Task RemoteListAnalysis(IBackendManager backendManager, Options options, LocalDatabase database, IBackendWriter log, IEnumerable protectedFiles, VerifyMode verifyMode) { + // If the last operation completed, no cleanup should be required + if (verifyMode == VerifyMode.VerifyAndClean && !database.UncleanShutdown) + verifyMode = VerifyMode.VerifyStrict; + + // Force cleanup should set the mode to cleanup + if (verifyMode == VerifyMode.VerifyAndCleanForced) + verifyMode = VerifyMode.VerifyAndClean; + var rawlist = await backendManager.ListAsync(CancellationToken.None).ConfigureAwait(false); var lookup = new Dictionary(); protectedFiles = protectedFiles ?? Enumerable.Empty(); @@ -280,7 +315,10 @@ namespace Duplicati.Library.Main.Operation if (remoteFound) { Logging.Log.WriteInformationMessage(LOGTAG, "RemoveUnwantedRemoteFile", "removing remote file listed as {0}: {1}", i.State, i.Name); - await backendManager.DeleteAsync(i.Name, i.Size, true, CancellationToken.None).ConfigureAwait(false); + if (options.Dryrun) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteRemoteFile", "Would delete remote file: {0}", i.Name); + else if (verifyMode != VerifyMode.VerifyOnly) + await backendManager.DeleteAsync(i.Name, i.Size, true, CancellationToken.None).ConfigureAwait(false); } else { @@ -315,6 +353,11 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteInformationMessage(LOGTAG, "KeepIncompleteFile", "keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name); database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Temporary, i.Size, i.Hash, false, new TimeSpan(0), null); } + else if (verifyMode == VerifyMode.VerifyStrict) + { + Logging.Log.WriteErrorMessage(LOGTAG, "UnexpectedUploadingFile", null, "Unexpected file in uploading state: {0}", i.Name); + throw new RemoteListVerificationException($"The remote volume {i.Name} is in uploading state, failing", "UnexpectedUploadingFile"); + } else { Logging.Log.WriteInformationMessage(LOGTAG, "SchedulingMissingFileForDelete", "scheduling missing file for deletion, currently listed as {0}: {1}", i.State, i.Name); @@ -328,10 +371,18 @@ namespace Duplicati.Library.Main.Operation { Logging.Log.WriteInformationMessage(LOGTAG, "KeepIncompleteFile", "keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name); } + else if (verifyMode == VerifyMode.VerifyStrict) + { + Logging.Log.WriteErrorMessage(LOGTAG, "UnexpectedUploadingFile", null, "Unexpected file in uploading state: {0}", i.Name); + throw new RemoteListVerificationException($"The remote volume {i.Name} is in uploading state, failing", "UnexpectedUploadingFile"); + } else { Logging.Log.WriteInformationMessage(LOGTAG, "Remove incomplete file", "removing incomplete remote file listed as {0}: {1}", i.State, i.Name); - await backendManager.DeleteAsync(i.Name, i.Size, true, CancellationToken.None).ConfigureAwait(false); + if (options.Dryrun) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteRemoteFile", "Would delete remote file: {0}", i.Name); + else if (verifyMode != VerifyMode.VerifyOnly) + await backendManager.DeleteAsync(i.Name, i.Size, true, CancellationToken.None).ConfigureAwait(false); } } break; @@ -363,7 +414,15 @@ namespace Duplicati.Library.Main.Operation } // cleanup deleted volumes in DB en block - database.RemoveRemoteVolumes(cleanupRemovedRemoteVolumes, null); + if (verifyMode == VerifyMode.VerifyStrict && cleanupRemovedRemoteVolumes.Count > 0) + throw new RemoteListVerificationException($"The remote volumes {string.Join(", ", cleanupRemovedRemoteVolumes)} are supposed to be deleted, but strict mode is on. Try running the \"repair\" command", "DeleteDuringStrictMode"); + else if (verifyMode == VerifyMode.VerifyAndClean) + { + database.RemoveRemoteVolumes(cleanupRemovedRemoteVolumes, null); + // Clear the flag after we have cleaned up + if (!options.Dryrun) + database.UncleanShutdown = false; + } foreach (var i in missingHash) Logging.Log.WriteWarningMessage(LOGTAG, "MissingRemoteHash", null, "remote file {1} is listed as {0} with size {2} but should be {3}, please verify the sha256 hash \"{4}\"", i.Item2.State, i.Item2.Name, i.Item1, i.Item2.Size, i.Item2.Hash); diff --git a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs index 4212d1bb4..3b2ed5b09 100644 --- a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs @@ -65,7 +65,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesetsInDatabase", "No broken filesets found in database, checking for missing remote files"); - var remotestate = FilelistProcessor.RemoteListAnalysis(backendManager, options, db, result.BackendWriter, null).Await(); + var remotestate = FilelistProcessor.RemoteListAnalysis(backendManager, options, db, result.BackendWriter, null, FilelistProcessor.VerifyMode.VerifyOnly).Await(); if (!remotestate.ParsedVolumes.Any()) throw new UserInformationException("No remote volumes were found, refusing purge", "CannotPurgeWithNoRemoteVolumes"); diff --git a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs index a405d2a2b..3b8dafd69 100644 --- a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs @@ -91,7 +91,7 @@ namespace Duplicati.Library.Main.Operation if (m_options.NoBackendverification) FilelistProcessor.VerifyLocalList(backendManager, db, cancellationToken).Await(); else - FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, null).Await(); + FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, null, logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyAndClean).Await(); } var filesets = db.FilesetTimes.OrderByDescending(x => x.Value).ToArray(); @@ -103,6 +103,10 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Process); m_result.OperationProgressUpdater.UpdateProgress(currentprogress); + // If we crash now, it is possible that the remote storage contains partial files + if (!m_options.Dryrun) + db.UncleanShutdown = true; + // Reverse makes sure we re-write the old versions first foreach (var versionid in versions.Reverse()) { @@ -221,6 +225,8 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(currentprogress); } + if (!m_options.Dryrun) + db.UncleanShutdown = false; if (doCompactStep) { diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index a0b6b5eb5..880ea181d 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -134,7 +134,7 @@ namespace Duplicati.Library.Main.Operation if (db.RepairInProgress) throw new UserInformationException("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsInRepairState"); - var tp = await FilelistProcessor.RemoteListAnalysis(backendManager, m_options, db, m_result.BackendWriter, null).ConfigureAwait(false); + var tp = await FilelistProcessor.RemoteListAnalysis(backendManager, m_options, db, m_result.BackendWriter, null, FilelistProcessor.VerifyMode.VerifyAndCleanForced).ConfigureAwait(false); var buffer = new byte[m_options.Blocksize]; var hashsize = HashFactory.HashSizeBytes(m_options.BlockHashAlgorithm); @@ -297,6 +297,9 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException($"The backup storage destination is missing data files. You can either enable `--rebuild-missing-dblock-files` or run the purge command to remove these files. The following files are missing: {string.Join(", ", missingDblocks.Select(x => x.Name))}", "MissingDblockFiles"); } + if (!m_options.Dryrun && tp.MissingVolumes.Any()) + db.UncleanShutdown = true; + foreach (var n in tp.MissingVolumes) { IDisposable newEntry = null; @@ -496,6 +499,8 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(1); await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false); + if (!m_options.Dryrun) + db.UncleanShutdown = false; db.WriteResults(); } } diff --git a/Duplicati/Library/Main/Operation/RestoreHandler.cs b/Duplicati/Library/Main/Operation/RestoreHandler.cs index 5c7ef2525..901b7d5ad 100644 --- a/Duplicati/Library/Main/Operation/RestoreHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreHandler.cs @@ -317,7 +317,7 @@ namespace Duplicati.Library.Main.Operation if (!m_options.NoBackendverification) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PreRestoreVerify); - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, database, m_result.BackendWriter, false, null).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, database, m_result.BackendWriter, latestVolumesOnly: false, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly, null).ConfigureAwait(false); } // Prepare the block and file list and create the directory structure @@ -431,7 +431,7 @@ namespace Duplicati.Library.Main.Operation if (!m_options.NoBackendverification) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PreRestoreVerify); - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, database, m_result.BackendWriter, false, null).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, database, m_result.BackendWriter, latestVolumesOnly: false, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly, null).ConfigureAwait(false); } //Figure out what files are to be patched, and what blocks are needed diff --git a/Duplicati/Library/Main/Operation/TestHandler.cs b/Duplicati/Library/Main/Operation/TestHandler.cs index f73b5d745..5c2608d69 100644 --- a/Duplicati/Library/Main/Operation/TestHandler.cs +++ b/Duplicati/Library/Main/Operation/TestHandler.cs @@ -57,7 +57,7 @@ namespace Duplicati.Library.Main.Operation Utility.UpdateOptionsFromDb(db, m_options); Utility.VerifyOptionsAndUpdateDatabase(db, m_options); db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, !m_options.DisableFilelistConsistencyChecks, null); - FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_results.BackendWriter, true, null).Await(); + FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_results.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly, null).Await(); DoRun(samples, db, backendManager).Await(); db.WriteResults(); From 7beeb27ade795da064dfc0133c2467ccf4915425 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 6 Mar 2025 14:53:46 +0100 Subject: [PATCH 302/362] Extended tests and fixed a case where a left-over upload could cause errors. --- .../Library/Main/Database/LocalDatabase.cs | 4 +- .../Backup/UploadSyntheticFilelist.cs | 14 +- .../Library/Main/Operation/BackupHandler.cs | 10 +- .../Main/Operation/FilelistProcessor.cs | 37 +-- .../Main/Operation/ListBrokenFilesHandler.cs | 2 +- .../Main/Operation/PurgeFilesHandler.cs | 2 +- .../Library/Main/Operation/RepairHandler.cs | 2 +- Duplicati/UnitTest/DisruptionTests.cs | 226 +++--------------- 8 files changed, 75 insertions(+), 222 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index bca0d32af..55db5b70f 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -742,9 +742,9 @@ AND Fileset.ID NOT IN var opts = GetDbOptions(); if (value) - opts["repair-in-progress"] = "true"; + opts[key] = "true"; else - opts.Remove("repair-in-progress"); + opts.Remove(key); SetDbOptions(opts); } diff --git a/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs b/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs index e788e0469..e78935340 100644 --- a/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs +++ b/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs @@ -77,15 +77,13 @@ namespace Duplicati.Library.Main.Operation.Backup Logging.Log.WriteInformationMessage(LOGTAG, "PreviousBackupFilelistUpload", "Uploading filelist from previous interrupted backup"); var incompleteSet = incompleteFilesets.Last(); - var badIds = from n in incompleteFilesets select n.Key; + var badIds = incompleteFilesets.Select(n => n.Key); - var prevs = (from n in await database.GetFilesetTimesAsync() - where - n.Key < incompleteSet.Key - && - !badIds.Contains(n.Key) - orderby n.Key - select n.Key).ToArray(); + var prevs = (await database.GetFilesetTimesAsync()) + .Where(n => n.Key < incompleteSet.Key && !badIds.Contains(n.Key)) + .OrderBy(n => n.Key) + .Select(n => n.Key) + .ToArray(); var prevId = prevs.Length == 0 ? -1 : prevs.Last(); diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 8f5b86f1d..185fc0797 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -344,7 +344,7 @@ namespace Duplicati.Library.Main.Operation } else { - await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, new string[] { lastTempFilelist }, logErrors: false, verifyMode: FilelistProcessor.VerifyMode.VerifyAndClean).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, [lastTempFilelist], [], logErrors: false, verifyMode: FilelistProcessor.VerifyMode.VerifyAndClean).ConfigureAwait(false); } } catch (RemoteListVerificationException ex) @@ -364,7 +364,7 @@ namespace Duplicati.Library.Main.Operation result.SetDatabase(database); Log.WriteInformationMessage(LOGTAG, "BackendCleanupFinished", "Backend cleanup finished, retrying verification"); - await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, new string[] { lastTempFilelist }, logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, [lastTempFilelist], [], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); } else throw; @@ -504,11 +504,11 @@ namespace Duplicati.Library.Main.Operation } } - private async Task PostBackupVerification(string currentFilelistVolume, IBackendManager backendManager) + private async Task PostBackupVerification(string currentFilelistVolume, string previousTemporaryFilelist, IBackendManager backendManager) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PostBackupVerify); using (new Logging.Timer(LOGTAG, "AfterBackupVerify", "AfterBackupVerify")) - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, m_database, m_result.BackendWriter, new string[] { currentFilelistVolume }, logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, m_database, m_result.BackendWriter, [currentFilelistVolume], [previousTemporaryFilelist], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); await backendManager.WaitForEmptyAsync(m_database, m_transaction, m_taskReader.ProgressToken); long remoteVolumeCount = m_database.GetRemoteVolumes().LongCount(x => x.State == RemoteVolumeState.Verified); @@ -752,7 +752,7 @@ namespace Duplicati.Library.Main.Operation if (m_options.NoBackendverification) await UpdateStorageStatsFromDatabase(m_result, m_database, m_options, backendManager, m_taskReader.ProgressToken).ConfigureAwait(false); else - await PostBackupVerification(filesetvolume.RemoteFilename, backendManager).ConfigureAwait(false); + await PostBackupVerification(filesetvolume.RemoteFilename, lastTempFilelist, backendManager).ConfigureAwait(false); } } diff --git a/Duplicati/Library/Main/Operation/FilelistProcessor.cs b/Duplicati/Library/Main/Operation/FilelistProcessor.cs index d6b79ece3..d836a0416 100644 --- a/Duplicati/Library/Main/Operation/FilelistProcessor.cs +++ b/Duplicati/Library/Main/Operation/FilelistProcessor.cs @@ -108,7 +108,7 @@ namespace Duplicati.Library.Main.Operation { LocalBackupDatabase backupDatabase = new LocalBackupDatabase(database, options); IEnumerable protectedFiles = backupDatabase.GetTemporaryFilelistVolumeNames(latestVolumesOnly, transaction); - await VerifyRemoteList(backend, options, database, backendWriter, protectedFiles, logErrors: true, verifyMode: verifyMode).ConfigureAwait(false); + await VerifyRemoteList(backend, options, database, backendWriter, protectedFiles, [], logErrors: true, verifyMode: verifyMode).ConfigureAwait(false); } } @@ -121,11 +121,12 @@ namespace Duplicati.Library.Main.Operation /// The database to compare with /// The log instance to use /// Filenames that should be exempted from deletion + /// Filenames that should be exempted from strict verification /// The mode to use for verification /// Disable the logging of errors to prevent spamming the log; exceptions will be thrown regardless - public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter log, IEnumerable protectedFiles, bool logErrors, VerifyMode verifyMode) + public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter log, IEnumerable protectedFiles, IEnumerable strictExcemptFiles, bool logErrors, VerifyMode verifyMode) { - var tp = await RemoteListAnalysis(backend, options, database, log, protectedFiles, verifyMode).ConfigureAwait(false); + var tp = await RemoteListAnalysis(backend, options, database, log, protectedFiles, strictExcemptFiles, verifyMode).ConfigureAwait(false); long extraCount = 0; long missingCount = 0; @@ -232,9 +233,10 @@ namespace Duplicati.Library.Main.Operation /// The options used /// The database to compare with /// Filenames that should be exempted from deletion + /// Filenames that should be exempted from strict verification /// The mode to use for verification /// The result of the analysis - public static async Task RemoteListAnalysis(IBackendManager backendManager, Options options, LocalDatabase database, IBackendWriter log, IEnumerable protectedFiles, VerifyMode verifyMode) + public static async Task RemoteListAnalysis(IBackendManager backendManager, Options options, LocalDatabase database, IBackendWriter log, IEnumerable protectedFiles, IEnumerable strictExcemptFiles, VerifyMode verifyMode) { // If the last operation completed, no cleanup should be required if (verifyMode == VerifyMode.VerifyAndClean && !database.UncleanShutdown) @@ -246,7 +248,8 @@ namespace Duplicati.Library.Main.Operation var rawlist = await backendManager.ListAsync(CancellationToken.None).ConfigureAwait(false); var lookup = new Dictionary(); - protectedFiles = protectedFiles ?? Enumerable.Empty(); + protectedFiles = protectedFiles ?? []; + strictExcemptFiles = strictExcemptFiles ?? []; var remotelist = (from n in rawlist let p = Volumes.VolumeBase.ParseFilename(n) @@ -284,7 +287,8 @@ namespace Duplicati.Library.Main.Operation var missing = new List(); var missingHash = new List>(); - var cleanupRemovedRemoteVolumes = new HashSet(); + var temporaryAndDeletedVolumes = new HashSet(); + var missingUploadingVolumes = new HashSet(); foreach (var e in database.DuplicateRemoteVolumes()) { @@ -297,8 +301,7 @@ namespace Duplicati.Library.Main.Operation var locallist = database.GetRemoteVolumes(); foreach (var i in locallist) { - Volumes.IParsedVolume r; - var remoteFound = lookup.TryGetValue(i.Name, out r); + var remoteFound = lookup.TryGetValue(i.Name, out var r); var correctSize = remoteFound && i.Size >= 0 && (i.Size == r.File.Size || r.File.Size < 0); lookup.Remove(i.Name); @@ -336,7 +339,7 @@ namespace Duplicati.Library.Main.Operation else { Logging.Log.WriteInformationMessage(LOGTAG, "RemoteUnwantedMissingFile", "removing file listed as {0}: {1}", i.State, i.Name); - cleanupRemovedRemoteVolumes.Add(i.Name); + temporaryAndDeletedVolumes.Add(i.Name); } } } @@ -354,15 +357,15 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteInformationMessage(LOGTAG, "KeepIncompleteFile", "keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name); database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Temporary, i.Size, i.Hash, false, new TimeSpan(0), null); } - else if (verifyMode == VerifyMode.VerifyStrict) + else if (verifyMode == VerifyMode.VerifyStrict && !strictExcemptFiles.Any(pf => pf == i.Name)) { Logging.Log.WriteErrorMessage(LOGTAG, "UnexpectedUploadingFile", null, "Unexpected file in uploading state: {0}", i.Name); - throw new RemoteListVerificationException($"The remote volume {i.Name} is in uploading state, failing", "UnexpectedUploadingFile"); + throw new RemoteListVerificationException($"The missing remote volume {i.Name} is in uploading state and strict mode is on", "UnexpectedUploadingFile"); } else { Logging.Log.WriteInformationMessage(LOGTAG, "SchedulingMissingFileForDelete", "scheduling missing file for deletion, currently listed as {0}: {1}", i.State, i.Name); - cleanupRemovedRemoteVolumes.Add(i.Name); + missingUploadingVolumes.Add(i.Name); database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Deleting, i.Size, i.Hash, false, TimeSpan.FromHours(2), null); } } @@ -372,10 +375,10 @@ namespace Duplicati.Library.Main.Operation { Logging.Log.WriteInformationMessage(LOGTAG, "KeepIncompleteFile", "keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name); } - else if (verifyMode == VerifyMode.VerifyStrict) + else if (verifyMode == VerifyMode.VerifyStrict && !strictExcemptFiles.Any(pf => pf == i.Name)) { Logging.Log.WriteErrorMessage(LOGTAG, "UnexpectedUploadingFile", null, "Unexpected file in uploading state: {0}", i.Name); - throw new RemoteListVerificationException($"The remote volume {i.Name} is in uploading state, failing", "UnexpectedUploadingFile"); + throw new RemoteListVerificationException($"The remote volume {i.Name} is in uploading state, partially uploaded, and strict mode is on", "UnexpectedUploadingFile"); } else { @@ -415,11 +418,11 @@ namespace Duplicati.Library.Main.Operation } // cleanup deleted volumes in DB en block - if (verifyMode == VerifyMode.VerifyStrict && cleanupRemovedRemoteVolumes.Count > 0) - throw new RemoteListVerificationException($"The remote volumes {string.Join(", ", cleanupRemovedRemoteVolumes)} are supposed to be deleted, but strict mode is on. Try running the \"repair\" command", "DeleteDuringStrictMode"); + if (verifyMode == VerifyMode.VerifyStrict && missingUploadingVolumes.Count > 0) + throw new RemoteListVerificationException($"The remote volumes {string.Join(", ", missingUploadingVolumes)} are supposed to be deleted, but strict mode is on. Try running the \"repair\" command", "DeleteDuringStrictMode"); else if (verifyMode == VerifyMode.VerifyAndClean) { - database.RemoveRemoteVolumes(cleanupRemovedRemoteVolumes, null); + database.RemoveRemoteVolumes(missingUploadingVolumes.Concat(temporaryAndDeletedVolumes), null); // Clear the flag after we have cleaned up if (!options.Dryrun) database.UncleanShutdown = false; diff --git a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs index 3b2ed5b09..0df3125b6 100644 --- a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs @@ -65,7 +65,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesetsInDatabase", "No broken filesets found in database, checking for missing remote files"); - var remotestate = FilelistProcessor.RemoteListAnalysis(backendManager, options, db, result.BackendWriter, null, FilelistProcessor.VerifyMode.VerifyOnly).Await(); + var remotestate = FilelistProcessor.RemoteListAnalysis(backendManager, options, db, result.BackendWriter, null, null, FilelistProcessor.VerifyMode.VerifyOnly).Await(); if (!remotestate.ParsedVolumes.Any()) throw new UserInformationException("No remote volumes were found, refusing purge", "CannotPurgeWithNoRemoteVolumes"); diff --git a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs index 3b8dafd69..d113fe905 100644 --- a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs @@ -91,7 +91,7 @@ namespace Duplicati.Library.Main.Operation if (m_options.NoBackendverification) FilelistProcessor.VerifyLocalList(backendManager, db, cancellationToken).Await(); else - FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, null, logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyAndClean).Await(); + FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, null, null, logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyAndClean).Await(); } var filesets = db.FilesetTimes.OrderByDescending(x => x.Value).ToArray(); diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 880ea181d..2e77e378c 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -134,7 +134,7 @@ namespace Duplicati.Library.Main.Operation if (db.RepairInProgress) throw new UserInformationException("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsInRepairState"); - var tp = await FilelistProcessor.RemoteListAnalysis(backendManager, m_options, db, m_result.BackendWriter, null, FilelistProcessor.VerifyMode.VerifyAndCleanForced).ConfigureAwait(false); + var tp = await FilelistProcessor.RemoteListAnalysis(backendManager, m_options, db, m_result.BackendWriter, null, null, FilelistProcessor.VerifyMode.VerifyAndCleanForced).ConfigureAwait(false); var buffer = new byte[m_options.Blocksize]; var hashsize = HashFactory.HashSizeBytes(m_options.BlockHashAlgorithm); diff --git a/Duplicati/UnitTest/DisruptionTests.cs b/Duplicati/UnitTest/DisruptionTests.cs index 32c053fe6..f3c63f968 100644 --- a/Duplicati/UnitTest/DisruptionTests.cs +++ b/Duplicati/UnitTest/DisruptionTests.cs @@ -887,102 +887,25 @@ namespace Duplicati.UnitTest [Category("Disruption")] public void TestFailedDblockUploadWithOperationCancellation() { - var testopts = TestOptions; - testopts["number-of-retries"] = "1"; - testopts["dblock-size"] = "10mb"; - - // Make a base backup - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) - { - IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER }); - Assert.AreEqual(0, backupResults.Errors.Count()); - Assert.AreEqual(0, backupResults.Warnings.Count()); - } - - // Make a new backup that fails uploading a dblock file - ModifySourceFiles(); - - // Deterministic error backend - Library.DynamicLoader.BackendLoader.AddBackend(new DeterministicErrorBackend()); - var failtarget = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER; - var uploadCount = 0; - var failedFile = string.Empty; - - // Fail the dlist upload - DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) => - { - if (action == DeterministicErrorBackend.BackendAction.PutBefore && remotename.Contains(".dblock.")) - { - // Fail once with a cancellation - if (Interlocked.Increment(ref uploadCount) == 1) - { - failedFile = remotename; - throw new OperationCanceledException(); - } - } - - return false; - }; - - using (var c = new Library.Main.Controller(failtarget, testopts, null)) - { - var sink = new LogSink(); - c.AppendSink(sink); - var res = c.Backup(new string[] { DATAFOLDER }); - if (uploadCount == 0) - Assert.Fail("Upload count was not incremented"); - - Assert.AreEqual(2, c.List().Filesets.Count()); - Assert.AreEqual(7 * 2 + 1, sink.CompletedFiles.Count); // 7 completed (dblock + dindex) + 1 dlist - Assert.AreEqual(7 * 2 + 2, sink.StartedFiles.Count); // 1 retry - Assert.AreEqual(1, res.BackendStatistics.RetryAttempts); - } - - ModifySourceFiles(); - // Create a regular backup - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) - { - IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER }); - Assert.AreEqual(0, backupResults.Errors.Count()); - Assert.AreEqual(0, backupResults.Warnings.Count()); - } - - // Verify that all is in order - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null)) - { - var r = c.Test(long.MaxValue); - Assert.AreEqual(0, r.Errors.Count()); - Assert.AreEqual(0, r.Warnings.Count()); - Assert.IsFalse(r.Verifications.Any(p => p.Value.Any())); - } - - // Test that we can recreate - var recreatedDatabaseFile = Path.Combine(BASEFOLDER, "recreated-database.sqlite"); - if (File.Exists(recreatedDatabaseFile)) - File.Delete(recreatedDatabaseFile); - - testopts["dbpath"] = recreatedDatabaseFile; - - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) - { - IRepairResults repairResults = c.Repair(); - Assert.AreEqual(0, repairResults.Errors.Count()); - Assert.AreEqual(0, repairResults.Warnings.Count()); - } - - // Check that we have 3 versions - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) - { - IListResults listResults = c.List(); - Assert.AreEqual(0, listResults.Errors.Count()); - Assert.AreEqual(0, listResults.Warnings.Count()); - Assert.AreEqual(3, listResults.Filesets.Count()); - } + TestFailedUploadWithOperationCancellation(".dblock."); } + [Test] [Category("Disruption")] public void TestFailedDindexUploadWithOperationCancellation() + { + TestFailedUploadWithOperationCancellation(".dindex."); + } + + [Test] + [Category("Disruption")] + public void TestFailedDlistUploadWithOperationCancellation() + { + TestFailedUploadWithOperationCancellation(".dlist."); + } + + private void TestFailedUploadWithOperationCancellation(string filefragment) { var testopts = TestOptions; testopts["number-of-retries"] = "1"; @@ -1003,19 +926,16 @@ namespace Duplicati.UnitTest Library.DynamicLoader.BackendLoader.AddBackend(new DeterministicErrorBackend()); var failtarget = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER; var uploadCount = 0; - var failedFile = string.Empty; + var failstep = DeterministicErrorBackend.BackendAction.PutBefore; - // Fail the dlist upload + // Fail the upload before DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) => { - if (action == DeterministicErrorBackend.BackendAction.PutBefore && remotename.Contains(".dindex.")) + if (action == failstep && remotename.Contains(filefragment)) { // Fail once with a cancellation if (Interlocked.Increment(ref uploadCount) == 1) - { - failedFile = remotename; throw new OperationCanceledException(); - } } return false; @@ -1030,6 +950,27 @@ namespace Duplicati.UnitTest Assert.Fail("Upload count was not incremented"); Assert.AreEqual(2, c.List().Filesets.Count()); + Assert.AreEqual(2, Directory.GetFiles(TARGETFOLDER, "*.dlist.*", SearchOption.TopDirectoryOnly).Count()); + Assert.AreEqual(7 * 2 + 1, sink.CompletedFiles.Count); // 7 completed (dblock + dindex) + 1 dlist + Assert.AreEqual(7 * 2 + 2, sink.StartedFiles.Count); // 1 retry + Assert.AreEqual(sink.CompletedFiles.Count(x => x.Contains(".dblock.")), sink.CompletedFiles.Count(x => x.Contains(".dindex."))); + Assert.AreEqual(1, res.BackendStatistics.RetryAttempts); + } + + uploadCount = 0; + failstep = DeterministicErrorBackend.BackendAction.PutAfter; + + ModifySourceFiles(); + using (var c = new Library.Main.Controller(failtarget, testopts, null)) + { + var sink = new LogSink(); + c.AppendSink(sink); + var res = c.Backup(new string[] { DATAFOLDER }); + if (uploadCount == 0) + Assert.Fail("Upload count was not incremented"); + + Assert.AreEqual(3, c.List().Filesets.Count()); + Assert.AreEqual(3, Directory.GetFiles(TARGETFOLDER, "*.dlist.*", SearchOption.TopDirectoryOnly).Count()); Assert.AreEqual(7 * 2 + 1, sink.CompletedFiles.Count); // 7 completed (dblock + dindex) + 1 dlist Assert.AreEqual(7 * 2 + 2, sink.StartedFiles.Count); // 1 retry Assert.AreEqual(sink.CompletedFiles.Count(x => x.Contains(".dblock.")), sink.CompletedFiles.Count(x => x.Contains(".dindex."))); @@ -1074,96 +1015,7 @@ namespace Duplicati.UnitTest IListResults listResults = c.List(); Assert.AreEqual(0, listResults.Errors.Count()); Assert.AreEqual(0, listResults.Warnings.Count()); - Assert.AreEqual(3, listResults.Filesets.Count()); - } - } - - [Test] - [Category("Disruption")] - public void TestFailedDlistUploadWithOperationCancellation() - { - var testopts = TestOptions; - testopts["number-of-retries"] = "1"; - testopts["dblock-size"] = "10mb"; - - // Make a base backup - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) - { - IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER }); - Assert.AreEqual(0, backupResults.Errors.Count()); - Assert.AreEqual(0, backupResults.Warnings.Count()); - } - - // Make a new backup that fails uploading a dblock file - ModifySourceFiles(); - - // Deterministic error backend - Library.DynamicLoader.BackendLoader.AddBackend(new DeterministicErrorBackend()); - var failtarget = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER; - var uploadCount = 0; - - // Fail the dlist upload - DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) => - { - if (action == DeterministicErrorBackend.BackendAction.PutBefore && remotename.Contains(".dlist.")) - { - if (Interlocked.Increment(ref uploadCount) == 1) - throw new OperationCanceledException(); - } - - return false; - }; - - using (var c = new Library.Main.Controller(failtarget, testopts, null)) - { - var res = c.Backup(new string[] { DATAFOLDER }); - var dlistCount = Directory.GetFiles(TARGETFOLDER, "*.dlist.*", SearchOption.TopDirectoryOnly).Count(); - if (uploadCount == 0) - Assert.Fail("Upload count was not incremented"); - Assert.AreEqual(2, c.List().Filesets.Count()); - Assert.AreEqual(2, dlistCount); - Assert.AreEqual(1, res.BackendStatistics.RetryAttempts); - } - - ModifySourceFiles(); - // Create a regular backup - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) - { - IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER }); - Assert.AreEqual(0, backupResults.Errors.Count()); - Assert.AreEqual(0, backupResults.Warnings.Count()); - } - - // Verify that all is in order - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null)) - { - var r = c.Test(long.MaxValue); - Assert.AreEqual(0, r.Errors.Count()); - Assert.AreEqual(0, r.Warnings.Count()); - Assert.IsFalse(r.Verifications.Any(p => p.Value.Any())); - } - - // Test that we can recreate - var recreatedDatabaseFile = Path.Combine(BASEFOLDER, "recreated-database.sqlite"); - if (File.Exists(recreatedDatabaseFile)) - File.Delete(recreatedDatabaseFile); - - testopts["dbpath"] = recreatedDatabaseFile; - - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) - { - IRepairResults repairResults = c.Repair(); - Assert.AreEqual(0, repairResults.Errors.Count()); - Assert.AreEqual(0, repairResults.Warnings.Count()); - } - - // Check that we have 3 versions - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) - { - IListResults listResults = c.List(); - Assert.AreEqual(0, listResults.Errors.Count()); - Assert.AreEqual(0, listResults.Warnings.Count()); - Assert.AreEqual(3, listResults.Filesets.Count()); + Assert.AreEqual(4, listResults.Filesets.Count()); } } } From ca236f03e3967ed7825bc22d9409710874521921 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 6 Mar 2025 14:56:35 +0100 Subject: [PATCH 303/362] Renamed flag to more precisely describe the purpose --- Duplicati/Library/Main/Database/LocalDatabase.cs | 6 +++--- Duplicati/Library/Main/Operation/BackupHandler.cs | 4 ++-- Duplicati/Library/Main/Operation/CompactHandler.cs | 4 ++-- Duplicati/Library/Main/Operation/FilelistProcessor.cs | 4 ++-- Duplicati/Library/Main/Operation/PurgeFilesHandler.cs | 4 ++-- Duplicati/Library/Main/Operation/RepairHandler.cs | 4 ++-- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 55db5b70f..efa59a460 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -761,10 +761,10 @@ AND Fileset.ID NOT IN set => UpdateDbOption("partially-recreated", value); } - public bool UncleanShutdown + public bool TerminatedWithActiveUploads { - get => GetDbOptions().ContainsKey("unclean-shutdown"); - set => UpdateDbOption("unclean-shutdown", value); + get => GetDbOptions().ContainsKey("terminated-with-active-uploads"); + set => UpdateDbOption("terminated-with-active-uploads", value); } public void SetDbOptions(IDictionary options, System.Data.IDbTransaction transaction = null) diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 185fc0797..9c8262ed4 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -618,7 +618,7 @@ namespace Duplicati.Library.Main.Operation { // If we crash now, there is a chance we leave behind partial remote files if (!m_options.Dryrun) - database.UncleanShutdown = true; + database.TerminatedWithActiveUploads = true; using (m_database = database) using (var db = new Backup.BackupDatabase(m_database, m_options)) @@ -699,7 +699,7 @@ namespace Duplicati.Library.Main.Operation var lastVolumeSize = await FlushBackend(m_database, m_transaction, m_result, backendManager).ConfigureAwait(false); if (!m_options.Dryrun) - database.UncleanShutdown = false; + database.TerminatedWithActiveUploads = false; // Make sure we have the database up-to-date await db.CommitTransactionAsync("CommitAfterUpload", false); diff --git a/Duplicati/Library/Main/Operation/CompactHandler.cs b/Duplicati/Library/Main/Operation/CompactHandler.cs index abd279869..30ae68618 100644 --- a/Duplicati/Library/Main/Operation/CompactHandler.cs +++ b/Duplicati/Library/Main/Operation/CompactHandler.cs @@ -148,7 +148,7 @@ namespace Duplicati.Library.Main.Operation { // If we crash now, we may leave partial files if (!m_options.Dryrun) - db.UncleanShutdown = true; + db.TerminatedWithActiveUploads = true; newvolindex?.StartVolume(newvol.RemoteFilename); List volumesToDownload = []; @@ -247,7 +247,7 @@ namespace Duplicati.Library.Main.Operation // The remainder of the operation cannot leave partial files if (!m_options.Dryrun) - db.UncleanShutdown = false; + db.TerminatedWithActiveUploads = false; } else { diff --git a/Duplicati/Library/Main/Operation/FilelistProcessor.cs b/Duplicati/Library/Main/Operation/FilelistProcessor.cs index d836a0416..2fb1690b5 100644 --- a/Duplicati/Library/Main/Operation/FilelistProcessor.cs +++ b/Duplicati/Library/Main/Operation/FilelistProcessor.cs @@ -239,7 +239,7 @@ namespace Duplicati.Library.Main.Operation public static async Task RemoteListAnalysis(IBackendManager backendManager, Options options, LocalDatabase database, IBackendWriter log, IEnumerable protectedFiles, IEnumerable strictExcemptFiles, VerifyMode verifyMode) { // If the last operation completed, no cleanup should be required - if (verifyMode == VerifyMode.VerifyAndClean && !database.UncleanShutdown) + if (verifyMode == VerifyMode.VerifyAndClean && !database.TerminatedWithActiveUploads) verifyMode = VerifyMode.VerifyStrict; // Force cleanup should set the mode to cleanup @@ -425,7 +425,7 @@ namespace Duplicati.Library.Main.Operation database.RemoveRemoteVolumes(missingUploadingVolumes.Concat(temporaryAndDeletedVolumes), null); // Clear the flag after we have cleaned up if (!options.Dryrun) - database.UncleanShutdown = false; + database.TerminatedWithActiveUploads = false; } foreach (var i in missingHash) diff --git a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs index d113fe905..16506287d 100644 --- a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs @@ -105,7 +105,7 @@ namespace Duplicati.Library.Main.Operation // If we crash now, it is possible that the remote storage contains partial files if (!m_options.Dryrun) - db.UncleanShutdown = true; + db.TerminatedWithActiveUploads = true; // Reverse makes sure we re-write the old versions first foreach (var versionid in versions.Reverse()) @@ -226,7 +226,7 @@ namespace Duplicati.Library.Main.Operation } if (!m_options.Dryrun) - db.UncleanShutdown = false; + db.TerminatedWithActiveUploads = false; if (doCompactStep) { diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 2e77e378c..7dbc1e6ec 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -298,7 +298,7 @@ namespace Duplicati.Library.Main.Operation } if (!m_options.Dryrun && tp.MissingVolumes.Any()) - db.UncleanShutdown = true; + db.TerminatedWithActiveUploads = true; foreach (var n in tp.MissingVolumes) { @@ -500,7 +500,7 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(1); await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - db.UncleanShutdown = false; + db.TerminatedWithActiveUploads = false; db.WriteResults(); } } From 774e468287b9a4eb30d372520caca77d79732dc6 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 6 Mar 2025 15:12:38 +0100 Subject: [PATCH 304/362] Placed modify correctly --- Duplicati/UnitTest/DisruptionTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Duplicati/UnitTest/DisruptionTests.cs b/Duplicati/UnitTest/DisruptionTests.cs index 083a6bab9..e98bde24e 100644 --- a/Duplicati/UnitTest/DisruptionTests.cs +++ b/Duplicati/UnitTest/DisruptionTests.cs @@ -237,6 +237,8 @@ namespace Duplicati.UnitTest secondBackupTime = c.List().Filesets.First().Time; } + ModifySourceFiles(); + // Run a partial backup. DateTime thirdBackupTime; using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) @@ -264,8 +266,6 @@ namespace Duplicati.UnitTest Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[0].IsFullBackup); } - ModifySourceFiles(); - // Run another partial backup. We will then verify that a full backup is retained // even when all the "recent" backups are partial. using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) From e3727d8f18e047bf5f0734c5245038d8faea4c88 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 6 Mar 2025 15:22:57 +0100 Subject: [PATCH 305/362] Finding list of modified files to trace the error --- Duplicati/UnitTest/DisruptionTests.cs | 30 +++++++++++++-------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Duplicati/UnitTest/DisruptionTests.cs b/Duplicati/UnitTest/DisruptionTests.cs index e98bde24e..8a7d99eab 100644 --- a/Duplicati/UnitTest/DisruptionTests.cs +++ b/Duplicati/UnitTest/DisruptionTests.cs @@ -51,11 +51,12 @@ namespace Duplicati.UnitTest } } - private async Task RunPartialBackup(Controller controller) + private async Task<(IBackupResults Result, List ModifiedFiles)> RunPartialBackup(Controller controller) { this.ModifySourceFiles(); var stopped = new TaskCompletionSource(); + var modifiedFiles = new List(); controller.OnOperationStarted = r => { var pv = r as ITaskControlProvider; @@ -64,6 +65,7 @@ namespace Duplicati.UnitTest #if DEBUG pv.TaskControl.TestMethodCallback = (path) => { + modifiedFiles.Add(path); if (path.EndsWith(this.fileSizes[2] + "MB")) { Thread.Sleep(500); @@ -81,7 +83,7 @@ namespace Duplicati.UnitTest if (t != stopped.Task) throw new Exception("Backup task completed before we could stop it"); - return await backupTask.ConfigureAwait(false); + return (await backupTask.ConfigureAwait(false), modifiedFiles); } [SetUp] @@ -122,7 +124,7 @@ namespace Duplicati.UnitTest // Run a partial backup. using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { - IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false); + (var backupResults, _) = await this.RunPartialBackup(c).ConfigureAwait(false); Assert.AreEqual(0, backupResults.Errors.Count()); Assert.AreEqual(1, backupResults.Warnings.Count()); } @@ -237,13 +239,11 @@ namespace Duplicati.UnitTest secondBackupTime = c.List().Filesets.First().Time; } - ModifySourceFiles(); - // Run a partial backup. DateTime thirdBackupTime; using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { - IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false); + (var backupResults, _) = await this.RunPartialBackup(c).ConfigureAwait(false); Assert.AreEqual(0, backupResults.Errors.Count()); Assert.AreEqual(1, backupResults.Warnings.Count()); thirdBackupTime = c.List().Filesets.First().Time; @@ -270,7 +270,7 @@ namespace Duplicati.UnitTest // even when all the "recent" backups are partial. using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { - IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false); + (var backupResults, _) = await this.RunPartialBackup(c).ConfigureAwait(false); Assert.AreEqual(0, backupResults.Errors.Count()); Assert.AreEqual(1, backupResults.Warnings.Count()); DateTime fourthBackupTime = c.List().Filesets.First().Time; @@ -322,7 +322,7 @@ namespace Duplicati.UnitTest // Run a partial backup. using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { - IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false); + (var backupResults, _) = await this.RunPartialBackup(c).ConfigureAwait(false); Assert.AreEqual(0, backupResults.Errors.Count()); Assert.AreEqual(1, backupResults.Warnings.Count()); } @@ -330,7 +330,7 @@ namespace Duplicati.UnitTest // Run a partial backup. using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { - IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false); + (var backupResults, _) = await this.RunPartialBackup(c).ConfigureAwait(false); Assert.AreEqual(0, backupResults.Errors.Count()); Assert.AreEqual(1, backupResults.Warnings.Count()); } @@ -350,7 +350,7 @@ namespace Duplicati.UnitTest using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { options["keep-versions"] = "2"; - IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false); + (var backupResults, _) = await this.RunPartialBackup(c).ConfigureAwait(false); Assert.AreEqual(0, backupResults.Errors.Count()); Assert.AreEqual(1, backupResults.Warnings.Count()); DateTime fifthBackupTime = c.List().Filesets.First().Time; @@ -416,7 +416,7 @@ namespace Duplicati.UnitTest // Run a partial backup. using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { - IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false); + (var backupResults, _) = await this.RunPartialBackup(c).ConfigureAwait(false); Assert.AreEqual(0, backupResults.Errors.Count()); Assert.AreEqual(1, backupResults.Warnings.Count()); @@ -480,7 +480,7 @@ namespace Duplicati.UnitTest DateTime thirdBackupTime; using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { - IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false); + (var backupResults, _) = await this.RunPartialBackup(c).ConfigureAwait(false); Assert.AreEqual(0, backupResults.Errors.Count()); Assert.AreEqual(1, backupResults.Warnings.Count()); thirdBackupTime = c.List().Filesets.First().Time; @@ -596,14 +596,14 @@ namespace Duplicati.UnitTest Assert.AreEqual(BackupType.FULL_BACKUP, c.List().Filesets.Single(x => x.Version == 0).IsFullBackup); } - ModifySourceFiles(); - // Run a partial backup. using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { - IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false); + (var backupResults, var modified) = await this.RunPartialBackup(c).ConfigureAwait(false); Assert.AreEqual(0, backupResults.Errors.Count()); Assert.AreEqual(1, backupResults.Warnings.Count()); + if (backupResults.ModifiedFiles == 0) + throw new Exception($"No files were modified, likely the stop was issued too early, list is: {string.Join(", ", modified)}"); Assert.That(backupResults.ModifiedFiles, Is.GreaterThan(0), "No files were added, likely the stop was issued too early"); Assert.That(backupResults.ModifiedFiles, Is.LessThan(fileSizes.Length), "All files were added, likely the stop was issued too late"); From d75e690495bc7609532e3d675fcf3639389fdff4 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 6 Mar 2025 18:26:56 +0100 Subject: [PATCH 306/362] Fixed not being dependent on filesystem order --- Duplicati/UnitTest/DisruptionTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/UnitTest/DisruptionTests.cs b/Duplicati/UnitTest/DisruptionTests.cs index 8a7d99eab..6e27637bf 100644 --- a/Duplicati/UnitTest/DisruptionTests.cs +++ b/Duplicati/UnitTest/DisruptionTests.cs @@ -66,7 +66,7 @@ namespace Duplicati.UnitTest pv.TaskControl.TestMethodCallback = (path) => { modifiedFiles.Add(path); - if (path.EndsWith(this.fileSizes[2] + "MB")) + if (modifiedFiles.Count == 3) { Thread.Sleep(500); controller.Stop(); From b6f085f440cb63a2729dba26d0c710a4cab85c86 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 6 Mar 2025 19:17:21 +0100 Subject: [PATCH 307/362] Fix for wrong tests being executed --- .github/workflows/backendtests.yml | 144 ++++++++++++++--------------- 1 file changed, 72 insertions(+), 72 deletions(-) diff --git a/.github/workflows/backendtests.yml b/.github/workflows/backendtests.yml index bb5b0765f..9cd5239df 100644 --- a/.github/workflows/backendtests.yml +++ b/.github/workflows/backendtests.yml @@ -484,76 +484,76 @@ jobs: --logger:"console;verbosity=detailed" LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln test_b2: - needs: check_secrets - if: needs.check_secrets.outputs.b2_secrets_available == 'true' - name: Backblaze B2 Tests - runs-on: "${{ matrix.os }}" - strategy: - max-parallel: 1 - fail-fast: false - matrix: - os: - - ubuntu-latest - - windows-latest - - macos-latest - steps: - - name: Set up .NET - uses: actions/setup-dotnet@v4 - with: - dotnet-version: 8.x - - name: Checkout source - uses: actions/checkout@v4 - - name: Restore NuGet dependencies - run: >- - dotnet restore - LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln - - name: Build project - run: >- - dotnet build --no-restore - LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln - - name: Run B2 tests - env: - TESTCREDENTIAL_B2_BUCKET: "${{ secrets.TESTCREDENTIAL_B2_BUCKET }}" - TESTCREDENTIAL_B2_FOLDER: "${{ secrets.TESTCREDENTIAL_B2_FOLDER }}" - TESTCREDENTIAL_B2_USERNAME: "${{ secrets.TESTCREDENTIAL_B2_USERNAME }}" - TESTCREDENTIAL_B2_PASSWORD: "${{ secrets.TESTCREDENTIAL_B2_PASSWORD }}" - run: >- - dotnet test --no-build --filter="ClassName~B2Tests" - --logger:"console;verbosity=detailed" - LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln + needs: check_secrets + if: needs.check_secrets.outputs.b2_secrets_available == 'true' + name: Backblaze B2 Tests + runs-on: "${{ matrix.os }}" + strategy: + max-parallel: 1 + fail-fast: false + matrix: + os: + - ubuntu-latest + - windows-latest + - macos-latest + steps: + - name: Set up .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 8.x + - name: Checkout source + uses: actions/checkout@v4 + - name: Restore NuGet dependencies + run: >- + dotnet restore + LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln + - name: Build project + run: >- + dotnet build --no-restore + LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln + - name: Run B2 tests + env: + TESTCREDENTIAL_B2_BUCKET: "${{ secrets.TESTCREDENTIAL_B2_BUCKET }}" + TESTCREDENTIAL_B2_FOLDER: "${{ secrets.TESTCREDENTIAL_B2_FOLDER }}" + TESTCREDENTIAL_B2_USERNAME: "${{ secrets.TESTCREDENTIAL_B2_USERNAME }}" + TESTCREDENTIAL_B2_PASSWORD: "${{ secrets.TESTCREDENTIAL_B2_PASSWORD }}" + run: >- + dotnet test --no-build --filter="ClassName~B2Tests" + --logger:"console;verbosity=detailed" + LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln test_box: - needs: check_secrets - if: needs.check_secrets.outputs.box_secrets_available == 'true' - name: Box.com Tests - runs-on: "${{ matrix.os }}" - strategy: - max-parallel: 1 - fail-fast: false - matrix: - os: - - ubuntu-latest - - windows-latest - - macos-latest - steps: - - name: Set up .NET - uses: actions/setup-dotnet@v4 - with: - dotnet-version: 8.x - - name: Checkout source - uses: actions/checkout@v4 - - name: Restore NuGet dependencies - run: >- - dotnet restore - LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln - - name: Build project - run: >- - dotnet build --no-restore - LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln - - name: Run Box.com tests - env: - TESTCREDENTIAL_BOX_FOLDER: "${{ secrets.TESTCREDENTIAL_BOX_FOLDER }}" - TESTCREDENTIAL_BOX_AUTHID: "${{ secrets.TESTCREDENTIAL_BOX_AUTHID }}" - run: >- - dotnet test --no-build --filter="ClassName~BoxTests" - --logger:"console;verbosity=detailed" - LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln \ No newline at end of file + needs: check_secrets + if: needs.check_secrets.outputs.box_secrets_available == 'true' + name: Box.com Tests + runs-on: "${{ matrix.os }}" + strategy: + max-parallel: 1 + fail-fast: false + matrix: + os: + - ubuntu-latest + - windows-latest + - macos-latest + steps: + - name: Set up .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 8.x + - name: Checkout source + uses: actions/checkout@v4 + - name: Restore NuGet dependencies + run: >- + dotnet restore + LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln + - name: Build project + run: >- + dotnet build --no-restore + LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln + - name: Run Box.com tests + env: + TESTCREDENTIAL_BOX_FOLDER: "${{ secrets.TESTCREDENTIAL_BOX_FOLDER }}" + TESTCREDENTIAL_BOX_AUTHID: "${{ secrets.TESTCREDENTIAL_BOX_AUTHID }}" + run: >- + dotnet test --no-build --filter="ClassName~Box.BoxTests" + --logger:"console;verbosity=detailed" + LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln From 57335aadaf54bdca58a591209a5a1b3597d9e66a Mon Sep 17 00:00:00 2001 From: "Marcelo C." Date: Thu, 6 Mar 2025 16:40:09 -0300 Subject: [PATCH 308/362] Fix to parameter name Param is authid not auth-id --- LiveTests/Duplicati.Backend.Tests/Box/BoxTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LiveTests/Duplicati.Backend.Tests/Box/BoxTests.cs b/LiveTests/Duplicati.Backend.Tests/Box/BoxTests.cs index e4bd6e6b0..d393a2118 100644 --- a/LiveTests/Duplicati.Backend.Tests/Box/BoxTests.cs +++ b/LiveTests/Duplicati.Backend.Tests/Box/BoxTests.cs @@ -35,7 +35,7 @@ public sealed class BoxTests : BaseTest var exitCode = CommandLine.BackendTester.Program.Main( new[] { - $"box://{Environment.GetEnvironmentVariable("TESTCREDENTIAL_BOX_FOLDER")}/?auth-id={Uri.EscapeDataString(Environment.GetEnvironmentVariable("TESTCREDENTIAL_BOX_AUTHID")!)}", + $"box://{Environment.GetEnvironmentVariable("TESTCREDENTIAL_BOX_FOLDER")}/?authid={Uri.EscapeDataString(Environment.GetEnvironmentVariable("TESTCREDENTIAL_BOX_AUTHID")!)}", }.Concat(Parameters.GlobalTestParameters).ToArray()); From 71c4527d9c8b29ef3c708b223c2e372c34984d0b Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 6 Mar 2025 20:59:29 +0100 Subject: [PATCH 309/362] Fixed argument passing --- LiveTests/Duplicati.Backend.Tests/Box/BoxTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LiveTests/Duplicati.Backend.Tests/Box/BoxTests.cs b/LiveTests/Duplicati.Backend.Tests/Box/BoxTests.cs index e4bd6e6b0..d393a2118 100644 --- a/LiveTests/Duplicati.Backend.Tests/Box/BoxTests.cs +++ b/LiveTests/Duplicati.Backend.Tests/Box/BoxTests.cs @@ -35,7 +35,7 @@ public sealed class BoxTests : BaseTest var exitCode = CommandLine.BackendTester.Program.Main( new[] { - $"box://{Environment.GetEnvironmentVariable("TESTCREDENTIAL_BOX_FOLDER")}/?auth-id={Uri.EscapeDataString(Environment.GetEnvironmentVariable("TESTCREDENTIAL_BOX_AUTHID")!)}", + $"box://{Environment.GetEnvironmentVariable("TESTCREDENTIAL_BOX_FOLDER")}/?authid={Uri.EscapeDataString(Environment.GetEnvironmentVariable("TESTCREDENTIAL_BOX_AUTHID")!)}", }.Concat(Parameters.GlobalTestParameters).ToArray()); From 9953f6d536c39608a7df08989692843d6b51f24d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Mar 2025 22:32:26 +0000 Subject: [PATCH 310/362] Bump SixLabors.ImageSharp Bumps the nuget group with 1 update in the /Duplicati/WebserverCore directory: [SixLabors.ImageSharp](https://github.com/SixLabors/ImageSharp). Updates `SixLabors.ImageSharp` from 3.1.5 to 3.1.7 - [Release notes](https://github.com/SixLabors/ImageSharp/releases) - [Commits](https://github.com/SixLabors/ImageSharp/compare/v3.1.5...v3.1.7) --- updated-dependencies: - dependency-name: SixLabors.ImageSharp dependency-type: direct:production dependency-group: nuget ... Signed-off-by: dependabot[bot] --- Duplicati/WebserverCore/Duplicati.WebserverCore.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/WebserverCore/Duplicati.WebserverCore.csproj b/Duplicati/WebserverCore/Duplicati.WebserverCore.csproj index be4dbc9f8..12af07a08 100644 --- a/Duplicati/WebserverCore/Duplicati.WebserverCore.csproj +++ b/Duplicati/WebserverCore/Duplicati.WebserverCore.csproj @@ -17,7 +17,7 @@ - + From 7c78a879d92ba873ddb10c0afc4c7c11a2a38145 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 7 Mar 2025 08:26:59 +0100 Subject: [PATCH 311/362] Remove imaging library The image library was used in the captcha code, which was previously removed. --- Duplicati/WebserverCore/Duplicati.WebserverCore.csproj | 2 -- 1 file changed, 2 deletions(-) diff --git a/Duplicati/WebserverCore/Duplicati.WebserverCore.csproj b/Duplicati/WebserverCore/Duplicati.WebserverCore.csproj index 12af07a08..93da37923 100644 --- a/Duplicati/WebserverCore/Duplicati.WebserverCore.csproj +++ b/Duplicati/WebserverCore/Duplicati.WebserverCore.csproj @@ -17,8 +17,6 @@ - - From 02d1775a11cfbe715be9fc805f00784655dd2dd2 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 7 Mar 2025 08:59:19 +0100 Subject: [PATCH 312/362] Removed two `Await()` calls --- Duplicati/Library/Backend/Backblaze/B2.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Backend/Backblaze/B2.cs b/Duplicati/Library/Backend/Backblaze/B2.cs index 8d72ed7c0..4ed35e91b 100644 --- a/Duplicati/Library/Backend/Backblaze/B2.cs +++ b/Duplicati/Library/Backend/Backblaze/B2.cs @@ -272,7 +272,7 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend _uploadUrl = await _b2AuthHelper.PostAndGetJsonDataAsync( $"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_get_upload_url", - new UploadUrlRequest { BucketID = GetBucketAsync(cancellationToken).Await().BucketID }, + new UploadUrlRequest { BucketID = (await GetBucketAsync(cancellationToken)).BucketID }, combinedCancellationToken.Token ).ConfigureAwait(false); @@ -487,7 +487,7 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend $"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_list_file_versions", new ListFilesRequest { - BucketID = GetBucketAsync(cancellationToken).Await().BucketID, + BucketID = (await GetBucketAsync(cancellationToken)).BucketID, MaxFileCount = _pageSize, Prefix = _prefix, StartFileID = nextFileId, From 3eccc568438b0fd8b99948f86bf9725f68b6c74a Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 7 Mar 2025 10:46:08 +0100 Subject: [PATCH 313/362] Improve import feature This commit adds support for full names for day-of-week, as it appears that some exported documents contain the full day names instead of their three-letter abbreviations. In case of errors, the error message now also includes the value that was found. --- .../DayOfWeekStringEnumConverter.cs | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/Duplicati/WebserverCore/DayOfWeekStringEnumConverter.cs b/Duplicati/WebserverCore/DayOfWeekStringEnumConverter.cs index 27ebec38a..c0e93e3f6 100644 --- a/Duplicati/WebserverCore/DayOfWeekStringEnumConverter.cs +++ b/Duplicati/WebserverCore/DayOfWeekStringEnumConverter.cs @@ -32,16 +32,18 @@ public class DayOfWeekStringEnumConverter : JsonConverterFactory if (reader.TokenType != JsonTokenType.String) throw new JsonException(); - return reader.GetString()?.Trim()?.ToLowerInvariant() switch + var value = reader.GetString()?.Trim()?.ToLowerInvariant(); + + return value switch { - "sun" => DayOfWeek.Sunday, - "mon" => DayOfWeek.Monday, - "tue" => DayOfWeek.Tuesday, - "wed" => DayOfWeek.Wednesday, - "thu" => DayOfWeek.Thursday, - "fri" => DayOfWeek.Friday, - "sat" => DayOfWeek.Saturday, - _ => throw new JsonException() + "sun" or "sunday" => DayOfWeek.Sunday, + "mon" or "monday" => DayOfWeek.Monday, + "tue" or "tuesday" => DayOfWeek.Tuesday, + "wed" or "wednesday" => DayOfWeek.Wednesday, + "thu" or "thursday" => DayOfWeek.Thursday, + "fri" or "friday" => DayOfWeek.Friday, + "sat" or "saturday" => DayOfWeek.Saturday, + _ => throw new JsonException($"Invalid day of week: {value}"), }; } @@ -71,7 +73,7 @@ public class DayOfWeekStringEnumConverter : JsonConverterFactory writer.WriteStringValue("sat"); break; default: - throw new JsonException(); + throw new JsonException($"Invalid day of week: {value}"); } writer.WriteStringValue(value.ToString()); } From 29021e9255a743bd3524edff3aa41bfd183b2fef Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 7 Mar 2025 11:57:36 +0100 Subject: [PATCH 314/362] Remove call context in favor of AsyncLocal This removes the use of the `CallContext` substitute, which was a compatibility class from .NET4 times, and replaces it with `AsyncLocal` which retains the call state. --- Duplicati/Library/Logging/CallContext.cs | 50 ----- Duplicati/Library/Logging/Log.cs | 80 +++---- .../Library/Utility/CallContextSettings.cs | 207 +++++++----------- 3 files changed, 101 insertions(+), 236 deletions(-) delete mode 100644 Duplicati/Library/Logging/CallContext.cs diff --git a/Duplicati/Library/Logging/CallContext.cs b/Duplicati/Library/Logging/CallContext.cs deleted file mode 100644 index dfb6e8f36..000000000 --- a/Duplicati/Library/Logging/CallContext.cs +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. -using System.Collections.Concurrent; -using System.Threading; - -namespace Duplicati.Library.Logging -{ - /// - /// Provides a way to set contextual data that flows with the call and - /// async context of a test or invocation. - /// - public static class CallContext - { - static ConcurrentDictionary> state = new ConcurrentDictionary>(); - - /// - /// Stores a given object and associates it with the specified name. - /// - /// The name with which to associate the new item in the call context. - /// The object to store in the call context. - public static void SetData(string name, object data) => - state.GetOrAdd(name, _ => new AsyncLocal()).Value = data; - - /// - /// Retrieves an object with the specified name from the . - /// - /// The name of the item in the call context. - /// The object in the call context associated with the specified name, or if not found. - public static object GetData(string name) => - state.TryGetValue(name, out AsyncLocal data) ? data.Value : null; - } -} \ No newline at end of file diff --git a/Duplicati/Library/Logging/Log.cs b/Duplicati/Library/Logging/Log.cs index bea2236ab..d53a9de3f 100644 --- a/Duplicati/Library/Logging/Log.cs +++ b/Duplicati/Library/Logging/Log.cs @@ -1,27 +1,27 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; -using System.Text; +using System.Threading; namespace Duplicati.Library.Logging { @@ -72,18 +72,13 @@ namespace Duplicati.Library.Logging /// /// The key used to assign the current scope into the current call-context /// - private const string LOGICAL_CONTEXT_KEY = "Duplicati:LoggingEntry"; + private static readonly AsyncLocal _currentScope = new AsyncLocal(); /// /// The root scope /// private static readonly LogScope m_root = new LogScope(null, new LogTagFilter(LogMessageType.Error, null, null), null, true); - /// - /// The stored log instances - /// - private static readonly Dictionary m_log_instances = new Dictionary(); - /// /// Static lock object to provide thread safe logging /// @@ -216,15 +211,15 @@ namespace Duplicati.Library.Logging WriteMessage(LogMessageType.Verbose, tag, id, null, message, arguments); } - /// + /// /// Writes a verbose message to the current log destination /// /// The message to write /// The tag-type for this message /// The message id - /// The exception to log + /// The exception to log /// The message format arguments - public static void WriteVerboseMessage(string tag, string id, Exception ex, string message, params object[] arguments) + public static void WriteVerboseMessage(string tag, string id, Exception ex, string message, params object[] arguments) { WriteMessage(LogMessageType.Verbose, tag, id, ex, message, arguments); } @@ -428,11 +423,10 @@ namespace Duplicati.Library.Logging /// The scope to finish. internal static void CloseScope(LogScope scope) { - lock(m_lock) + lock (m_lock) { if (CurrentScope == scope && scope != m_root) CurrentScope = scope.Parent; - m_log_instances.Remove(scope.InstanceID); } } @@ -444,32 +438,12 @@ namespace Duplicati.Library.Logging get { lock (m_lock) - { - var cur = CallContext.GetData(LOGICAL_CONTEXT_KEY) as string; - if (cur == null || cur == m_root.InstanceID) - return m_root; - - LogScope sc; - if (!m_log_instances.TryGetValue(cur, out sc)) - throw new Exception("Unable to find log in lookup table, this may be caused by attempting to transport call contexts between AppDomains (e.g. with remoting calls)"); - - return sc; - } + return _currentScope.Value ?? m_root; } private set { lock (m_lock) - { - if (value != null) - { - m_log_instances[value.InstanceID] = value; - CallContext.SetData(LOGICAL_CONTEXT_KEY, value.InstanceID); - } - else - { - CallContext.SetData(LOGICAL_CONTEXT_KEY, null); - } - } + _currentScope.Value = value; } } } diff --git a/Duplicati/Library/Utility/CallContextSettings.cs b/Duplicati/Library/Utility/CallContextSettings.cs index c78bea518..66a1824cf 100644 --- a/Duplicati/Library/Utility/CallContextSettings.cs +++ b/Duplicati/Library/Utility/CallContextSettings.cs @@ -1,32 +1,32 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Net.Security; using System.Security.Cryptography.X509Certificates; - +using System.Threading; using Duplicati.Library.Interface; -using Duplicati.Library.Logging; namespace Duplicati.Library.Utility { @@ -81,12 +81,9 @@ namespace Duplicati.Library.Utility } set { - lock (CallContextSettings._lock) - { - var st = CallContextSettings.Settings; - st.Tempdir = value; - CallContextSettings.Settings = st; - } + var st = CallContextSettings.Settings; + st.Tempdir = value; + CallContextSettings.Settings = st; } } @@ -228,20 +225,58 @@ namespace Duplicati.Library.Utility public static class CallContextSettings { /// - /// The settings that are stored in the call context, which are not serializable + /// The call context settings for OAuth /// - private static readonly Dictionary _settings = new Dictionary(); + private static readonly AsyncLocal _settings = new AsyncLocal(); /// - /// The context setting types that are used and need their private setting key - /// to prevent overwriting each others settings. + /// The instances that are currently active /// - private static string contextSettingsType = null; + private static readonly ConcurrentDictionary _instances = new ConcurrentDictionary(); /// - /// Lock for protecting the dictionary + /// Disposable class for setting call context settings /// - public static readonly object _lock = new object(); + /// The type of the settings + private sealed class Disposer : IDisposable + { + /// + /// A flag indicating if the object has been disposed + /// + private bool m_disposed = false; + /// + /// The previous value + /// + private T m_prev; + /// + /// The current value + /// + private T m_current; + + /// + /// Initializes a new instance of the class. + /// + /// The current server URL + /// The previous server URL + public Disposer(T current, T prev) + { + _settings.Value = m_current = current; + this.m_prev = prev; + _instances.TryAdd(current, null); + } + + /// + public void Dispose() + { + if (m_disposed) + return; + GC.SuppressFinalize(this); + m_disposed = true; + if (_settings.Value.Equals(m_current)) + _settings.Value = m_prev; + _instances.TryRemove(m_current, out _); + } + } /// /// Gets or sets the values in the current call context @@ -249,40 +284,8 @@ namespace Duplicati.Library.Utility /// The settings. public static T Settings { - get - { - var key = ContextID; - if (string.IsNullOrWhiteSpace(key)) - return default(T); - - - if (!_settings.TryGetValue(key, out T res)) - lock (_lock) // if not present but context id is not null, wait - if (!_settings.TryGetValue(key, out res)) - return default(T); - - return res; - } - set - { - var key = ContextID; - if (!string.IsNullOrWhiteSpace(key)) - lock (_lock) - _settings[key] = value; - } - } - - /// - /// Starts the context wit a default value. - /// - /// The disposable handle for the context. - /// The initial value. - public static IDisposable StartContext(string settingsKey, T initial = default(T)) - { - contextSettingsType = settingsKey; - var res = new ContextGuard(); - Settings = initial; - return res; + get => _settings.Value; + set => _settings.Value = value; } /// @@ -291,72 +294,10 @@ namespace Duplicati.Library.Utility /// The disposable handle for the context. /// The initial value. public static IDisposable StartContext(T initial = default(T)) - { - return StartContext(typeof(T).ToString(), initial); - } + => new Disposer(initial, _settings.Value); - /// - /// Gets all instances currently registered - /// - internal static T[] GetAllInstances() - { - lock (_lock) - return _settings.Values.ToArray(); - } + public static IEnumerable GetAllInstances() + => _instances.Keys.ToList(); - /// - /// Help class for providing a way to release resources associated with a call context - /// - private class ContextGuard : IDisposable - { - /// - /// Randomly generated identifier - /// - private readonly string ID = Guid.NewGuid().ToString(); - - /// - /// Initializes a new instance of the - /// class, - /// and sets up the call context - /// - public ContextGuard() - { - CallContext.SetData(contextSettingsType, ID); - } - - /// - /// Releases all resource used by the - /// object. - /// - /// Call when you are finished using the - /// . The - /// method leaves the in an - /// unusable state. After calling , you must release all references to the - /// so the garbage collector - /// can reclaim the memory that the - /// was occupying. - public void Dispose() - { - lock (_lock) - { - // Release the resources, if any - _settings.Remove(ID); - } - - CallContext.SetData(contextSettingsType, null); - } - } - - /// - /// Gets the context ID for this call. - /// - /// The context identifier. - private static string ContextID - { - get - { - return contextSettingsType != null ? CallContext.GetData(contextSettingsType) as string : null; - } - } } } From 16aff9dc0bb34f2251cf324b2a691ffe52e2c69f Mon Sep 17 00:00:00 2001 From: "Marcelo C." Date: Fri, 7 Mar 2025 10:32:37 -0300 Subject: [PATCH 315/362] Corrected implementation around exception handling This implementation ensures the exception for FolderMissingException is treated properly in async enum circumventing the limitation of not having a yield within a try catch block. --- .../Backend/AzureBlob/AzureBlobWrapper.cs | 37 +++++++++++-------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs b/Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs index 2cdfbbe38..769d06f21 100644 --- a/Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs +++ b/Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs @@ -129,30 +129,37 @@ namespace Duplicati.Library.Backend.AzureBlob /// Thrown when the container is not found public virtual async IAsyncEnumerable ListContainerEntriesAsync([EnumeratorCancellation] CancellationToken cancelToken) { - IAsyncEnumerable blobs = GetBlobsWithExceptionHandling(); - - await foreach (var blobItem in blobs.WithCancellation(cancelToken).ConfigureAwait(false)) + await using var blobEnumerator = _container.GetBlobsAsync().GetAsyncEnumerator(cancelToken); + + while (true) { - var blobName = Uri.UnescapeDataString(blobItem.Name.Replace("+", "%2B")); + bool hasNext; - if (blobItem is { } bi) - { - var lastModified = bi.Properties.LastModified?.UtcDateTime ?? DateTime.UtcNow; - yield return new FileEntry(blobName, bi.Properties.ContentLength ?? 0, lastModified, lastModified); - } - } - - IAsyncEnumerable GetBlobsWithExceptionHandling() - { try { - // Note: AsyncPageable return internally uses .ConfigureAwait(false). - return _container.GetBlobsAsync(cancellationToken: cancelToken); + hasNext = await blobEnumerator.MoveNextAsync().ConfigureAwait(false); } catch (Azure.RequestFailedException ex) when (ex.Status == 404) { throw new FolderMissingException(ex); } + + if (!hasNext) break; + + cancelToken.ThrowIfCancellationRequested(); + + if (blobEnumerator.Current is { } blob) + { + var blobName = Uri.UnescapeDataString(blob.Name.Replace("+", "%2B")); + var lastModified = blob.Properties.LastModified?.UtcDateTime ?? DateTime.UtcNow; + + yield return new FileEntry( + blobName, + blob.Properties.ContentLength ?? 0, + lastModified, + lastModified + ); + } } } } From 58da574130d186654494776b3cf040b14a88d869 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 7 Mar 2025 15:19:40 +0100 Subject: [PATCH 316/362] Moved shutdown logic into separate method so it can be called for both uploads and downloads --- .../Main/Backend/BackendManager.Handler.cs | 63 ++++++++++++------- 1 file changed, 40 insertions(+), 23 deletions(-) diff --git a/Duplicati/Library/Main/Backend/BackendManager.Handler.cs b/Duplicati/Library/Main/Backend/BackendManager.Handler.cs index 248605a57..cf1198926 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.Handler.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.Handler.cs @@ -311,29 +311,8 @@ partial class BackendManager { Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerDisposeWhileActive", null, "Terminating {0} active uploads", activeUploads.Count); - // Wait for all active uploads to complete - await Task.WhenAny(Task.Delay(1000), Task.WhenAll(activeUploads)).ConfigureAwait(false); - for (int i = activeUploads.Count - 1; i >= 0; i--) - { - var t = activeUploads[i]; - if (t.IsCompleted) - { - activeUploads.RemoveAt(i); - if (t.IsCanceled) - Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerDisposeError", t.Exception, "Error in active upload: Cancelled"); - else if (t.IsFaulted) - Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerDisposeError", t.Exception, "Error in active upload: {0}", t.Exception?.Message ?? "null"); - else - Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerDisposeError", null, "Upload was active during termination, but completed successfully"); - } - else - { - Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerDisposeError", null, "Upload was active during termination, but had state: {0}", t.Status); - } - - if (activeUploads.Count > 0) - Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerDisposeError", null, "Terminating, but {0} active uploads are still active", activeUploads.Count); - } + await WaitForPendingItems("upload", activeUploads).ConfigureAwait(false); + await WaitForPendingItems("download", activeDownloads).ConfigureAwait(false); // Dispose of any remaining backends while (backendPool.TryDequeue(out var backend)) @@ -343,6 +322,44 @@ partial class BackendManager } } + /// + /// + /// + /// + /// + /// + private static async Task WaitForPendingItems(string description, List tasks) + { + if (tasks.Count > 0) + { + Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerDisposeWhileActive", null, "Terminating {0} active {1}s", tasks.Count, description); + + // Wait for all active tasks to complete + await Task.WhenAny(Task.Delay(1000), Task.WhenAll(tasks)).ConfigureAwait(false); + for (int i = tasks.Count - 1; i >= 0; i--) + { + var t = tasks[i]; + if (t.IsCompleted) + { + tasks.RemoveAt(i); + if (t.IsCanceled) + Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerDisposeError", t.Exception, "Error in active {0}: Cancelled", description); + else if (t.IsFaulted) + Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerDisposeError", t.Exception, "Error in active {0}: {1}", description, t.Exception?.Message ?? "null"); + else + Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerDisposeError", null, "{0} was active during termination, but completed successfully", description); + } + else + { + Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerDisposeError", null, "{0} was active during termination, but had state: {1}", description, t.Status); + } + + if (tasks.Count > 0) + Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerDisposeError", null, "Terminating, but {0} active {1}(s) are still active", tasks.Count, description); + } + } + } + /// /// Tries to create a folder, handling errors /// From c3762a57ca2ea409431d4d93d8b845c18907403c Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 7 Mar 2025 15:23:06 +0100 Subject: [PATCH 317/362] Added docs --- .../Library/Main/Operation/FilelistProcessor.cs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Duplicati/Library/Main/Operation/FilelistProcessor.cs b/Duplicati/Library/Main/Operation/FilelistProcessor.cs index 2fb1690b5..98b42d979 100644 --- a/Duplicati/Library/Main/Operation/FilelistProcessor.cs +++ b/Duplicati/Library/Main/Operation/FilelistProcessor.cs @@ -102,6 +102,18 @@ namespace Duplicati.Library.Main.Operation } } + /// + /// Helper method that verifies uploaded volumes and updates their state in the database. + /// Throws an error if there are issues with the remote storage + /// + /// The backend instance to use + /// The options used + /// The database to compare with + /// The backend writer to use + /// True if only the latest volumes should be verified + /// The mode to use for verification + /// An optional transaction object + /// An awaitable task public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter backendWriter, bool latestVolumesOnly, VerifyMode verifyMode, IDbTransaction transaction) { if (!options.NoBackendverification) @@ -124,6 +136,7 @@ namespace Duplicati.Library.Main.Operation /// Filenames that should be exempted from strict verification /// The mode to use for verification /// Disable the logging of errors to prevent spamming the log; exceptions will be thrown regardless + /// An awaitable task public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter log, IEnumerable protectedFiles, IEnumerable strictExcemptFiles, bool logErrors, VerifyMode verifyMode) { var tp = await RemoteListAnalysis(backend, options, database, log, protectedFiles, strictExcemptFiles, verifyMode).ConfigureAwait(false); From f6f0cf8b96ceff97a7e3e8945c4014b8879e04be Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 7 Mar 2025 15:24:56 +0100 Subject: [PATCH 318/362] More docs --- .../Library/Main/Database/LocalDatabase.cs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index efa59a460..26583fdb7 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -737,6 +737,11 @@ AND Fileset.ID NOT IN return GetDbOptionList(transaction).ToDictionary(x => x.Key, x => x.Value); } + /// + /// Updates a database option + /// + /// The key to update + /// The value to set private void UpdateDbOption(string key, bool value) { var opts = GetDbOptions(); @@ -749,24 +754,38 @@ AND Fileset.ID NOT IN SetDbOptions(opts); } + /// + /// Flag indicating if a repair is in progress + /// public bool RepairInProgress { get => GetDbOptions().ContainsKey("repair-in-progress"); set => UpdateDbOption("repair-in-progress", value); } + /// + /// Flag indicating if a repair is in progress + /// public bool PartiallyRecreated { get => GetDbOptions().ContainsKey("partially-recreated"); set => UpdateDbOption("partially-recreated", value); } + /// + /// Flag indicating if the database can contain partial uploads + /// public bool TerminatedWithActiveUploads { get => GetDbOptions().ContainsKey("terminated-with-active-uploads"); set => UpdateDbOption("terminated-with-active-uploads", value); } + /// + /// Sets the database options + /// + /// The options to set + /// An optional transaction public void SetDbOptions(IDictionary options, System.Data.IDbTransaction transaction = null) { using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) From 9952e31f4959ae1f2e55242f9aa298469baa19c4 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sat, 8 Mar 2025 15:13:03 +0100 Subject: [PATCH 319/362] Fixed a bug where retention time was incorrectly parsed when reading from the database. This effectively caused the retention time to be ignored when reading from the database. Also moved some code related to finding an unfinished fileset into the common database class. --- .../Main/Database/LocalBackupDatabase.cs | 33 --------- .../Library/Main/Database/LocalDatabase.cs | 68 +++++++++++++++++-- 2 files changed, 63 insertions(+), 38 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index 6417344d0..730126233 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -771,39 +771,6 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist return m_filesetId = base.CreateFileset(volumeid, timestamp, transaction); } - public IEnumerable> GetIncompleteFilesets(System.Data.IDbTransaction transaction) - { - using (var cmd = m_connection.CreateCommand(transaction)) - { - using (var rd = cmd.ExecuteReader(@$"SELECT DISTINCT ""Fileset"".""ID"", ""Fileset"".""Timestamp"" FROM ""Fileset"", ""RemoteVolume"" WHERE ""RemoteVolume"".""ID"" = ""Fileset"".""VolumeID"" AND ""Fileset"".""ID"" IN (SELECT ""FilesetID"" FROM ""FilesetEntry"") AND (""RemoteVolume"".""State"" = '{RemoteVolumeState.Uploading}' OR ""RemoteVolume"".""State"" = '{RemoteVolumeState.Temporary}')")) - while (rd.Read()) - { - yield return new KeyValuePair( - rd.GetInt64(0), - ParseFromEpochSeconds(rd.GetInt64(1)).ToLocalTime() - ); - } - } - } - - public RemoteVolumeEntry GetRemoteVolumeFromFilesetID(long filesetID, IDbTransaction transaction = null) - { - using (var cmd = m_connection.CreateCommand(transaction)) - using (var rd = cmd.ExecuteReader(@"SELECT ""RemoteVolume"".""ID"", ""Name"", ""Type"", ""Size"", ""Hash"", ""State"", ""DeleteGraceTime"" FROM ""RemoteVolume"", ""Fileset"" WHERE ""Fileset"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""Fileset"".""ID"" = ?", filesetID)) - if (rd.Read()) - return new RemoteVolumeEntry( - rd.ConvertValueToInt64(0, -1), - rd.GetValue(1).ToString(), - (rd.GetValue(4) == null || rd.GetValue(4) == DBNull.Value) ? null : rd.GetValue(4).ToString(), - rd.ConvertValueToInt64(3, -1), - (RemoteVolumeType)Enum.Parse(typeof(RemoteVolumeType), rd.GetValue(2).ToString()), - (RemoteVolumeState)Enum.Parse(typeof(RemoteVolumeState), rd.GetValue(5).ToString()), - new DateTime(rd.ConvertValueToInt64(6, 0), DateTimeKind.Utc) - ); - else - return default(RemoteVolumeEntry); - } - public IEnumerable GetTemporaryFilelistVolumeNames(bool latestOnly, IDbTransaction transaction = null) { var incompleteFilesetIDs = GetIncompleteFilesets(transaction).OrderBy(x => x.Value).Select(x => x.Key).ToArray(); diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index efa59a460..4a0341555 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -366,7 +366,7 @@ namespace Duplicati.Library.Main.Database rd.ConvertValueToInt64(3, -1), (RemoteVolumeType)Enum.Parse(typeof(RemoteVolumeType), rd.GetValue(2).ToString()), (RemoteVolumeState)Enum.Parse(typeof(RemoteVolumeState), rd.GetValue(5).ToString()), - new DateTime(rd.ConvertValueToInt64(6, 0), DateTimeKind.Utc) + ParseFromEpochSeconds(rd.ConvertValueToInt64(6, 0)) ); return RemoteVolumeEntry.Empty; @@ -397,7 +397,7 @@ namespace Duplicati.Library.Main.Database rd.ConvertValueToInt64(3, -1), (RemoteVolumeType)Enum.Parse(typeof(RemoteVolumeType), rd.GetValue(2).ToString()), (RemoteVolumeState)Enum.Parse(typeof(RemoteVolumeState), rd.GetValue(5).ToString()), - new DateTime(rd.ConvertValueToInt64(6, 0), DateTimeKind.Utc) + ParseFromEpochSeconds(rd.ConvertValueToInt64(6, 0)) ); } } @@ -451,12 +451,12 @@ namespace Duplicati.Library.Main.Database } } - public void RemoveRemoteVolume(string name, System.Data.IDbTransaction transaction = null) + public void RemoveRemoteVolume(string name, IDbTransaction transaction = null) { - RemoveRemoteVolumes(new string[] { name }, transaction); + RemoveRemoteVolumes([name], transaction); } - public void RemoveRemoteVolumes(IEnumerable names, System.Data.IDbTransaction transaction = null) + public void RemoveRemoteVolumes(IEnumerable names, IDbTransaction transaction = null) { if (names == null || !names.Any()) return; @@ -1440,6 +1440,64 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != ? ORDER BY Timest } } + /// + /// Gets the last previous fileset that was incomplete + /// + /// The transaction to use + /// The last incomplete fileset or default + public RemoteVolumeEntry GetLastIncompleteFilesetVolume(IDbTransaction transaction) + { + var candidates = GetIncompleteFilesets(transaction).OrderBy(x => x.Value).ToArray(); + if (candidates.Any()) + return GetRemoteVolumeFromFilesetID(candidates.Last().Key, transaction); + + return default; + } + + /// + /// Gets a list of incomplete filesets + /// + /// An optional transaction + /// A list of fileset IDs and timestamps + public IEnumerable> GetIncompleteFilesets(IDbTransaction transaction) + { + using (var cmd = m_connection.CreateCommand(transaction)) + { + using (var rd = cmd.ExecuteReader(@$"SELECT DISTINCT ""Fileset"".""ID"", ""Fileset"".""Timestamp"" FROM ""Fileset"", ""RemoteVolume"" WHERE ""RemoteVolume"".""ID"" = ""Fileset"".""VolumeID"" AND ""Fileset"".""ID"" IN (SELECT ""FilesetID"" FROM ""FilesetEntry"") AND (""RemoteVolume"".""State"" = '{RemoteVolumeState.Uploading}' OR ""RemoteVolume"".""State"" = '{RemoteVolumeState.Temporary}')")) + while (rd.Read()) + { + yield return new KeyValuePair( + rd.GetInt64(0), + ParseFromEpochSeconds(rd.GetInt64(1)).ToLocalTime() + ); + } + } + } + + /// + /// Gets the remote volume entry from the fileset ID + /// + /// The fileset ID + /// An optional transaction + /// The remote volume entry or default + public RemoteVolumeEntry GetRemoteVolumeFromFilesetID(long filesetID, IDbTransaction transaction = null) + { + using (var cmd = m_connection.CreateCommand(transaction)) + using (var rd = cmd.ExecuteReader(@"SELECT ""RemoteVolume"".""ID"", ""Name"", ""Type"", ""Size"", ""Hash"", ""State"", ""DeleteGraceTime"" FROM ""RemoteVolume"", ""Fileset"" WHERE ""Fileset"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""Fileset"".""ID"" = ?", filesetID)) + if (rd.Read()) + return new RemoteVolumeEntry( + rd.ConvertValueToInt64(0, -1), + rd.GetValue(1).ToString(), + (rd.GetValue(4) == null || rd.GetValue(4) == DBNull.Value) ? null : rd.GetValue(4).ToString(), + rd.ConvertValueToInt64(3, -1), + (RemoteVolumeType)Enum.Parse(typeof(RemoteVolumeType), rd.GetValue(2).ToString()), + (RemoteVolumeState)Enum.Parse(typeof(RemoteVolumeState), rd.GetValue(5).ToString()), + ParseFromEpochSeconds(rd.GetInt64(6)).ToLocalTime() + ); + else + return default(RemoteVolumeEntry); + } + public void PurgeLogData(DateTime threshold) { using (var tr = m_connection.BeginTransaction()) From d019845fad7448bff9544b8cce3aef4f4ac2e78f Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sat, 8 Mar 2025 15:14:03 +0100 Subject: [PATCH 320/362] Added a number of new tests to ensure correct handling of edge cases related to missing/interrupted uploads. --- Duplicati/UnitTest/DisruptionTests.cs | 242 ++++++++++++++++++++++++++ 1 file changed, 242 insertions(+) diff --git a/Duplicati/UnitTest/DisruptionTests.cs b/Duplicati/UnitTest/DisruptionTests.cs index f3c63f968..ff0f346f1 100644 --- a/Duplicati/UnitTest/DisruptionTests.cs +++ b/Duplicati/UnitTest/DisruptionTests.cs @@ -854,6 +854,248 @@ namespace Duplicati.UnitTest } } + [Test] + [Category("Disruption")] + [TestCase(true, false)] + [TestCase(false, false)] + [TestCase(true, true)] + [TestCase(false, true)] + public void TestFailedUploadOfFirstDlist(bool before, bool failOnLastDblock) + { + var testopts = TestOptions; + testopts["number-of-retries"] = "0"; + testopts["dblock-size"] = "10mb"; + + // Deterministic error backend + Library.DynamicLoader.BackendLoader.AddBackend(new DeterministicErrorBackend()); + var failtarget = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER; + var operation = before + ? DeterministicErrorBackend.BackendAction.PutBefore + : DeterministicErrorBackend.BackendAction.PutAfter; + + // We expect to upload 7 volumes, and fail on the last one + var count = 7; + + // Fail when uploading the dlist file + DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) => + { + if (failOnLastDblock) + { + if (action == operation && remotename.Contains(".dblock.")) + { + if (Interlocked.Decrement(ref count) == 0) + { + // Allow others to progress + Thread.Sleep(2000); + return true; + } + } + } + else + { + if (action == operation && remotename.Contains(".dlist.")) + return true; + } + + return false; + }; + + using (var c = new Library.Main.Controller(failtarget, testopts, null)) + Assert.Throws(() => c.Backup(new string[] { DATAFOLDER })); + + // Make spacing for the next backup + Thread.Sleep(3000); + + // Make a new backup, should continue as normal + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER }); + Assert.AreEqual(0, backupResults.Errors.Count()); + Assert.AreEqual(0, backupResults.Warnings.Count()); + Assert.AreEqual(2, c.List().Filesets.Count()); + } + + // Verify that all is in order + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null)) + { + var r = c.Test(long.MaxValue); + Assert.AreEqual(0, r.Errors.Count()); + Assert.AreEqual(0, r.Warnings.Count()); + Assert.IsFalse(r.Verifications.Any(p => p.Value.Any())); + } + + // Test that we can recreate + var recreatedDatabaseFile = Path.Combine(BASEFOLDER, "recreated-database.sqlite"); + if (File.Exists(recreatedDatabaseFile)) + File.Delete(recreatedDatabaseFile); + + testopts["dbpath"] = recreatedDatabaseFile; + + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IRepairResults repairResults = c.Repair(); + Assert.AreEqual(0, repairResults.Errors.Count()); + Assert.AreEqual(0, repairResults.Warnings.Count()); + } + + // Check that we have 2 versions + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IListResults listResults = c.List(); + Assert.AreEqual(0, listResults.Errors.Count()); + Assert.AreEqual(0, listResults.Warnings.Count()); + Assert.AreEqual(2, listResults.Filesets.Count()); + } + } + + [Test] + [Category("Disruption")] + [TestCase(true)] + [TestCase(false)] + public void TestRepairWithNoDlist(bool before) + { + var testopts = TestOptions; + testopts["number-of-retries"] = "0"; + testopts["dblock-size"] = "10mb"; + + // Deterministic error backend + Library.DynamicLoader.BackendLoader.AddBackend(new DeterministicErrorBackend()); + var failtarget = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER; + var operation = before + ? DeterministicErrorBackend.BackendAction.PutBefore + : DeterministicErrorBackend.BackendAction.PutAfter; + + // Fail when uploading the first dlist file + DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) => + { + if (action == operation && remotename.Contains(".dlist.")) + return true; + + return false; + }; + + using (var c = new Library.Main.Controller(failtarget, testopts, null)) + Assert.Throws(() => c.Backup(new string[] { DATAFOLDER })); + + // Test that we can repair + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IRepairResults repairResults = c.Repair(); + Assert.AreEqual(0, repairResults.Errors.Count()); + Assert.AreEqual(0, repairResults.Warnings.Count()); + } + + // Check that we have 1 version + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IListResults listResults = c.List(); + Assert.AreEqual(0, listResults.Errors.Count()); + Assert.AreEqual(0, listResults.Warnings.Count()); + Assert.AreEqual(1, listResults.Filesets.Count()); + } + } + + + [Test] + [Category("Disruption")] + [TestCase(true, true, 3, false)] + [TestCase(false, false, 3, false)] + [TestCase(true, false, 3, false)] + [TestCase(false, true, 3, false)] + [TestCase(true, true, 3, true)] + [TestCase(false, true, 3, true)] + public void TestMultiFailedDlist(bool before, bool modifyInBetween, int runs, bool withBase) + { + var testopts = TestOptions; + testopts["number-of-retries"] = "0"; + testopts["dblock-size"] = "10mb"; + var expectedFilesets = withBase ? 3 : 2; + + if (withBase) + { + // Make a regular backup + using (var c = new Controller("file://" + TARGETFOLDER, testopts, null)) + { + IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER }); + Assert.AreEqual(0, backupResults.Errors.Count()); + Assert.AreEqual(0, backupResults.Warnings.Count()); + } + } + + + // Deterministic error backend + Library.DynamicLoader.BackendLoader.AddBackend(new DeterministicErrorBackend()); + var failtarget = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER; + var operation = before + ? DeterministicErrorBackend.BackendAction.PutBefore + : DeterministicErrorBackend.BackendAction.PutAfter; + + // Fail when uploading the first dlist file + DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) => + { + if (action == operation && remotename.Contains(".dlist.")) + return true; + + return false; + }; + + for (var i = 0; i < runs; i++) + { + if (modifyInBetween) + ModifySourceFiles(); + + using (var c = new Controller(failtarget, testopts, null)) + Assert.Throws(() => c.Backup(new string[] { DATAFOLDER })); + + // Prevent clashes in timestamps + Thread.Sleep(3000); + } + + if (modifyInBetween) + ModifySourceFiles(); + + // Make a new backup, should continue as normal + using (var c = new Controller("file://" + TARGETFOLDER, testopts, null)) + { + IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER }); + Assert.AreEqual(0, backupResults.Errors.Count()); + Assert.AreEqual(0, backupResults.Warnings.Count()); + Assert.AreEqual(expectedFilesets, c.List().Filesets.Count()); + } + + // Verify that all is in order + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null)) + { + var r = c.Test(long.MaxValue); + Assert.AreEqual(0, r.Errors.Count()); + Assert.AreEqual(0, r.Warnings.Count()); + Assert.IsFalse(r.Verifications.Any(p => p.Value.Any())); + } + + // Test that we can recreate + var recreatedDatabaseFile = Path.Combine(BASEFOLDER, "recreated-database.sqlite"); + if (File.Exists(recreatedDatabaseFile)) + File.Delete(recreatedDatabaseFile); + + testopts["dbpath"] = recreatedDatabaseFile; + + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IRepairResults repairResults = c.Repair(); + Assert.AreEqual(0, repairResults.Errors.Count()); + Assert.AreEqual(0, repairResults.Warnings.Count()); + } + + // Check that we have 2 versions + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IListResults listResults = c.List(); + Assert.AreEqual(0, listResults.Errors.Count()); + Assert.AreEqual(0, listResults.Warnings.Count()); + Assert.AreEqual(expectedFilesets, listResults.Filesets.Count()); + } + } + private class LogSink : IMessageSink { public List StartedFiles { get; } = new List(); From fce09596f3b75760ccc3065a37d84f7a824403d8 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sat, 8 Mar 2025 15:18:19 +0100 Subject: [PATCH 321/362] Removed unused parameter and redundant namespace prefixes --- Duplicati/Library/Main/Controller.cs | 146 +++++++++--------- .../Library/Main/Operation/BackupHandler.cs | 4 +- 2 files changed, 74 insertions(+), 76 deletions(-) diff --git a/Duplicati/Library/Main/Controller.cs b/Duplicati/Library/Main/Controller.cs index 7d509685b..5104e2282 100644 --- a/Duplicati/Library/Main/Controller.cs +++ b/Duplicati/Library/Main/Controller.cs @@ -114,11 +114,11 @@ namespace Duplicati.Library.Main SecretProvider = secretProvider; } - public Duplicati.Library.Interface.IBackupResults Backup(string[] inputsources, IFilter filter = null) + public IBackupResults Backup(string[] inputsources, IFilter filter = null) { - Library.UsageReporter.Reporter.Report("USE_BACKEND", new Library.Utility.Uri(m_backendUrl).Scheme); - Library.UsageReporter.Reporter.Report("USE_COMPRESSION", m_options.CompressionModule); - Library.UsageReporter.Reporter.Report("USE_ENCRYPTION", m_options.EncryptionModule); + UsageReporter.Reporter.Report("USE_BACKEND", new Library.Utility.Uri(m_backendUrl).Scheme); + UsageReporter.Reporter.Report("USE_COMPRESSION", m_options.CompressionModule); + UsageReporter.Reporter.Report("USE_ENCRYPTION", m_options.EncryptionModule); CheckAutoCompactInterval(); CheckAutoVacuumInterval(); @@ -126,30 +126,30 @@ namespace Duplicati.Library.Main return RunAction(new BackupResults(), ref inputsources, ref filter, (result, backendManager) => { - using (var h = new Operation.BackupHandler(m_backendUrl, m_options, result)) + using (var h = new Operation.BackupHandler(m_options, result)) { h.RunAsync(ExpandInputSources(inputsources, filter), backendManager, filter).Await(); } - Library.UsageReporter.Reporter.Report("BACKUP_FILECOUNT", result.ExaminedFiles); - Library.UsageReporter.Reporter.Report("BACKUP_FILESIZE", result.SizeOfExaminedFiles); - Library.UsageReporter.Reporter.Report("BACKUP_DURATION", (long)result.Duration.TotalSeconds); + UsageReporter.Reporter.Report("BACKUP_FILECOUNT", result.ExaminedFiles); + UsageReporter.Reporter.Report("BACKUP_FILESIZE", result.SizeOfExaminedFiles); + UsageReporter.Reporter.Report("BACKUP_DURATION", (long)result.Duration.TotalSeconds); }); } - public Library.Interface.IRestoreResults Restore(string[] paths, Library.Utility.IFilter filter = null) + public IRestoreResults Restore(string[] paths, IFilter filter = null) { return RunAction(new RestoreResults(), ref paths, ref filter, (result, backendManager) => { new Operation.RestoreHandler(m_options, result).Run(paths, backendManager, filter); - Library.UsageReporter.Reporter.Report("RESTORE_FILECOUNT", result.RestoredFiles); - Library.UsageReporter.Reporter.Report("RESTORE_FILESIZE", result.SizeOfRestoredFiles); - Library.UsageReporter.Reporter.Report("RESTORE_DURATION", (long)result.Duration.TotalSeconds); + UsageReporter.Reporter.Report("RESTORE_FILECOUNT", result.RestoredFiles); + UsageReporter.Reporter.Report("RESTORE_FILESIZE", result.SizeOfRestoredFiles); + UsageReporter.Reporter.Report("RESTORE_DURATION", (long)result.Duration.TotalSeconds); }); } - public Duplicati.Library.Interface.IRestoreControlFilesResults RestoreControlFiles(IEnumerable files = null, Library.Utility.IFilter filter = null) + public IRestoreControlFilesResults RestoreControlFiles(IEnumerable files = null, IFilter filter = null) { return RunAction(new RestoreControlFilesResults(), ref filter, (result, backendManager) => { @@ -157,7 +157,7 @@ namespace Duplicati.Library.Main }); } - public Duplicati.Library.Interface.IDeleteResults Delete() + public IDeleteResults Delete() { return RunAction(new DeleteResults(), (result, backendManager) => { @@ -165,7 +165,7 @@ namespace Duplicati.Library.Main }); } - public Duplicati.Library.Interface.IRepairResults Repair(Library.Utility.IFilter filter = null) + public IRepairResults Repair(IFilter filter = null) { return RunAction(new RepairResults(), ref filter, (result, backendManager) => { @@ -173,17 +173,17 @@ namespace Duplicati.Library.Main }); } - public Duplicati.Library.Interface.IListResults List() + public IListResults List() { return List(null, null); } - public Duplicati.Library.Interface.IListResults List(string filterstring) + public IListResults List(string filterstring) { return List(filterstring == null ? null : new string[] { filterstring }, null); } - public Duplicati.Library.Interface.IListResults List(IEnumerable filterstrings, Library.Utility.IFilter filter) + public IListResults List(IEnumerable filterstrings, IFilter filter) { return RunAction(new ListResults(), ref filter, (result, backendManager) => { @@ -191,7 +191,7 @@ namespace Duplicati.Library.Main }); } - public Duplicati.Library.Interface.IListResults ListControlFiles(IEnumerable filterstrings, Library.Utility.IFilter filter) + public IListResults ListControlFiles(IEnumerable filterstrings, IFilter filter) { return RunAction(new ListResults(), ref filter, (result, backendManager) => { @@ -199,17 +199,17 @@ namespace Duplicati.Library.Main }); } - public Duplicati.Library.Interface.IListRemoteResults ListRemote() + public IListRemoteResults ListRemote() { return RunAction(new ListRemoteResults(), (result, backendManager) => { - using (var tf = System.IO.File.Exists(m_options.Dbpath) ? null : new Library.Utility.TempFile()) + using (var tf = File.Exists(m_options.Dbpath) ? null : new Library.Utility.TempFile()) using (var db = new Database.LocalDatabase(((string)tf) ?? m_options.Dbpath, "list-remote", true)) result.SetResult(backendManager.ListAsync(CancellationToken.None).Await()); }); } - public Duplicati.Library.Interface.IListRemoteResults DeleteAllRemoteFiles() + public IListRemoteResults DeleteAllRemoteFiles() { return RunAction(new ListRemoteResults(), (result, backendManager) => { @@ -227,7 +227,7 @@ namespace Duplicati.Library.Main // from the backend. Otherwise, we may accidentally delete non-Duplicati files, or // files from a different Duplicati configuration that points to the same backend location // and uses the same prefix (see issues #2678, #3845, and #4244). - if (System.IO.File.Exists(m_options.Dbpath)) + if (File.Exists(m_options.Dbpath)) { using (LocalDatabase db = new LocalDatabase(m_options.Dbpath, "list-remote", true)) { @@ -256,7 +256,7 @@ namespace Duplicati.Library.Main }); } - public Duplicati.Library.Interface.ICompactResults Compact() + public ICompactResults Compact() { CheckAutoVacuumInterval(); @@ -266,7 +266,7 @@ namespace Duplicati.Library.Main }); } - public Duplicati.Library.Interface.IRecreateDatabaseResults UpdateDatabaseWithVersions(Library.Utility.IFilter filter = null) + public IRecreateDatabaseResults UpdateDatabaseWithVersions(IFilter filter = null) { var filelistfilter = Operation.RestoreHandler.FilterNumberedFilelist(m_options.Time, m_options.Version, singleTimeMatch: true); @@ -277,7 +277,7 @@ namespace Duplicati.Library.Main }); } - public Duplicati.Library.Interface.ICreateLogDatabaseResults CreateLogDatabase(string targetpath) + public ICreateLogDatabaseResults CreateLogDatabase(string targetpath) { var t = new string[] { targetpath }; @@ -287,7 +287,7 @@ namespace Duplicati.Library.Main }); } - public Duplicati.Library.Interface.IListChangesResults ListChanges(string baseVersion, string targetVersion, IEnumerable filterstrings = null, Library.Utility.IFilter filter = null, Action>> callback = null) + public IListChangesResults ListChanges(string baseVersion, string targetVersion, IEnumerable filterstrings = null, IFilter filter = null, Action>> callback = null) { var t = new string[] { baseVersion, targetVersion }; @@ -297,7 +297,7 @@ namespace Duplicati.Library.Main }); } - public Duplicati.Library.Interface.IListAffectedResults ListAffected(List args, Action callback = null) + public IListAffectedResults ListAffected(List args, Action callback = null) { return RunAction(new ListAffectedResults(), (result, backendManager) => { @@ -305,7 +305,7 @@ namespace Duplicati.Library.Main }); } - public Duplicati.Library.Interface.ITestResults Test(long samples = 1) + public ITestResults Test(long samples = 1) { if (!m_options.RawOptions.ContainsKey("full-remote-verification")) m_options.RawOptions["full-remote-verification"] = "true"; @@ -316,7 +316,7 @@ namespace Duplicati.Library.Main }); } - public Library.Interface.ITestFilterResults TestFilter(string[] paths, Library.Utility.IFilter filter = null) + public ITestFilterResults TestFilter(string[] paths, IFilter filter = null) { m_options.RawOptions["dry-run"] = "true"; m_options.RawOptions["dbpath"] = "INVALID!"; @@ -332,7 +332,7 @@ namespace Duplicati.Library.Main } } - public Library.Interface.ISystemInfoResults SystemInfo() + public ISystemInfoResults SystemInfo() { return RunAction(new SystemInfoResults(), (result, backendManager) => { @@ -340,7 +340,7 @@ namespace Duplicati.Library.Main }); } - public Library.Interface.IPurgeFilesResults PurgeFiles(Library.Utility.IFilter filter) + public IPurgeFilesResults PurgeFiles(IFilter filter) { return RunAction(new PurgeFilesResults(), (result, backendManager) => { @@ -348,7 +348,7 @@ namespace Duplicati.Library.Main }); } - public Library.Interface.IListBrokenFilesResults ListBrokenFiles(Library.Utility.IFilter filter, Func callbackhandler = null) + public IListBrokenFilesResults ListBrokenFiles(IFilter filter, Func callbackhandler = null) { return RunAction(new ListBrokenFilesResults(), (result, backendManager) => { @@ -356,7 +356,7 @@ namespace Duplicati.Library.Main }); } - public Library.Interface.IPurgeBrokenFilesResults PurgeBrokenFiles(Library.Utility.IFilter filter) + public IPurgeBrokenFilesResults PurgeBrokenFiles(IFilter filter) { return RunAction(new PurgeBrokenFilesResults(), (result, backendManager) => { @@ -364,7 +364,7 @@ namespace Duplicati.Library.Main }); } - public Library.Interface.ISendMailResults SendMail() + public ISendMailResults SendMail() { m_options.RawOptions["send-mail-level"] = "all"; m_options.RawOptions["send-mail-any-operation"] = "true"; @@ -394,7 +394,7 @@ namespace Duplicati.Library.Main } } - public Library.Interface.IVacuumResults Vacuum() + public IVacuumResults Vacuum() { return RunAction(new VacuumResults(), (result, backendManager) => { @@ -489,7 +489,7 @@ namespace Duplicati.Library.Main { resultSetter.EndTime = DateTime.UtcNow; - if (ex is Library.Interface.OperationAbortException oae) + if (ex is OperationAbortException oae) { // Log this as a normal operation, as the script raising the exception, // has already populated either warning or log messages as required @@ -567,12 +567,12 @@ namespace Duplicati.Library.Main { if (m_options != null && m_options.LoadedModules != null) { - foreach (KeyValuePair mx in m_options.LoadedModules) + foreach (KeyValuePair mx in m_options.LoadedModules) if (mx.Key && mx.Value is IGenericCallbackModule module) try { module.OnFinish(result, exception); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, $"OnFinishError{mx.Key}", ex, "OnFinish callback {0} failed: {1}", mx.Key, ex.Message); } - foreach (KeyValuePair mx in m_options.LoadedModules) + foreach (KeyValuePair mx in m_options.LoadedModules) if (mx.Key && mx.Value is IDisposable disposable) try { disposable.Dispose(); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, $"DisposeError{mx.Key}", ex, "Dispose for {0} failed: {1}", mx.Key, ex.Message); } @@ -614,11 +614,11 @@ namespace Duplicati.Library.Main //Load all generic modules m_options.LoadedModules.Clear(); - foreach (Library.Interface.IGenericModule m in DynamicLoader.GenericLoader.Modules) - m_options.LoadedModules.Add(new KeyValuePair(!m_options.DisableModules.Contains(m.Key, StringComparer.OrdinalIgnoreCase) && (m.LoadAsDefault || m_options.EnableModules.Contains(m.Key, StringComparer.OrdinalIgnoreCase)), m)); + foreach (IGenericModule m in DynamicLoader.GenericLoader.Modules) + m_options.LoadedModules.Add(new KeyValuePair(!m_options.DisableModules.Contains(m.Key, StringComparer.OrdinalIgnoreCase) && (m.LoadAsDefault || m_options.EnableModules.Contains(m.Key, StringComparer.OrdinalIgnoreCase)), m)); // Make the filter read-n-write able in the generic modules - var pristinefilter = string.Join(System.IO.Path.PathSeparator.ToString(), FilterExpression.Serialize(filter)); + var pristinefilter = string.Join(Path.PathSeparator.ToString(), FilterExpression.Serialize(filter)); m_options.RawOptions["filter"] = pristinefilter; // Store the URL connection options separately, as these should only be visible to modules implementing IConnectionModule @@ -642,7 +642,7 @@ namespace Duplicati.Library.Main foreach (var mx in LoadedModules) if (mx.Key) { - if (mx.Value is Library.Interface.IConnectionModule) + if (mx.Value is IConnectionModule) mx.Value.Configure(conopts); else mx.Value.Configure(m_options.RawOptions); @@ -665,15 +665,15 @@ namespace Duplicati.Library.Main // If the filters were changed by a module, read them back in if (pristinefilter != m_options.RawOptions["filter"]) { - filter = FilterExpression.Deserialize(m_options.RawOptions["filter"].Split(new string[] { System.IO.Path.PathSeparator.ToString() }, StringSplitOptions.RemoveEmptyEntries)); + filter = FilterExpression.Deserialize(m_options.RawOptions["filter"].Split(new string[] { Path.PathSeparator.ToString() }, StringSplitOptions.RemoveEmptyEntries)); } m_options.RawOptions.Remove("filter"); // "--filter" is not a supported command line option if (!string.IsNullOrEmpty(m_options.Logfile)) { - var path = System.IO.Path.GetDirectoryName(System.IO.Path.GetFullPath(m_options.Logfile)); - if (!System.IO.Directory.Exists(path)) - System.IO.Directory.CreateDirectory(path); + var path = Path.GetDirectoryName(Path.GetFullPath(m_options.Logfile)); + if (!Directory.Exists(path)) + Directory.CreateDirectory(path); m_logTarget.AddTarget( new Library.Logging.StreamLogDestination(m_options.Logfile), @@ -770,14 +770,14 @@ namespace Duplicati.Library.Main var ropts = new Dictionary(m_options.RawOptions); //Keep a list of all supported options - var supportedOptions = new Dictionary(); + var supportedOptions = new Dictionary(); //There are a few internal options that are not accessible from outside, and thus not listed foreach (string s in Options.InternalOptions) supportedOptions[s] = null; //Figure out what module options are supported in the current setup - var moduleOptions = new List(); + var moduleOptions = new List(); var disabledModuleOptions = new Dictionary(); foreach (var m in m_options.LoadedModules) @@ -785,7 +785,7 @@ namespace Duplicati.Library.Main if (m.Key) moduleOptions.AddRange(m.Value.SupportedCommands); else - foreach (Library.Interface.ICommandLineArgument c in m.Value.SupportedCommands) + foreach (ICommandLineArgument c in m.Value.SupportedCommands) { disabledModuleOptions[c.Name] = m.Value.DisplayName + " (" + m.Value.Key + ")"; @@ -809,7 +809,7 @@ namespace Duplicati.Library.Main DynamicLoader.CompressionLoader.GetSupportedCommands(m_options.CompressionModule) }) { if (l != null) - foreach (Library.Interface.ICommandLineArgument a in l) + foreach (ICommandLineArgument a in l) { if (supportedOptions.ContainsKey(a.Name) && !Options.KnownDuplicates.Contains(a.Name, StringComparer.OrdinalIgnoreCase)) Logging.Log.WriteWarningMessage(LOGTAG, "DuplicateOption", null, Strings.Controller.DuplicateOptionNameWarning(a.Name)); @@ -884,11 +884,11 @@ namespace Duplicati.Library.Main private string[] ExpandInputSources(string[] inputsources, IFilter filter) { if (inputsources == null || inputsources.Length == 0) - throw new Duplicati.Library.Interface.UserInformationException(Strings.Controller.NoSourceFoldersError, "NoSourceFolders"); + throw new UserInformationException(Strings.Controller.NoSourceFoldersError, "NoSourceFolders"); var sources = new List(inputsources.Length); - System.IO.DriveInfo[] drives = null; + DriveInfo[] drives = null; //Make sure they all have the same format and exist foreach (var inputsource in inputsources) @@ -899,11 +899,11 @@ namespace Duplicati.Library.Main { // *: drive paths are only supported on Windows clients // Lazily load the drive info - drives = drives ?? System.IO.DriveInfo.GetDrives(); + drives = drives ?? DriveInfo.GetDrives(); // Replace the drive letter with each available drive string sourcePath = inputsource.Substring(1); - foreach (System.IO.DriveInfo drive in drives) + foreach (DriveInfo drive in drives) { string expandedSource = drive.Name[0] + sourcePath; Logging.Log.WriteVerboseMessage(LOGTAG, "AddingSourcePathFromWildcard", @"Adding source path ""{0}"" due to wildcard source path ""{1}""", expandedSource, inputsource); @@ -930,14 +930,14 @@ namespace Duplicati.Library.Main { // If we aren't allow to have missing sources, throw an exception indicating we couldn't find a drive where this volume is mounted if (!m_options.AllowMissingSource) - throw new Duplicati.Library.Interface.UserInformationException(Strings.Controller.SourceVolumeNameNotFoundError(inputsource, volumeGuid), "MissingSourceFolder"); + throw new UserInformationException(Strings.Controller.SourceVolumeNameNotFoundError(inputsource, volumeGuid), "MissingSourceFolder"); } } else { // If we aren't allow to have missing sources, throw an exception indicating we couldn't find this volume if (!m_options.AllowMissingSource) - throw new Duplicati.Library.Interface.UserInformationException(Strings.Controller.SourceVolumeNameInvalidError(inputsource), "SourceVolumeNameInvalid"); + throw new UserInformationException(Strings.Controller.SourceVolumeNameInvalidError(inputsource), "SourceVolumeNameInvalid"); } } else @@ -966,16 +966,16 @@ namespace Duplicati.Library.Main } // TODO: This expands "C:" to CWD, but not C:\ - source = System.IO.Path.GetFullPath(expandedSource); + source = Path.GetFullPath(expandedSource); } catch (Exception ex) { // Note that we use the original source (with the *) in the error - throw new Duplicati.Library.Interface.UserInformationException(Strings.Controller.InvalidPathError(expandedSource, ex.Message), "InputSourceInvalid", ex); + throw new UserInformationException(Strings.Controller.InvalidPathError(expandedSource, ex.Message), "InputSourceInvalid", ex); } - var fi = new System.IO.FileInfo(source); - var di = new System.IO.DirectoryInfo(source); + var fi = new FileInfo(source); + var di = new DirectoryInfo(source); if (fi.Exists || di.Exists) { foundAnyPaths = true; @@ -1007,10 +1007,10 @@ namespace Duplicati.Library.Main { if (unauthorized) { - throw new System.IO.IOException(Strings.Controller.SourceUnauthorizedError(inputsource)); + throw new IOException(Strings.Controller.SourceUnauthorizedError(inputsource)); } - throw new System.IO.IOException(Strings.Controller.SourceIsMissingError(inputsource)); + throw new IOException(Strings.Controller.SourceIsMissingError(inputsource)); } } @@ -1059,9 +1059,9 @@ namespace Duplicati.Library.Main /// The name of the option to validate /// The value to check /// Null if no errors are found, an error message otherwise - private static string ValidateOptionValue(Library.Interface.ICommandLineArgument arg, string optionname, string value) + private static string ValidateOptionValue(ICommandLineArgument arg, string optionname, string value) { - if (arg.Type == Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Enumeration) + if (arg.Type == CommandLineArgument.ArgumentType.Enumeration) { bool found = false; foreach (string v in arg.ValidValues ?? new string[0]) @@ -1075,7 +1075,7 @@ namespace Duplicati.Library.Main return Strings.Controller.UnsupportedEnumerationValue(optionname, value, arg.ValidValues ?? new string[0]); } - else if (arg.Type == Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Flags) + else if (arg.Type == CommandLineArgument.ArgumentType.Flags) { bool validatedAllFlags = false; var flags = (value ?? string.Empty).ToLowerInvariant().Split(new[] { "," }, StringSplitOptions.None).Select(flag => flag.Trim()).Distinct(); @@ -1097,23 +1097,23 @@ namespace Duplicati.Library.Main return Strings.Controller.UnsupportedFlagsValue(optionname, value, validFlags); } } - else if (arg.Type == Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean) + else if (arg.Type == CommandLineArgument.ArgumentType.Boolean) { if (!string.IsNullOrEmpty(value) && Library.Utility.Utility.ParseBool(value, true) != Library.Utility.Utility.ParseBool(value, false)) return Strings.Controller.UnsupportedBooleanValue(optionname, value); } - else if (arg.Type == Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Integer) + else if (arg.Type == CommandLineArgument.ArgumentType.Integer) { if (!long.TryParse(value, out _)) return Strings.Controller.UnsupportedIntegerValue(optionname, value); } - else if (arg.Type == Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Path) + else if (arg.Type == CommandLineArgument.ArgumentType.Path) { - foreach (string p in value.Split(System.IO.Path.DirectorySeparatorChar)) - if (p.IndexOfAny(System.IO.Path.GetInvalidPathChars()) >= 0) + foreach (string p in value.Split(Path.DirectorySeparatorChar)) + if (p.IndexOfAny(Path.GetInvalidPathChars()) >= 0) return Strings.Controller.UnsupportedPathValue(optionname, p); } - else if (arg.Type == Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Size) + else if (arg.Type == CommandLineArgument.ArgumentType.Size) { try { @@ -1127,7 +1127,7 @@ namespace Duplicati.Library.Main if (!string.IsNullOrWhiteSpace(value) && char.IsDigit(value.Last())) return Strings.Controller.NonQualifiedSizeValue(optionname, value); } - else if (arg.Type == Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Timespan) + else if (arg.Type == CommandLineArgument.ArgumentType.Timespan) { try { diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 9c8262ed4..b9ac5928d 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -55,7 +55,6 @@ namespace Duplicati.Library.Main.Operation private static readonly string LOGTAG = Logging.Log.LogTagFromType(); private readonly Options m_options; - private readonly string m_backendurl; private LocalBackupDatabase m_database; private System.Data.IDbTransaction m_transaction; @@ -68,11 +67,10 @@ namespace Duplicati.Library.Main.Operation public readonly CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); - public BackupHandler(string backendurl, Options options, BackupResults results) + public BackupHandler(Options options, BackupResults results) { m_options = options; m_result = results; - m_backendurl = backendurl; m_taskReader = results.TaskControl; if (options.AllowPassphraseChange) From 5f8ff74bd562382a0b1687ee1d03fc647f0119c5 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sat, 8 Mar 2025 15:25:55 +0100 Subject: [PATCH 322/362] Streamlined handling of previous backup's fileset and dlist file if it failed to upload. --- .../Operation/Backup/UploadRealFilelist.cs | 4 +- .../Backup/UploadSyntheticFilelist.cs | 36 +++++++------ .../Library/Main/Operation/BackupHandler.cs | 52 +++++++++---------- .../Main/Operation/FilelistProcessor.cs | 6 +-- .../Library/Main/Operation/RepairHandler.cs | 5 +- 5 files changed, 52 insertions(+), 51 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Backup/UploadRealFilelist.cs b/Duplicati/Library/Main/Operation/Backup/UploadRealFilelist.cs index 44df554ba..3d8431f62 100644 --- a/Duplicati/Library/Main/Operation/Backup/UploadRealFilelist.cs +++ b/Duplicati/Library/Main/Operation/Backup/UploadRealFilelist.cs @@ -32,7 +32,7 @@ internal static class UploadRealFilelist /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(UploadRealFilelist)); - public static async Task Run(BackupResults result, BackupDatabase db, IBackendManager backendManager, Options options, FilesetVolumeWriter filesetvolume, long filesetid, Common.ITaskReader taskreader) + public static async Task Run(BackupResults result, BackupDatabase db, IBackendManager backendManager, Options options, FilesetVolumeWriter filesetvolume, long filesetid, Common.ITaskReader taskreader, bool lastWasPartial) { // We ignore the stop signal, but not the pause and terminate await taskreader.ProgressRendevouz().ConfigureAwait(false); @@ -47,7 +47,7 @@ internal static class UploadRealFilelist result.AddedSymlinks + result.ModifiedSymlinks + result.DeletedSymlinks; //Changes in the filelist triggers a filelist upload - if (options.UploadUnchangedBackups || changeCount > 0) + if (options.UploadUnchangedBackups || changeCount > 0 || lastWasPartial) { using (new Logging.Timer(LOGTAG, "UploadNewFileset", "Uploading a new fileset")) { diff --git a/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs b/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs index e78935340..4e1bfc9e0 100644 --- a/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs +++ b/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs @@ -19,6 +19,7 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +using Duplicati.Library.Main.Database; using Duplicati.Library.Main.Operation.Common; using Duplicati.Library.Main.Volumes; using System; @@ -37,27 +38,26 @@ namespace Duplicati.Library.Main.Operation.Backup /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(UploadSyntheticFilelist)); - public static async Task Run(BackupDatabase database, Options options, BackupResults result, ITaskReader taskreader, IBackendManager backendManager, string lastTempFilelist, long lastTempFilesetId) + /// + /// This method is used to generate and upload a synthetic file list, if needed + /// + /// The database to use + /// The options to use + /// The backup results + /// The task reader to use + /// The backend manager to use + /// The last temporary file list volume + /// + public static async Task Run(BackupDatabase database, Options options, BasicResults result, ITaskReader taskreader, IBackendManager backendManager, RemoteVolumeEntry lastTempFilelist) { // Check if we should upload a synthetic filelist - if (options.DisableSyntheticFilelist || string.IsNullOrWhiteSpace(lastTempFilelist) || lastTempFilesetId <= 0) + if (options.DisableSyntheticFilelist || string.IsNullOrWhiteSpace(lastTempFilelist.Name) || lastTempFilelist.ID <= 0) return; - // Check that we still need to process this after the cleanup has performed its duties - var syntbase = await database.GetRemoteVolumeFromFilesetIDAsync(lastTempFilesetId); - - // If we do not have a valid entry, warn and quit - if (syntbase.Name == null) - { - // TODO: If the repair succeeds, this could give a false warning? - Logging.Log.WriteWarningMessage(LOGTAG, "MissingTemporaryFilelist", null, "Expected there to be a temporary fileset for synthetic filelist ({0}, {1}), but none was found?", lastTempFilesetId, lastTempFilelist); - return; - } - // Files is missing or repaired - if (syntbase.State != RemoteVolumeState.Uploading && syntbase.State != RemoteVolumeState.Temporary) + if (lastTempFilelist.State != RemoteVolumeState.Uploading && lastTempFilelist.State != RemoteVolumeState.Temporary) { - Logging.Log.WriteInformationMessage(LOGTAG, "SkippingSyntheticListUpload", "Skipping synthetic upload because temporary fileset appers to be complete: ({0}, {1}, {2})", lastTempFilesetId, lastTempFilelist, syntbase.State); + Logging.Log.WriteInformationMessage(LOGTAG, "SkippingSyntheticListUpload", "Skipping synthetic upload because temporary fileset appers to be complete: ({0}, {1}, {2})", lastTempFilelist.ID, lastTempFilelist.Name, lastTempFilelist.State); return; } @@ -66,9 +66,7 @@ namespace Duplicati.Library.Main.Operation.Backup var incompleteFilesets = (await database.GetIncompleteFilesetsAsync()).OrderBy(x => x.Value).ToList(); if (!incompleteFilesets.Any()) - { return; - } if (!await taskreader.ProgressRendevouz().ConfigureAwait(false)) return; @@ -124,6 +122,10 @@ namespace Duplicati.Library.Main.Operation.Backup return; await database.UpdateRemoteVolumeAsync(fsw.RemoteFilename, RemoteVolumeState.Uploading, -1, null); + // If the previous filelist was not uploaded, we register it for deletion, as we have created a new synthetic one + // Because it is registered as "Deleting", it will be removed from remote storage by the cleanup process if it exists + if (!string.IsNullOrWhiteSpace(lastTempFilelist.Name) && (lastTempFilelist.State == RemoteVolumeState.Uploading || lastTempFilelist.State == RemoteVolumeState.Temporary)) + await database.UpdateRemoteVolumeAsync(lastTempFilelist.Name, RemoteVolumeState.Deleting, -1, null); await database.CommitTransactionAsync("CommitUpdateFilelistVolume"); await backendManager.PutAsync(fsw, null, null, false, taskreader.ProgressToken); diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index b9ac5928d..297947e5f 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -52,15 +52,14 @@ namespace Duplicati.Library.Main.Operation /// /// The tag used for logging /// - private static readonly string LOGTAG = Logging.Log.LogTagFromType(); + private static readonly string LOGTAG = Log.LogTagFromType(); private readonly Options m_options; private LocalBackupDatabase m_database; - private System.Data.IDbTransaction m_transaction; + private IDbTransaction m_transaction; - private Library.Utility.IFilter m_filter; - private Library.Utility.IFilter m_sourceFilter; + private IFilter m_filter; private readonly BackupResults m_result; private readonly ITaskReader m_taskReader; @@ -266,8 +265,7 @@ namespace Duplicati.Library.Main.Operation private sealed record PreBackupVerifyResult( LocalBackupDatabase Database, - string LastTempFilelist, - long LastTempFilesetId + RemoteVolumeEntry LastTempFilelist ); /// @@ -292,8 +290,7 @@ namespace Duplicati.Library.Main.Operation LocalBackupDatabase database = null; // If we have an interrupted backup, grab the fileset - string lastTempFilelist = null; - long lastTempFilesetId = -1; + RemoteVolumeEntry lastTempFilelist = default; using (new Logging.Timer(LOGTAG, "PreBackupVerify", "PreBackupVerify")) { @@ -322,15 +319,9 @@ namespace Duplicati.Library.Main.Operation // Make sure the database is sane await db.VerifyConsistencyAsync(options.Blocksize, options.BlockhashSize, !options.DisableFilelistConsistencyChecks); + // Guard the last filelist if (!options.DisableSyntheticFilelist) - { - var candidates = (await db.GetIncompleteFilesetsAsync()).OrderBy(x => x.Value).ToArray(); - if (candidates.Any()) - { - lastTempFilesetId = candidates.Last().Key; - lastTempFilelist = database.GetRemoteVolumeFromFilesetID(lastTempFilesetId).Name; - } - } + lastTempFilelist = database.GetLastIncompleteFilesetVolume(null); } try @@ -342,7 +333,7 @@ namespace Duplicati.Library.Main.Operation } else { - await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, [lastTempFilelist], [], logErrors: false, verifyMode: FilelistProcessor.VerifyMode.VerifyAndClean).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, [lastTempFilelist.Name], [], logErrors: false, verifyMode: FilelistProcessor.VerifyMode.VerifyAndClean).ConfigureAwait(false); } } catch (RemoteListVerificationException ex) @@ -362,13 +353,13 @@ namespace Duplicati.Library.Main.Operation result.SetDatabase(database); Log.WriteInformationMessage(LOGTAG, "BackendCleanupFinished", "Backend cleanup finished, retrying verification"); - await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, [lastTempFilelist], [], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, [lastTempFilelist.Name], [], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); } else throw; } - return new PreBackupVerifyResult(database, lastTempFilelist, lastTempFilesetId); + return new PreBackupVerifyResult(database, lastTempFilelist); } catch { @@ -382,7 +373,7 @@ namespace Duplicati.Library.Main.Operation /// /// Performs the bulk of work by starting all relevant processes /// - private static async Task RunMainOperation(Backup.Channels channels, ISourceProvider source, UsnJournalService journalService, Backup.BackupDatabase database, IBackendManager backendManager, Backup.BackupStatsCollector stats, Options options, IFilter sourcefilter, IFilter filter, BackupResults result, ITaskReader taskreader, long filesetid, long lastfilesetid) + private static async Task RunMainOperation(Backup.Channels channels, ISourceProvider source, UsnJournalService journalService, Backup.BackupDatabase database, IBackendManager backendManager, Backup.BackupStatsCollector stats, Options options, IFilter filter, BackupResults result, ITaskReader taskreader, long filesetid, long lastfilesetid) { using (new Logging.Timer(LOGTAG, "BackupMainOperation", "BackupMainOperation")) { @@ -597,19 +588,24 @@ namespace Duplicati.Library.Main.Operation return -1; } - public async Task RunAsync(string[] sources, IBackendManager backendManager, Library.Utility.IFilter filter) + public async Task RunAsync(string[] sources, IBackendManager backendManager, IFilter filter) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Begin); // Do a remote verification, unless disabled - var (database, lastTempFilelist, lastTempFilesetId) = await PreBackupVerify(m_options, m_result, backendManager); + var (database, lastTempFilelist) = await PreBackupVerify(m_options, m_result, backendManager); + var lastTempVolumeIncomplete = false; + + if (!string.IsNullOrWhiteSpace(lastTempFilelist.Name)) + { + if (lastTempFilelist.State == RemoteVolumeState.Temporary || lastTempFilelist.State == RemoteVolumeState.Uploading) + lastTempVolumeIncomplete = true; + } Backup.Channels channels = new(); // If there is no filter, we set an empty filter to simplify the code - // If there is a filter, we make sure that the sources are included m_filter = filter ?? new Library.Utility.FilterExpression(); - m_sourceFilter = new Library.Utility.FilterExpression(sources, true); Task parallelScanner = null; try @@ -631,7 +627,7 @@ namespace Duplicati.Library.Main.Operation try { // If the previous backup was interrupted, send a synthetic list - await Backup.UploadSyntheticFilelist.Run(db, m_options, m_result, m_result.TaskControl, backendManager, lastTempFilelist, lastTempFilesetId); + await Backup.UploadSyntheticFilelist.Run(db, m_options, m_result, m_result.TaskControl, backendManager, lastTempFilelist); // Grab the previous backup ID, if any var prevfileset = m_database.FilesetTimes.FirstOrDefault(); @@ -672,7 +668,7 @@ namespace Duplicati.Library.Main.Operation // Run the backup operation if (await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await RunMainOperation(channels, source, journalService, db, backendManager, stats, m_options, m_sourceFilter, m_filter, m_result, m_result.TaskControl, filesetid, lastfilesetid).ConfigureAwait(false); + await RunMainOperation(channels, source, journalService, db, backendManager, stats, m_options, m_filter, m_result, m_result.TaskControl, filesetid, lastfilesetid).ConfigureAwait(false); } } finally @@ -690,7 +686,7 @@ namespace Duplicati.Library.Main.Operation await db.VerifyConsistencyAsync(m_options.Blocksize, m_options.BlockhashSize, false); // Send the actual filelist - await Backup.UploadRealFilelist.Run(m_result, db, backendManager, m_options, filesetvolume, filesetid, m_result.TaskControl); + await Backup.UploadRealFilelist.Run(m_result, db, backendManager, m_options, filesetvolume, filesetid, m_result.TaskControl, lastTempVolumeIncomplete); // Wait for upload completion m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_WaitForUpload); @@ -750,7 +746,7 @@ namespace Duplicati.Library.Main.Operation if (m_options.NoBackendverification) await UpdateStorageStatsFromDatabase(m_result, m_database, m_options, backendManager, m_taskReader.ProgressToken).ConfigureAwait(false); else - await PostBackupVerification(filesetvolume.RemoteFilename, lastTempFilelist, backendManager).ConfigureAwait(false); + await PostBackupVerification(filesetvolume.RemoteFilename, lastTempFilelist.Name, backendManager).ConfigureAwait(false); } } diff --git a/Duplicati/Library/Main/Operation/FilelistProcessor.cs b/Duplicati/Library/Main/Operation/FilelistProcessor.cs index 2fb1690b5..37aced690 100644 --- a/Duplicati/Library/Main/Operation/FilelistProcessor.cs +++ b/Duplicati/Library/Main/Operation/FilelistProcessor.cs @@ -413,14 +413,14 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteWarningMessage(LOGTAG, "UnknownFileState", null, "unknown state for remote file listed as {0}: {1}", i.State, i.Name); break; } - - await backendManager.WaitForEmptyAsync(database, null, CancellationToken.None).ConfigureAwait(false); } + await backendManager.WaitForEmptyAsync(database, null, CancellationToken.None).ConfigureAwait(false); + // cleanup deleted volumes in DB en block if (verifyMode == VerifyMode.VerifyStrict && missingUploadingVolumes.Count > 0) throw new RemoteListVerificationException($"The remote volumes {string.Join(", ", missingUploadingVolumes)} are supposed to be deleted, but strict mode is on. Try running the \"repair\" command", "DeleteDuringStrictMode"); - else if (verifyMode == VerifyMode.VerifyAndClean) + else if (verifyMode == VerifyMode.VerifyAndClean || verifyMode == VerifyMode.VerifyStrict) { database.RemoveRemoteVolumes(missingUploadingVolumes.Concat(temporaryAndDeletedVolumes), null); // Clear the flag after we have cleaned up diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 7dbc1e6ec..4b280b1f6 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -134,7 +134,10 @@ namespace Duplicati.Library.Main.Operation if (db.RepairInProgress) throw new UserInformationException("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsInRepairState"); - var tp = await FilelistProcessor.RemoteListAnalysis(backendManager, m_options, db, m_result.BackendWriter, null, null, FilelistProcessor.VerifyMode.VerifyAndCleanForced).ConfigureAwait(false); + // If the last backup failed, guard the incomplete fileset, so we can create a synthetic filelist + var lastTempFilelist = db.GetLastIncompleteFilesetVolume(null); + var tp = await FilelistProcessor.RemoteListAnalysis(backendManager, m_options, db, m_result.BackendWriter, [lastTempFilelist.Name], null, FilelistProcessor.VerifyMode.VerifyAndCleanForced).ConfigureAwait(false); + var buffer = new byte[m_options.Blocksize]; var hashsize = HashFactory.HashSizeBytes(m_options.BlockHashAlgorithm); From 1de0a924134e3797cdaf2ac597f676dc5ed28527 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sat, 8 Mar 2025 15:28:02 +0100 Subject: [PATCH 323/362] Strategy is now to disallow compact/delete/purge if the remote state contains partial data. Backup will auto-repair, and user otherwise need to call "repair" to recover the state, before invoking compact/delete/purge. This is done to prevent cases where the remote state contiains partial data and the automatic cleanup process would remove remote files. --- Duplicati/Library/Main/Operation/CompactHandler.cs | 2 +- Duplicati/Library/Main/Operation/DeleteHandler.cs | 2 +- Duplicati/Library/Main/Operation/PurgeFilesHandler.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Main/Operation/CompactHandler.cs b/Duplicati/Library/Main/Operation/CompactHandler.cs index 30ae68618..5144211dc 100644 --- a/Duplicati/Library/Main/Operation/CompactHandler.cs +++ b/Duplicati/Library/Main/Operation/CompactHandler.cs @@ -104,7 +104,7 @@ namespace Duplicati.Library.Main.Operation { // Workaround where we allow a running backendmanager to be used if (!hasVerifiedBackend) - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, true, FilelistProcessor.VerifyMode.VerifyAndClean, transaction).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, true, FilelistProcessor.VerifyMode.VerifyStrict, transaction).ConfigureAwait(false); var newvol = new BlockVolumeWriter(m_options); newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, transaction); diff --git a/Duplicati/Library/Main/Operation/DeleteHandler.cs b/Duplicati/Library/Main/Operation/DeleteHandler.cs index 477574311..cc19cf748 100644 --- a/Duplicati/Library/Main/Operation/DeleteHandler.cs +++ b/Duplicati/Library/Main/Operation/DeleteHandler.cs @@ -86,7 +86,7 @@ namespace Duplicati.Library.Main.Operation { CancellationToken cancellationToken = CancellationToken.None; if (!hasVerifiedBackend) - FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyAndClean, transaction).Await(); + FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict, transaction).Await(); IListResultFileset[] filesets = db.FilesetsWithBackupVersion.ToArray(); List versionsToDelete = diff --git a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs index 16506287d..718793687 100644 --- a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs @@ -91,7 +91,7 @@ namespace Duplicati.Library.Main.Operation if (m_options.NoBackendverification) FilelistProcessor.VerifyLocalList(backendManager, db, cancellationToken).Await(); else - FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, null, null, logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyAndClean).Await(); + FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, null, null, logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).Await(); } var filesets = db.FilesetTimes.OrderByDescending(x => x.Value).ToArray(); From ceff365ff8ef4b6f7d45d44ec230a52f11ae1084 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sat, 8 Mar 2025 15:30:17 +0100 Subject: [PATCH 324/362] Fixed compact count to use sample count, instead of file count. --- Duplicati/Library/Main/Operation/BackupHandler.cs | 8 +++++++- Duplicati/Library/Main/Strings.cs | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index ba4165515..fedff204b 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -509,7 +509,13 @@ namespace Duplicati.Library.Main.Operation await FilelistProcessor.VerifyRemoteList(backendManager, m_options, m_database, m_result.BackendWriter, new string[] { currentFilelistVolume }).ConfigureAwait(false); await backendManager.WaitForEmptyAsync(m_database, m_transaction, m_taskReader.ProgressToken); - long remoteVolumeCount = m_database.GetRemoteVolumes().LongCount(x => x.State == RemoteVolumeState.Verified); + // Calculate the number of samples to test, using the largest number of file of a given type + long remoteVolumeCount = m_database.GetRemoteVolumes() + .Where(x => x.State == RemoteVolumeState.Verified) + .GroupBy(x => x.Type) + .Select(x => x.LongCount()) + .Max(); + long samplesToTest = Math.Max(m_options.BackupTestSampleCount, (long)Math.Round(remoteVolumeCount * (m_options.BackupTestPercentage / 100m), MidpointRounding.AwayFromZero)); if (samplesToTest > 0 && remoteVolumeCount > 0) { diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs index 9ba7b35c5..c6d5f018e 100644 --- a/Duplicati/Library/Main/Strings.cs +++ b/Duplicati/Library/Main/Strings.cs @@ -230,7 +230,7 @@ namespace Duplicati.Library.Main.Strings public static string UploadverificationfileShort { get { return LC.L(@"Determine if verification files are uploaded"); } } public static string BackendtestsamplesLong(string optionname) { return LC.L(@"After a backup is completed, some (dblock, dindex, dlist) files from the remote backend are selected for verification. Use this option to change how many. If the option --{0} is also provided, the number of samples tested is the maximum implied by the two options. If this value is set to 0 or the option --{1} is set, no remote files are verified.", "backup-test-percentage", optionname); } public static string BackendtestsamplesShort { get { return LC.L(@"The number of samples to test after a backup"); } } - public static string BackendtestpercentageLong { get { return LC.L(@"After a backup is completed, some (dblock, dindex, dlist) files from the remote backend are selected for verification. Use this option to specify the percentage (between 0 and 100) of files to test. If the option --{0} is also provided, the number of samples tested is the maximum implied by the two options. If the option --{1} is provided, no remote files are verified.", "backup-test-samples", "no-backend-verification"); } } + public static string BackendtestpercentageLong { get { return LC.L(@"After a backup is completed, some samples (one sample is 1 dblock, 1 dindex, and 1 dlist) files from the remote backend are selected for verification. Use this option to specify the percentage (between 0 and 100) of samples to test. If the option --{0} is also provided, the number of samples tested is the maximum implied by the two options. If the option --{1} is provided, no remote files are verified.", "backup-test-samples", "no-backend-verification"); } } public static string BackendtestpercentageShort { get { return LC.L(@"The percentage of samples to test after a backup"); } } public static string FullremoteverificationLong(string optionname) { return LC.L(@"After a backup is completed, some (dblock, dindex, dlist) files from the remote backend are selected for verification. Use this option to turn on full verification, which will decrypt the files and examine the insides of each volume, instead of simply verifying the external hash. If the option --{0} is set, no remote files are verified. This option is automatically set when then verification is performed directly. ListAndIndexes is like True but only dlist and index volumes are handled.", optionname); } public static string FullremoteverificationShort { get { return LC.L(@"Activate in-depth verification of files"); } } From 63464342cdd0cfbb7f71db11e929f8cab213d5b5 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Sun, 9 Mar 2025 08:16:10 +0100 Subject: [PATCH 325/362] Changed the request timeout to issue a warning, rather than an error --- Duplicati/Library/Main/Operation/Restore/BlockManager.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs index 140032238..74f71a102 100644 --- a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs @@ -273,14 +273,13 @@ namespace Duplicati.Library.Main.Operation.Restore { Logging.Log.WriteExplicitMessage(LOGTAG, "BlockCacheGet", "Requesting block {0} from volume {1}", block_request.BlockID, block_request.VolumeID); - // TODO add a debugging timeout. As this can potentially take a long time in a real setup with a slow machine and slew connection. + // Adding a timeout to warn of potential deadlocks. Task.Run(async () => { await Task.Delay(TimeSpan.FromMinutes(5)).ConfigureAwait(false); if (!tcs.Task.IsCompleted) { - tcs.SetException(new TimeoutException("Block request timeout")); - Logging.Log.WriteErrorMessage(LOGTAG, "BlockRequestTimeout", null, $"Block request for block {block_request.BlockID} timed out"); + Logging.Log.WriteWarningMessage(LOGTAG, "BlockRequestTimeout", null, "Block request for block {0} has been in flight for over 5 minutes. This may be a deadlock.", block_request.BlockID); } }); From 599203a2693770167d75cf05d4f405d608d1a7a8 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sun, 9 Mar 2025 22:01:01 +0100 Subject: [PATCH 326/362] Added check to see that injected failure is actually triggered --- Duplicati/UnitTest/DisruptionTests.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Duplicati/UnitTest/DisruptionTests.cs b/Duplicati/UnitTest/DisruptionTests.cs index 30346e89d..32c0e064f 100644 --- a/Duplicati/UnitTest/DisruptionTests.cs +++ b/Duplicati/UnitTest/DisruptionTests.cs @@ -879,6 +879,7 @@ namespace Duplicati.UnitTest // We expect to upload 7 volumes, and fail on the last one var count = 7; + var failed = false; // Fail when uploading the dlist file DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) => @@ -891,6 +892,7 @@ namespace Duplicati.UnitTest { // Allow others to progress Thread.Sleep(2000); + failed = true; return true; } } @@ -898,7 +900,10 @@ namespace Duplicati.UnitTest else { if (action == operation && remotename.Contains(".dlist.")) + { + failed = true; return true; + } } return false; @@ -907,6 +912,8 @@ namespace Duplicati.UnitTest using (var c = new Library.Main.Controller(failtarget, testopts, null)) Assert.Throws(() => c.Backup(new string[] { DATAFOLDER })); + Assert.That(failed, Is.True, "Failed to fail the upload"); + // Make spacing for the next backup Thread.Sleep(3000); From 83dc334069af046a7981874270c3c7e36c8dd2e2 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 10 Mar 2025 05:44:31 +0100 Subject: [PATCH 327/362] Improve error handling in restore operations by retiring inputs and outputs on exceptions --- .../Library/Main/Operation/Restore/BlockManager.cs | 11 ++++++++++- .../Main/Operation/Restore/VolumeDecompressor.cs | 2 ++ .../Library/Main/Operation/Restore/VolumeDecryptor.cs | 2 ++ .../Main/Operation/Restore/VolumeDownloader.cs | 2 ++ 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs index 74f71a102..0bc8787ce 100644 --- a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs @@ -447,10 +447,12 @@ namespace Duplicati.Library.Main.Operation.Restore } catch (Exception ex) { - Logging.Log.WriteWarningMessage(LOGTAG, "VolumeConsumerError", ex, "Error in volume consumer"); + Logging.Log.WriteErrorMessage(LOGTAG, "VolumeConsumerError", ex, "Error in volume consumer"); + self.Input.Retire(); // Cancel any remaining readers - although there shouldn't be any. cache.CancelAll(); + cache.Retire(); } }); @@ -498,6 +500,13 @@ namespace Duplicati.Library.Main.Operation.Restore cache.Retire(); } + catch (Exception ex) + { + Logging.Log.WriteErrorMessage(LOGTAG, "BlockHandlerError", ex, "Error in block handler"); + req.Retire(); + res.Retire(); + cache.Retire(); + } })).ToArray(); await Task.WhenAll([volume_consumer, .. block_handlers]).ConfigureAwait(false); diff --git a/Duplicati/Library/Main/Operation/Restore/VolumeDecompressor.cs b/Duplicati/Library/Main/Operation/Restore/VolumeDecompressor.cs index 2c2c4da6f..0d2de024f 100644 --- a/Duplicati/Library/Main/Operation/Restore/VolumeDecompressor.cs +++ b/Duplicati/Library/Main/Operation/Restore/VolumeDecompressor.cs @@ -110,6 +110,8 @@ namespace Duplicati.Library.Main.Operation.Restore catch (Exception ex) { Logging.Log.WriteErrorMessage(LOGTAG, "DecompressionError", ex, "Error during decompression"); + self.Input.Retire(); + self.Output.Retire(); throw; } }); diff --git a/Duplicati/Library/Main/Operation/Restore/VolumeDecryptor.cs b/Duplicati/Library/Main/Operation/Restore/VolumeDecryptor.cs index 22fb03e88..0d769c256 100644 --- a/Duplicati/Library/Main/Operation/Restore/VolumeDecryptor.cs +++ b/Duplicati/Library/Main/Operation/Restore/VolumeDecryptor.cs @@ -90,6 +90,8 @@ namespace Duplicati.Library.Main.Operation.Restore catch (Exception ex) { Logging.Log.WriteErrorMessage(LOGTAG, "DecryptionError", ex, "Error during decryption"); + self.Input.Retire(); + self.Output.Retire(); throw; } }); diff --git a/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs b/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs index 2f2b543f9..b615c4d8e 100644 --- a/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs +++ b/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs @@ -107,6 +107,8 @@ namespace Duplicati.Library.Main.Operation.Restore catch (Exception ex) { Logging.Log.WriteErrorMessage(LOGTAG, "DownloadError", ex, "Error during download"); + self.Input.Retire(); + self.Output.Retire(); throw; } }); From 33c133fad153426d12336a6351228fabe8f027b6 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 10 Mar 2025 06:39:50 +0100 Subject: [PATCH 328/362] Added a sleep in BorderTests, as fast machines will trigger two backups in quick succession, which will cause the test to fail. --- Duplicati/UnitTest/BorderTests.cs | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/Duplicati/UnitTest/BorderTests.cs b/Duplicati/UnitTest/BorderTests.cs index 0e4bc93fc..9bdd5b098 100644 --- a/Duplicati/UnitTest/BorderTests.cs +++ b/Duplicati/UnitTest/BorderTests.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -270,6 +270,7 @@ namespace Duplicati.UnitTest File.WriteAllBytes(Path.Combine(DATAFOLDER, "c" + k.Key), data.Take(k.Value).ToArray()); } + System.Threading.Tasks.Task.Delay(1000).Wait(); using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) { From 6f7f2757a9ed4b93ebdb6184fbae429b8dd0a3c5 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 10 Mar 2025 10:22:34 +0100 Subject: [PATCH 329/362] Rewrote timeout monitoring --- .../Main/Operation/Restore/BlockManager.cs | 41 +++++++++---------- .../Main/Operation/Restore/FileProcessor.cs | 8 ++-- .../Library/Main/Operation/RestoreHandler.cs | 4 +- 3 files changed, 26 insertions(+), 27 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs index 0bc8787ce..9294b0e1d 100644 --- a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs @@ -250,7 +250,7 @@ namespace Duplicati.Library.Main.Operation.Restore /// /// The requested block. /// A `Task` holding the data block. - public Task Get(BlockRequest block_request) + public async Task Get(BlockRequest block_request) { Logging.Log.WriteExplicitMessage(LOGTAG, "BlockCacheGet", "Getting block {0} from cache", block_request.BlockID); @@ -261,9 +261,7 @@ namespace Duplicati.Library.Main.Operation.Restore // Check if the block is already in the cache, and return it if it is. if (m_block_cache.TryGetValue(block_request.BlockID, out byte[] value)) - { - return Task.FromResult(value); - } + return value; // If the block is not in the cache, request it from the volume. sw_get_wait?.Start(); @@ -273,22 +271,23 @@ namespace Duplicati.Library.Main.Operation.Restore { Logging.Log.WriteExplicitMessage(LOGTAG, "BlockCacheGet", "Requesting block {0} from volume {1}", block_request.BlockID, block_request.VolumeID); - // Adding a timeout to warn of potential deadlocks. - Task.Run(async () => - { - await Task.Delay(TimeSpan.FromMinutes(5)).ConfigureAwait(false); - if (!tcs.Task.IsCompleted) - { - Logging.Log.WriteWarningMessage(LOGTAG, "BlockRequestTimeout", null, "Block request for block {0} has been in flight for over 5 minutes. This may be a deadlock.", block_request.BlockID); - } - }); - // We are the first to request this block - m_volume_request.Write(block_request); - } - sw_get_wait?.Stop(); + await m_volume_request.WriteAsync(block_request).ConfigureAwait(false); - return new_tcs.Task; + sw_get_wait?.Stop(); + + // Add a timeout monitor + using var tcs1 = new CancellationTokenSource(); + var t = await Task.WhenAny(Task.Delay(TimeSpan.FromMinutes(5), tcs1.Token), new_tcs.Task).ConfigureAwait(false); + if (t != new_tcs.Task) + Logging.Log.WriteWarningMessage(LOGTAG, "BlockRequestTimeout", null, "Block request for block {0} has been in flight for over 5 minutes. This may be a deadlock.", block_request.BlockID); + } + else + { + sw_get_wait?.Stop(); + } + + return await new_tcs.Task; } /// @@ -402,7 +401,7 @@ namespace Duplicati.Library.Main.Operation.Restore /// The restore options. /// The channels for reading block requests from the `FileProcessor`. /// The channels for writing block responses back to the `FileProcessor`. - public static Task Run(Channels channels, LocalRestoreDatabase db, Options options, IChannel[] fp_requests, IChannel[] fp_responses) + public static Task Run(Channels channels, LocalRestoreDatabase db, Options options, IChannel[] fp_requests, IChannel>[] fp_responses) { return AutomationExtensions.RunTask( new @@ -480,11 +479,11 @@ namespace Duplicati.Library.Main.Operation.Restore else { sw_get?.Start(); - var data = await cache.Get(block_request).ConfigureAwait(false); + var datatask = cache.Get(block_request); sw_get?.Stop(); sw_resp?.Start(); - await res.WriteAsync(data).ConfigureAwait(false); + await res.WriteAsync(datatask).ConfigureAwait(false); sw_resp?.Stop(); } } diff --git a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs index d393c5f71..7ef1b1fe0 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs @@ -58,7 +58,7 @@ namespace Duplicati.Library.Main.Operation.Restore /// The channel to receive blocks from the block manager. /// The restore options. /// The restore results. - public static Task Run(Channels channels, LocalRestoreDatabase db, IChannel block_request, IChannel block_response, Options options, RestoreResults results) + public static Task Run(Channels channels, LocalRestoreDatabase db, IChannel block_request, IChannel> block_response, Options options, RestoreResults results) { return AutomationExtensions.RunTask( new @@ -285,7 +285,7 @@ namespace Duplicati.Library.Main.Operation.Restore { // Read the block from the response and issue a new request, if more blocks are missing sw_resp?.Start(); - var data = await block_response.ReadAsync().ConfigureAwait(false); + var data = await (await block_response.ReadAsync().ConfigureAwait(false)).ConfigureAwait(false); sw_resp?.Stop(); sw_req?.Start(); @@ -535,7 +535,7 @@ namespace Duplicati.Library.Main.Operation.Restore /// The stopwatch for internal profiling of the general processing. /// The stopwatch for internal profiling of the block requests. /// The stopwatch for internal profiling of the block responses. - private static async Task RestoreMetadata(LocalRestoreDatabase db, FileRequest file, IChannel block_request, IChannel block_response, Options options, Stopwatch sw_meta, Stopwatch sw_work, Stopwatch sw_req, Stopwatch sw_resp) + private static async Task RestoreMetadata(LocalRestoreDatabase db, FileRequest file, IChannel block_request, IChannel> block_response, Options options, Stopwatch sw_meta, Stopwatch sw_work, Stopwatch sw_req, Stopwatch sw_resp) { sw_meta?.Start(); var blocks = db.GetMetadataBlocksFromFile(file.ID); @@ -560,7 +560,7 @@ namespace Duplicati.Library.Main.Operation.Restore sw_req?.Stop(); sw_resp?.Start(); - var data = await block_response.ReadAsync().ConfigureAwait(false); + var data = await (await block_response.ReadAsync().ConfigureAwait(false)).ConfigureAwait(false); sw_resp?.Stop(); sw_work?.Start(); diff --git a/Duplicati/Library/Main/Operation/RestoreHandler.cs b/Duplicati/Library/Main/Operation/RestoreHandler.cs index 5c7ef2525..5f27b6da0 100644 --- a/Duplicati/Library/Main/Operation/RestoreHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreHandler.cs @@ -332,8 +332,8 @@ namespace Duplicati.Library.Main.Operation using var setup_log_timer = new Logging.Timer(LOGTAG, "RestoreNetworkSetup", "RestoreNetworkSetup"); // Create the channels between BlockManager and FileProcessor Restore.Channels.BufferSize = m_options.RestoreChannelBufferSize; - var fileprocessor_requests = new Channel[m_options.RestoreFileProcessors].Select(_ => ChannelManager.CreateChannel(buffersize: Restore.Channels.BufferSize)).ToArray(); - var fileprocessor_responses = new Channel[m_options.RestoreFileProcessors].Select(_ => ChannelManager.CreateChannel(buffersize: Restore.Channels.BufferSize)).ToArray(); + var fileprocessor_requests = Enumerable.Range(0, m_options.RestoreFileProcessors).Select(_ => ChannelManager.CreateChannel(buffersize: Restore.Channels.BufferSize)).ToArray(); + var fileprocessor_responses = Enumerable.Range(0, m_options.RestoreFileProcessors).Select(_ => ChannelManager.CreateChannel>(buffersize: Restore.Channels.BufferSize)).ToArray(); // Create the process network Restore.Channels channels = new(); From 9a82beaf8b329a67f2cf4187b0aefeb55b433020 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 10 Mar 2025 10:55:07 +0100 Subject: [PATCH 330/362] Fixed log messages to correspond to actual operation performed --- .../Main/Operation/FilelistProcessor.cs | 44 +++++++++++++------ 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/Duplicati/Library/Main/Operation/FilelistProcessor.cs b/Duplicati/Library/Main/Operation/FilelistProcessor.cs index 5fe91d036..288269410 100644 --- a/Duplicati/Library/Main/Operation/FilelistProcessor.cs +++ b/Duplicati/Library/Main/Operation/FilelistProcessor.cs @@ -323,7 +323,7 @@ namespace Duplicati.Library.Main.Operation { case RemoteVolumeState.Deleted: if (remoteFound) - Logging.Log.WriteInformationMessage(LOGTAG, "IgnoreRemoteDeletedFile", "ignoring remote file listed as {0}: {1}", i.State, i.Name); + Logging.Log.WriteInformationMessage(LOGTAG, "IgnoreRemoteDeletedFile", "Ignoring remote file listed as {0}: {1}", i.State, i.Name); break; @@ -331,27 +331,35 @@ namespace Duplicati.Library.Main.Operation case RemoteVolumeState.Deleting: if (remoteFound) { - Logging.Log.WriteInformationMessage(LOGTAG, "RemoveUnwantedRemoteFile", "removing remote file listed as {0}: {1}", i.State, i.Name); if (options.Dryrun) + { Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteRemoteFile", "Would delete remote file: {0}", i.Name); - else if (verifyMode != VerifyMode.VerifyOnly) + } + else if (verifyMode == VerifyMode.VerifyOnly) + { + Logging.Log.WriteInformationMessage(LOGTAG, "WouldRemoveUnwantedRemoteFile", "Would remove remote file listed as {0}: {1}, but defering as {2} is requested", i.State, i.Name, verifyMode); + } + else + { + Logging.Log.WriteInformationMessage(LOGTAG, "RemoveUnwantedRemoteFile", "Removing remote file listed as {0}: {1}", i.State, i.Name); await backendManager.DeleteAsync(i.Name, i.Size, true, CancellationToken.None).ConfigureAwait(false); + } } else { if (i.DeleteGracePeriod > DateTime.UtcNow) { - Logging.Log.WriteInformationMessage(LOGTAG, "KeepDeleteRequest", "keeping delete request for {0} until {1}", i.Name, i.DeleteGracePeriod.ToLocalTime()); + Logging.Log.WriteInformationMessage(LOGTAG, "KeepDeleteRequest", "Keeping delete request for {0} until {1}", i.Name, i.DeleteGracePeriod.ToLocalTime()); } else { if (i.State == RemoteVolumeState.Temporary && protectedFiles.Any(pf => pf == i.Name)) { - Logging.Log.WriteInformationMessage(LOGTAG, "KeepIncompleteFile", "keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name); + Logging.Log.WriteInformationMessage(LOGTAG, "KeepIncompleteFile", "Keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name); } else { - Logging.Log.WriteInformationMessage(LOGTAG, "RemoteUnwantedMissingFile", "removing file listed as {0}: {1}", i.State, i.Name); + Logging.Log.WriteInformationMessage(LOGTAG, "RemoteUnwantedMissingFile", "Removing file listed as {0}: {1}", i.State, i.Name); temporaryAndDeletedVolumes.Add(i.Name); } } @@ -360,14 +368,14 @@ namespace Duplicati.Library.Main.Operation case RemoteVolumeState.Uploading: if (remoteFound && correctSize && r.File.Size >= 0) { - Logging.Log.WriteInformationMessage(LOGTAG, "PromotingCompleteFile", "promoting uploaded complete file from {0} to {2}: {1}", i.State, i.Name, RemoteVolumeState.Uploaded); + Logging.Log.WriteInformationMessage(LOGTAG, "PromotingCompleteFile", "Promoting uploaded complete file from {0} to {2}: {1}", i.State, i.Name, RemoteVolumeState.Uploaded); database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Uploaded, i.Size, i.Hash); } else if (!remoteFound) { if (protectedFiles.Any(pf => pf == i.Name)) { - Logging.Log.WriteInformationMessage(LOGTAG, "KeepIncompleteFile", "keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name); + Logging.Log.WriteInformationMessage(LOGTAG, "KeepIncompleteFile", "Keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name); database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Temporary, i.Size, i.Hash, false, new TimeSpan(0), null); } else if (verifyMode == VerifyMode.VerifyStrict && !strictExcemptFiles.Any(pf => pf == i.Name)) @@ -377,7 +385,7 @@ namespace Duplicati.Library.Main.Operation } else { - Logging.Log.WriteInformationMessage(LOGTAG, "SchedulingMissingFileForDelete", "scheduling missing file for deletion, currently listed as {0}: {1}", i.State, i.Name); + Logging.Log.WriteInformationMessage(LOGTAG, "SchedulingMissingFileForDelete", "Scheduling missing file for deletion, currently listed as {0}: {1}", i.State, i.Name); missingUploadingVolumes.Add(i.Name); database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Deleting, i.Size, i.Hash, false, TimeSpan.FromHours(2), null); } @@ -386,7 +394,7 @@ namespace Duplicati.Library.Main.Operation { if (protectedFiles.Any(pf => pf == i.Name)) { - Logging.Log.WriteInformationMessage(LOGTAG, "KeepIncompleteFile", "keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name); + Logging.Log.WriteInformationMessage(LOGTAG, "KeepIncompleteFile", "Keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name); } else if (verifyMode == VerifyMode.VerifyStrict && !strictExcemptFiles.Any(pf => pf == i.Name)) { @@ -395,11 +403,19 @@ namespace Duplicati.Library.Main.Operation } else { - Logging.Log.WriteInformationMessage(LOGTAG, "Remove incomplete file", "removing incomplete remote file listed as {0}: {1}", i.State, i.Name); if (options.Dryrun) + { Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteRemoteFile", "Would delete remote file: {0}", i.Name); - else if (verifyMode != VerifyMode.VerifyOnly) + } + else if (verifyMode == VerifyMode.VerifyOnly) + { + Logging.Log.WriteInformationMessage(LOGTAG, "WouldDeleteIncompleteFile", "Would remove incomplete remote file listed as {0}: {1}, but defering as {2} is requested", i.State, i.Name, verifyMode); + } + else + { + Logging.Log.WriteInformationMessage(LOGTAG, "RemovingIncompleteFile", "Removing incomplete remote file listed as {0}: {1}", i.State, i.Name); await backendManager.DeleteAsync(i.Name, i.Size, true, CancellationToken.None).ConfigureAwait(false); + } } } break; @@ -423,7 +439,7 @@ namespace Duplicati.Library.Main.Operation break; default: - Logging.Log.WriteWarningMessage(LOGTAG, "UnknownFileState", null, "unknown state for remote file listed as {0}: {1}", i.State, i.Name); + Logging.Log.WriteWarningMessage(LOGTAG, "UnknownFileState", null, "Unknown state for remote file listed as {0}: {1}", i.State, i.Name); break; } } @@ -442,7 +458,7 @@ namespace Duplicati.Library.Main.Operation } foreach (var i in missingHash) - Logging.Log.WriteWarningMessage(LOGTAG, "MissingRemoteHash", null, "remote file {1} is listed as {0} with size {2} but should be {3}, please verify the sha256 hash \"{4}\"", i.Item2.State, i.Item2.Name, i.Item1, i.Item2.Size, i.Item2.Hash); + Logging.Log.WriteWarningMessage(LOGTAG, "MissingRemoteHash", null, "Remote file {1} is listed as {0} with size {2} but should be {3}, please verify the sha256 hash \"{4}\"", i.Item2.State, i.Item2.Name, i.Item1, i.Item2.Size, i.Item2.Hash); return new RemoteAnalysisResult() { From 32762bec342afd56c5b7c9768b1856e1256bbc46 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 10 Mar 2025 10:59:53 +0100 Subject: [PATCH 331/362] Fixed comment and added warning message --- Duplicati/Library/Main/Operation/FilelistProcessor.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Operation/FilelistProcessor.cs b/Duplicati/Library/Main/Operation/FilelistProcessor.cs index 288269410..c47bfedab 100644 --- a/Duplicati/Library/Main/Operation/FilelistProcessor.cs +++ b/Duplicati/Library/Main/Operation/FilelistProcessor.cs @@ -446,7 +446,7 @@ namespace Duplicati.Library.Main.Operation await backendManager.WaitForEmptyAsync(database, null, CancellationToken.None).ConfigureAwait(false); - // cleanup deleted volumes in DB en block + // Batch cleanup deleted volumes in DB if (verifyMode == VerifyMode.VerifyStrict && missingUploadingVolumes.Count > 0) throw new RemoteListVerificationException($"The remote volumes {string.Join(", ", missingUploadingVolumes)} are supposed to be deleted, but strict mode is on. Try running the \"repair\" command", "DeleteDuringStrictMode"); else if (verifyMode == VerifyMode.VerifyAndClean || verifyMode == VerifyMode.VerifyStrict) @@ -456,6 +456,10 @@ namespace Duplicati.Library.Main.Operation if (!options.Dryrun) database.TerminatedWithActiveUploads = false; } + else if (verifyMode == VerifyMode.VerifyOnly && database.TerminatedWithActiveUploads) + { + Logging.Log.WriteWarningMessage(LOGTAG, "ActiveUploadsDetected", null, "Active uploads detected, but no cleanup was performed. Run the \"repair\" command to clean up the incomplete files"); + } foreach (var i in missingHash) Logging.Log.WriteWarningMessage(LOGTAG, "MissingRemoteHash", null, "Remote file {1} is listed as {0} with size {2} but should be {3}, please verify the sha256 hash \"{4}\"", i.Item2.State, i.Item2.Name, i.Item1, i.Item2.Size, i.Item2.Hash); From 750ef90e14f288572ea98874bcd7e677f620fd40 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Mon, 10 Mar 2025 13:26:15 +0100 Subject: [PATCH 332/362] Clarified file overwrite conditions in the help text of the Remote Synchronization tool. --- Tools/RemoteSynchronization/Program.cs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index cda66367d..91c8ae793 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -87,8 +87,12 @@ to have the destination match the source. If the destination has files that are not in the source, they will be deleted (or renamed if the retention option is set). -If the destination has files that are in the source but are different in size or -has a newer timestamp, they will be overwritten by the source files. +If the destination has files that are also present in the source, but the files +differ in size, or if the source files have a newer (more recent) timestamp, +the destination files will be overwritten by the source files. Given that some +backends do not allow for metadata or timestamp modification, and that the tool +is run after backup, the destination files should always have a timestamp that +is newer (or the same if run promptly) compared to the source files. If the force option is set, the destination will be overwritten by the source, regardless of the state of the files. It will also skip the initial comparison, @@ -554,7 +558,7 @@ destination will be verified before being overwritten (if they seemingly match). var to_delete = new HashSet(); var to_verify = new List(); - // Find all of the files in src that are not in dst, have a different size or have a more recent modification date + // Find all of the files in src that are not in dst, where the dst has a different size than src or src a more recent modification date than dst using (new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Prepare | Check the files that are present in source against destination")) foreach (var f_src in files_src) { From be16323f24129f035900fd8326ebca989473d216 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 10 Mar 2025 14:37:30 +0100 Subject: [PATCH 333/362] General cleanup, common database validation exception --- .../Library/Interface/CustomExceptions.cs | 25 ++++- .../Library/Main/Database/ExtensionMethods.cs | 12 --- .../Main/Database/LocalBackupDatabase.cs | 11 -- .../Library/Main/Database/LocalDatabase.cs | 25 +++-- .../Main/Database/LocalRecreateDatabase.cs | 23 ++-- .../Backup/UploadSyntheticFilelist.cs | 14 +-- .../Main/Volumes/FilesetVolumeWriter.cs | 100 ++++++++++++++---- Duplicati/Library/Utility/IFilter.cs | 40 ++++--- 8 files changed, 156 insertions(+), 94 deletions(-) diff --git a/Duplicati/Library/Interface/CustomExceptions.cs b/Duplicati/Library/Interface/CustomExceptions.cs index bbd50cc87..27068d494 100644 --- a/Duplicati/Library/Interface/CustomExceptions.cs +++ b/Duplicati/Library/Interface/CustomExceptions.cs @@ -226,7 +226,7 @@ namespace Duplicati.Library.Interface : base(Strings.Common.SettingsKeyMissingExceptionError, "SettingsKeyMissing") { } } - + /// /// An exception for carrying the Retry-After header value on 429 Exceptions /// @@ -235,4 +235,27 @@ namespace Duplicati.Library.Interface { public readonly RetryConditionHeaderValue RetryAfter = retryAfter; } + + /// + /// An exception indicating that the database state is inconsistent + /// + [Serializable] + public class DatabaseInconsistencyException : UserInformationException + { + public DatabaseInconsistencyException(string message) + : base(message, "DatabaseInconsistency") + { } + + public DatabaseInconsistencyException(string message, string helpId) + : base(message, helpId) + { } + + public DatabaseInconsistencyException(string message, Exception innerException) + : base(message, "DatabaseInconsistency", innerException) + { } + + public DatabaseInconsistencyException(string message, string helpId, Exception innerException) + : base(message, helpId, innerException) + { } + } } diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index 8a850cb7c..5f172c5e1 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -373,18 +373,6 @@ public static class ExtensionMethods return self.ExecuteReader(); } - /// - /// Executes the command and returns an enumerable of data readers. - /// - /// The database command to execute. - /// An enumerable of data readers. - public static IEnumerable ExecuteReaderEnumerable(this System.Data.IDbCommand self) - { - using var rd = self.ExecuteReader(); - while (rd.Read()) - yield return rd; - } - /// /// Executes the given command string `cmd` on the given database command `self` with the given values `values` and returns an enumerable of data readers. /// diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index 730126233..510bab3dd 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -796,17 +796,6 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist yield return rd.GetValue(0).ToString(); } - public void LinkFilesetToVolume(long filesetid, long volumeid, System.Data.IDbTransaction transaction) - { - using (var cmd = m_connection.CreateCommand()) - { - cmd.Transaction = transaction; - var c = cmd.ExecuteNonQuery(@"UPDATE ""Fileset"" SET ""VolumeID"" = ? WHERE ""ID"" = ?", volumeid, filesetid); - if (c != 1) - throw new Exception(string.Format("Failed to link filesetid {0} to volumeid {1}", filesetid, volumeid)); - } - } - public void MoveBlockToVolume(string blockkey, long size, long sourcevolumeid, long targetvolumeid, System.Data.IDbTransaction transaction) { using (var cmd = m_connection.CreateCommand()) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index f3f6c886b..02edb5c52 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -856,26 +856,24 @@ ON sb.AppendFormat("... and {0} more", c); sb.Append(". Run repair to fix it."); - throw new InvalidDataException(sb.ToString()); + throw new DatabaseInconsistencyException(sb.ToString()); } var real_count = cmd.ExecuteScalarInt64(@"SELECT Count(*) FROM ""BlocklistHash""", 0); var unique_count = cmd.ExecuteScalarInt64(@"SELECT Count(*) FROM (SELECT DISTINCT ""BlocksetID"", ""Index"" FROM ""BlocklistHash"")", 0); if (real_count != unique_count) - throw new InvalidDataException(string.Format("Found {0} blocklist hashes, but there should be {1}. Run repair to fix it.", real_count, unique_count)); + throw new DatabaseInconsistencyException(string.Format("Found {0} blocklist hashes, but there should be {1}. Run repair to fix it.", real_count, unique_count)); var itemswithnoblocklisthash = cmd.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM (SELECT * FROM (SELECT ""N"".""BlocksetID"", ((""N"".""BlockCount"" + {0} - 1) / {0}) AS ""BlocklistHashCountExpected"", CASE WHEN ""G"".""BlocklistHashCount"" IS NULL THEN 0 ELSE ""G"".""BlocklistHashCount"" END AS ""BlocklistHashCountActual"" FROM (SELECT ""BlocksetID"", COUNT(*) AS ""BlockCount"" FROM ""BlocksetEntry"" GROUP BY ""BlocksetID"") ""N"" LEFT OUTER JOIN (SELECT ""BlocksetID"", COUNT(*) AS ""BlocklistHashCount"" FROM ""BlocklistHash"" GROUP BY ""BlocksetID"") ""G"" ON ""N"".""BlocksetID"" = ""G"".""BlocksetID"" WHERE ""N"".""BlockCount"" > 1) WHERE ""BlocklistHashCountExpected"" != ""BlocklistHashCountActual"")", blocksize / hashsize), 0); if (itemswithnoblocklisthash != 0) - throw new InvalidDataException(string.Format("Found {0} file(s) with missing blocklist hashes", itemswithnoblocklisthash)); + throw new DatabaseInconsistencyException(string.Format("Found {0} file(s) with missing blocklist hashes", itemswithnoblocklisthash)); if (cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""Blockset"" WHERE ""Length"" > 0 AND ""ID"" NOT IN (SELECT ""BlocksetId"" FROM ""BlocksetEntry"")") != 0) - { - throw new Exception("Detected non-empty blocksets with no associated blocks!"); - } + throw new DatabaseInconsistencyException("Detected non-empty blocksets with no associated blocks!"); if (cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""FileLookup"" WHERE ""BlocksetID"" != ? AND ""BlocksetID"" != ? AND NOT ""BlocksetID"" IN (SELECT ""ID"" FROM ""Blockset"")", 0, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID) != 0) - throw new Exception("Detected files associated with non-existing blocksets!"); + throw new DatabaseInconsistencyException("Detected files associated with non-existing blocksets!"); if (verifyfilelists) { @@ -899,7 +897,7 @@ ON } if (anyError.Any()) { - throw new Interface.UserInformationException(string.Join("\n\r", anyError), "FilesetDifferences"); + throw new DatabaseInconsistencyException(string.Join("\n\r", anyError), "FilesetDifferences"); } } } @@ -1222,6 +1220,17 @@ ORDER BY } } + public void LinkFilesetToVolume(long filesetid, long volumeid, System.Data.IDbTransaction transaction) + { + using (var cmd = m_connection.CreateCommand()) + { + cmd.Transaction = transaction; + var c = cmd.ExecuteNonQuery(@"UPDATE ""Fileset"" SET ""VolumeID"" = ? WHERE ""ID"" = ?", volumeid, filesetid); + if (c != 1) + throw new Exception(string.Format("Failed to link filesetid {0} to volumeid {1}", filesetid, volumeid)); + } + } + public void PushTimestampChangesToPreviousVersion(long filesetId, System.Data.IDbTransaction transaction) { using (var cmd = m_connection.CreateCommand()) diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index ccc4f2939..dc6a890b1 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -20,9 +20,6 @@ // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; namespace Duplicati.Library.Main.Database { @@ -792,8 +789,14 @@ DELETE FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND "" cmd.CommandText = string.Format(@"DROP TABLE IF EXISTS ""{0}""", m_tempblocklist); cmd.ExecuteNonQuery(); } - catch { } - finally { m_tempblocklist = null; } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "FailedToDropTempBlocklist", ex, "Failed to drop temporary blocklist table {0}", m_tempblocklist); + } + finally + { + m_tempblocklist = null; + } if (m_tempsmalllist != null) try @@ -801,8 +804,14 @@ DELETE FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND "" cmd.CommandText = string.Format(@"DROP TABLE IF EXISTS ""{0}""", m_tempsmalllist); cmd.ExecuteNonQuery(); } - catch { } - finally { m_tempsmalllist = null; } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "FailedToDropTempSmalllist", ex, "Failed to drop temporary smalllist table {0}", m_tempsmalllist); + } + finally + { + m_tempsmalllist = null; + } } diff --git a/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs b/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs index 4e1bfc9e0..7efa1b13e 100644 --- a/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs +++ b/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs @@ -88,19 +88,7 @@ namespace Duplicati.Library.Main.Operation.Backup FilesetVolumeWriter fsw = null; try { - var s = 1; - var fileTime = incompleteSet.Value + TimeSpan.FromSeconds(s); - - // Probe for an unused filename - while (s < 60) - { - var id = await database.GetRemoteVolumeIDAsync(VolumeBase.GenerateFilename(RemoteVolumeType.Files, options, null, fileTime)); - if (id < 0) - break; - - fileTime = incompleteSet.Value + TimeSpan.FromSeconds(++s); - } - + var fileTime = await FilesetVolumeWriter.ProbeUnusedFilenameName(database, options, incompleteSet.Value).ConfigureAwait(false); fsw = new FilesetVolumeWriter(options, fileTime); fsw.VolumeID = await database.RegisterRemoteVolumeAsync(fsw.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary); diff --git a/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs b/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs index a4845b47b..4bf79d88e 100644 --- a/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs +++ b/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs @@ -1,28 +1,32 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; using System.IO; using Newtonsoft.Json; using Duplicati.Library.Interface; +using Duplicati.Library.Main.Database; +using Duplicati.Library.Main.Operation.Common; +using System.Threading.Tasks; +using Duplicati.Library.Utility; namespace Duplicati.Library.Main.Volumes { @@ -146,7 +150,7 @@ namespace Duplicati.Library.Main.Volumes m_writer.WriteValue(name); if (metahash != null) WriteMetaProperties(metahash, metasize, metablockhash, metablocklisthashes); - + m_writer.WriteEndObject(); } @@ -222,5 +226,59 @@ namespace Duplicati.Library.Main.Volumes sr.Write(FilesetData.GetFilesetInstance(isFullBackup)); } } + + /// + /// Probes for an unused filename, using the current time as a starting point + /// + /// The database to check for clashes + /// The options to use for the filename + /// The starting time to probe from + /// The time to increment by each probe + /// The maximum number of tries to probe + /// The first unused filename + internal static Task ProbeUnusedFilenameName(DatabaseCommon database, Options options, DateTime start, TimeSpan increment = default, int maxTries = 60) + => ProbeUnusedFilenameName(name => database.GetRemoteVolumeIDAsync(name), options, start, increment, maxTries); + + /// + /// Probes for an unused filename, using the current time as a starting point + /// + /// The database to check for clashes + /// The options to use for the filename + /// The starting time to probe from + /// The time to increment by each probe + /// The maximum number of tries to probe + /// The first unused filename + internal static DateTime ProbeUnusedFilenameName(LocalDatabase database, Options options, DateTime start, TimeSpan increment = default, int maxTries = 60) + => ProbeUnusedFilenameName((name) => Task.FromResult(database.GetRemoteVolumeID(name)), options, start, increment, maxTries).Await(); + + /// + /// Probes for an unused filename, using the current time as a starting point + /// + /// The function to use to probe for the ID + /// The options to use for the filename + /// The starting time to probe from + /// The time to increment by each probe + /// The maximum number of tries to probe + /// The first unused filename + private static async Task ProbeUnusedFilenameName(Func> ProbeForId, Options options, DateTime start, TimeSpan increment = default, int maxTries = 60) + { + if (increment == default) + increment = TimeSpan.FromSeconds(1); + + var s = 1; + var time = start; + + while (s < maxTries) + { + var id = await ProbeForId(GenerateFilename(RemoteVolumeType.Files, options, null, time)).ConfigureAwait(false); + if (id < 0) + return time; + + time += increment; + } + + throw new UserInformationException($"Failed to find an unused filename, tried {increment} * {maxTries} from {start}", "FailedToFindUnusedFilename"); + + } } } diff --git a/Duplicati/Library/Utility/IFilter.cs b/Duplicati/Library/Utility/IFilter.cs index 994fff493..28e83ea7f 100644 --- a/Duplicati/Library/Utility/IFilter.cs +++ b/Duplicati/Library/Utility/IFilter.cs @@ -1,26 +1,24 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System; - namespace Duplicati.Library.Utility { /// From c673cba6071d7a6d53f327ec9179e5f9a0f66e02 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 10 Mar 2025 14:39:31 +0100 Subject: [PATCH 334/362] Cleanups in tests --- Duplicati/UnitTest/DisruptionTests.cs | 372 +++++++++++++------------- Duplicati/UnitTest/TestUtils.cs | 3 + 2 files changed, 183 insertions(+), 192 deletions(-) diff --git a/Duplicati/UnitTest/DisruptionTests.cs b/Duplicati/UnitTest/DisruptionTests.cs index 32c0e064f..8c1b4702e 100644 --- a/Duplicati/UnitTest/DisruptionTests.cs +++ b/Duplicati/UnitTest/DisruptionTests.cs @@ -155,9 +155,7 @@ namespace Duplicati.UnitTest Dictionary backupTypeMap; using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { - IPurgeFilesResults purgeResults = c.PurgeFiles(new Library.Utility.FilterExpression($"{this.DATAFOLDER}/*{this.fileSizes[0]}*")); - Assert.AreEqual(0, purgeResults.Errors.Count()); - Assert.AreEqual(0, purgeResults.Warnings.Count()); + TestUtils.AssertResults(c.PurgeFiles(new Library.Utility.FilterExpression($"{this.DATAFOLDER}/*{this.fileSizes[0]}*"))); List filesets = c.List().Filesets.ToList(); Assert.AreEqual(2, filesets.Count); @@ -181,9 +179,7 @@ namespace Duplicati.UnitTest // Run a repair and verify that the fileset file exists in the new dlist files. using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { - IRepairResults repairResults = c.Repair(); - Assert.AreEqual(0, repairResults.Errors.Count()); - Assert.AreEqual(0, repairResults.Warnings.Count()); + TestUtils.AssertResults(c.Repair()); List filesets = c.List().Filesets.ToList(); Assert.AreEqual(2, filesets.Count); @@ -220,9 +216,7 @@ namespace Duplicati.UnitTest DateTime firstBackupTime; using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { - IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER }); - Assert.AreEqual(0, backupResults.Errors.Count()); - Assert.AreEqual(0, backupResults.Warnings.Count()); + TestUtils.AssertResults(c.Backup(new[] { this.DATAFOLDER })); firstBackupTime = c.List().Filesets.First().Time; } @@ -233,9 +227,7 @@ namespace Duplicati.UnitTest using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { this.ModifySourceFiles(); - IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER }); - Assert.AreEqual(0, backupResults.Errors.Count()); - Assert.AreEqual(0, backupResults.Warnings.Count()); + TestUtils.AssertResults(c.Backup(new[] { this.DATAFOLDER })); secondBackupTime = c.List().Filesets.First().Time; } @@ -254,9 +246,7 @@ namespace Duplicati.UnitTest using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { options["keep-time"] = $"{(int)((DateTime.Now - firstBackupTime).TotalSeconds - (secondBackupTime - firstBackupTime).TotalSeconds / 2)}s"; - IDeleteResults deleteResults = c.Delete(); - Assert.AreEqual(0, deleteResults.Errors.Count()); - Assert.AreEqual(0, deleteResults.Warnings.Count()); + TestUtils.AssertResults(c.Delete()); List filesets = c.List().Filesets.ToList(); Assert.AreEqual(2, filesets.Count); @@ -278,9 +268,7 @@ namespace Duplicati.UnitTest // Set the keep-time option so that the threshold lies after the most recent full backup // and run the delete operation. options["keep-time"] = "1s"; - IDeleteResults deleteResults = c.Delete(); - Assert.AreEqual(0, deleteResults.Errors.Count()); - Assert.AreEqual(0, deleteResults.Warnings.Count()); + TestUtils.AssertResults(c.Delete()); List filesets = c.List().Filesets.ToList(); Assert.AreEqual(3, filesets.Count); @@ -313,9 +301,7 @@ namespace Duplicati.UnitTest DateTime firstBackupTime; using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { - IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER }); - Assert.AreEqual(0, backupResults.Errors.Count()); - Assert.AreEqual(0, backupResults.Warnings.Count()); + TestUtils.AssertResults(c.Backup(new[] { this.DATAFOLDER })); firstBackupTime = c.List().Filesets.First().Time; } @@ -340,9 +326,7 @@ namespace Duplicati.UnitTest using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { this.ModifySourceFiles(); - IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER }); - Assert.AreEqual(0, backupResults.Errors.Count()); - Assert.AreEqual(0, backupResults.Warnings.Count()); + TestUtils.AssertResults(c.Backup(new[] { this.DATAFOLDER })); fourthBackupTime = c.List().Filesets.First().Time; } @@ -373,9 +357,7 @@ namespace Duplicati.UnitTest using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { this.ModifySourceFiles(); - IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER }); - Assert.AreEqual(0, backupResults.Errors.Count()); - Assert.AreEqual(0, backupResults.Warnings.Count()); + TestUtils.AssertResults(c.Backup(new[] { this.DATAFOLDER })); DateTime sixthBackupTime = c.List().Filesets.First().Time; // Since the last backup was full, we can now expect to have just the 2 most recent full backups. @@ -407,11 +389,7 @@ namespace Duplicati.UnitTest // Run a full backup. using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) - { - IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER }); - Assert.AreEqual(0, backupResults.Errors.Count()); - Assert.AreEqual(0, backupResults.Warnings.Count()); - } + TestUtils.AssertResults(c.Backup(new[] { this.DATAFOLDER })); // Run a partial backup. using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) @@ -450,18 +428,14 @@ namespace Duplicati.UnitTest DateTime firstBackupTime; using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { - IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER }); - Assert.AreEqual(0, backupResults.Errors.Count()); - Assert.AreEqual(0, backupResults.Warnings.Count()); + TestUtils.AssertResults(c.Backup(new[] { this.DATAFOLDER })); firstBackupTime = c.List().Filesets.First().Time; List filesets = c.List().Filesets.ToList(); Assert.AreEqual(1, filesets.Count); this.ModifySourceFiles(); - backupResults = c.Backup(new[] { this.DATAFOLDER }); - Assert.AreEqual(0, backupResults.Errors.Count()); - Assert.AreEqual(0, backupResults.Warnings.Count()); + TestUtils.AssertResults(c.Backup(new[] { this.DATAFOLDER })); DateTime secondBackupTime = c.List().Filesets.First().Time; // Since the most recent backup is not considered in the retention logic, the only backup in the first time frame @@ -503,9 +477,7 @@ namespace Duplicati.UnitTest using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { this.ModifySourceFiles(); - IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER }); - Assert.AreEqual(0, backupResults.Errors.Count()); - Assert.AreEqual(0, backupResults.Warnings.Count()); + TestUtils.AssertResults(c.Backup(new[] { this.DATAFOLDER })); fourthBackupTime = c.List().Filesets.First().Time; // Since the most recent backup is not considered in the retention logic, the third backup is the only backup @@ -521,9 +493,7 @@ namespace Duplicati.UnitTest Assert.AreEqual(BackupType.FULL_BACKUP, filesets[0].IsFullBackup); this.ModifySourceFiles(); - backupResults = c.Backup(new[] { this.DATAFOLDER }); - Assert.AreEqual(0, backupResults.Errors.Count()); - Assert.AreEqual(0, backupResults.Warnings.Count()); + TestUtils.AssertResults(c.Backup(new[] { this.DATAFOLDER })); DateTime fifthBackupTime = c.List().Filesets.First().Time; // Since the most recent backup is not considered in the retention logic, we now have two backups in the @@ -588,9 +558,7 @@ namespace Duplicati.UnitTest // Run a complete backup. using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { - IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER }); - Assert.AreEqual(0, backupResults.Errors.Count()); - Assert.AreEqual(0, backupResults.Warnings.Count()); + TestUtils.AssertResults(c.Backup(new[] { this.DATAFOLDER })); Assert.AreEqual(1, c.List().Filesets.Count()); Assert.AreEqual(BackupType.FULL_BACKUP, c.List().Filesets.Single(x => x.Version == 0).IsFullBackup); @@ -617,14 +585,14 @@ namespace Duplicati.UnitTest Dictionary restoreOptions = new Dictionary(options) { ["restore-path"] = this.RESTOREFOLDER }; using (Controller c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null)) { - IListResults lastResults = c.List("*"); - string[] partialVersionFiles = lastResults.Files.Select(x => x.Path).Where(x => !Utility.IsFolder(x, File.GetAttributes)).ToArray(); + var lastResults = c.List("*"); + var partialVersionFiles = lastResults.Files.Select(x => x.Path).Where(x => !Utility.IsFolder(x, File.GetAttributes)).ToArray(); Assert.GreaterOrEqual(partialVersionFiles.Length, 1); c.Restore(partialVersionFiles); foreach (string filepath in partialVersionFiles) { - string filename = Path.GetFileName(filepath); + var filename = Path.GetFileName(filepath); TestUtils.AssertFilesAreEqual(filepath, Path.Combine(this.RESTOREFOLDER, filename ?? String.Empty), false, filename); } } @@ -633,9 +601,7 @@ namespace Duplicati.UnitTest File.Delete(this.DBFILE); using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { - IRepairResults repairResults = c.Repair(); - Assert.AreEqual(0, repairResults.Errors.Count()); - Assert.AreEqual(0, repairResults.Warnings.Count()); + TestUtils.AssertResults(c.Repair()); Assert.AreEqual(2, c.List().Filesets.Count()); Assert.AreEqual(BackupType.FULL_BACKUP, c.List().Filesets.Single(x => x.Version == 1).IsFullBackup); @@ -645,9 +611,7 @@ namespace Duplicati.UnitTest // Run a complete backup. Listing the Filesets should include both full and partial backups. using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { - IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER }); - Assert.AreEqual(0, backupResults.Errors.Count()); - Assert.AreEqual(0, backupResults.Warnings.Count()); + TestUtils.AssertResults(c.Backup(new[] { this.DATAFOLDER })); Assert.AreEqual(3, c.List().Filesets.Count()); Assert.AreEqual(BackupType.FULL_BACKUP, c.List().Filesets.Single(x => x.Version == 2).IsFullBackup); @@ -659,13 +623,11 @@ namespace Duplicati.UnitTest restoreOptions["overwrite"] = "true"; using (Controller c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null)) { - IListResults lastResults = c.List("*"); - string[] fullVersionFiles = lastResults.Files.Select(x => x.Path).Where(x => !Utility.IsFolder(x, File.GetAttributes)).ToArray(); + var lastResults = c.List("*"); + var fullVersionFiles = lastResults.Files.Select(x => x.Path).Where(x => !Utility.IsFolder(x, File.GetAttributes)).ToArray(); Assert.AreEqual(this.fileSizes.Length, fullVersionFiles.Length); - IRestoreResults restoreResults = c.Restore(fullVersionFiles); - Assert.AreEqual(0, restoreResults.Errors.Count()); - Assert.AreEqual(0, restoreResults.Warnings.Count()); + TestUtils.AssertResults(c.Restore(fullVersionFiles)); foreach (string filepath in fullVersionFiles) { @@ -695,9 +657,7 @@ namespace Duplicati.UnitTest // Run a complete backup. using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { - IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER }); - Assert.AreEqual(0, backupResults.Errors.Count()); - Assert.AreEqual(0, backupResults.Warnings.Count()); + TestUtils.AssertResults(c.Backup(new[] { this.DATAFOLDER })); List filesets = c.List().Filesets.ToList(); Assert.AreEqual(1, filesets.Count); @@ -728,9 +688,7 @@ namespace Duplicati.UnitTest // The next backup should proceed without issues. using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { - IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER }); - Assert.AreEqual(0, backupResults.Errors.Count()); - Assert.AreEqual(0, backupResults.Warnings.Count()); + TestUtils.AssertResults(c.Backup(new[] { this.DATAFOLDER })); List filesets = c.List().Filesets.ToList(); Assert.AreEqual(2, filesets.Count); @@ -742,14 +700,11 @@ namespace Duplicati.UnitTest Dictionary restoreOptions = new Dictionary(options) { ["restore-path"] = this.RESTOREFOLDER }; using (Controller c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null)) { - IListResults lastResults = c.List("*"); - string[] fullVersionFiles = lastResults.Files.Select(x => x.Path).Where(x => !Utility.IsFolder(x, File.GetAttributes)).ToArray(); + var lastResults = c.List("*"); + var fullVersionFiles = lastResults.Files.Select(x => x.Path).Where(x => !Utility.IsFolder(x, File.GetAttributes)).ToArray(); Assert.AreEqual(this.fileSizes.Length, fullVersionFiles.Length); - IRestoreResults restoreResults = c.Restore(fullVersionFiles); - Assert.AreEqual(0, restoreResults.Errors.Count()); - Assert.AreEqual(0, restoreResults.Warnings.Count()); - + TestUtils.AssertResults(c.Restore(fullVersionFiles)); foreach (string filepath in fullVersionFiles) { string filename = Path.GetFileName(filepath); @@ -767,12 +722,8 @@ namespace Duplicati.UnitTest testopts["dblock-size"] = "10mb"; // Make a base backup - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) - { - IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER }); - Assert.AreEqual(0, backupResults.Errors.Count()); - Assert.AreEqual(0, backupResults.Warnings.Count()); - } + using (var c = new Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup(new string[] { DATAFOLDER })); // Make a new backup that fails uploading a dblock file ModifySourceFiles(); @@ -811,27 +762,112 @@ namespace Duplicati.UnitTest return false; }; - using (var c = new Library.Main.Controller(failtarget, testopts, null)) + using (var c = new Controller(failtarget, testopts, null)) Assert.Throws(() => c.Backup(new string[] { DATAFOLDER })); Assert.That(secondUploadStarted, Is.True, "Second upload was not started"); Assert.That(secondUploadCompleted, Is.True, "Second upload was not started"); // Create a regular backup - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) - { - IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER }); - Assert.AreEqual(0, backupResults.Errors.Count()); - Assert.AreEqual(0, backupResults.Warnings.Count()); - } + using (var c = new Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup(new string[] { DATAFOLDER })); // Verify that all is in order - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null)) + using (var c = new Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null)) + TestUtils.AssertResults(c.Test(long.MaxValue)); + + // Test that we can recreate + var recreatedDatabaseFile = Path.Combine(BASEFOLDER, "recreated-database.sqlite"); + if (File.Exists(recreatedDatabaseFile)) + File.Delete(recreatedDatabaseFile); + + testopts["dbpath"] = recreatedDatabaseFile; + + using (var c = new Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Repair()); + + // Check that we have 3 versions + using (var c = new Controller("file://" + TARGETFOLDER, testopts, null)) { - var r = c.Test(long.MaxValue); - Assert.AreEqual(0, r.Errors.Count()); - Assert.AreEqual(0, r.Warnings.Count()); - Assert.IsFalse(r.Verifications.Any(p => p.Value.Any())); + var listResults = c.List(); + TestUtils.AssertResults(listResults); + Assert.AreEqual(3, listResults.Filesets.Count()); + } + } + + [Test] + [Category("Disruption")] + [TestCase(true, false)] + [TestCase(false, false)] + [TestCase(true, true)] + [TestCase(false, true)] + public void TestFailedUploadOfFirstDlistWithRepair(bool before, bool failOnLastDblock) + { + var testopts = TestOptions; + testopts["number-of-retries"] = "0"; + testopts["dblock-size"] = "10mb"; + + // Deterministic error backend + Library.DynamicLoader.BackendLoader.AddBackend(new DeterministicErrorBackend()); + var failtarget = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER; + var operation = before + ? DeterministicErrorBackend.BackendAction.PutBefore + : DeterministicErrorBackend.BackendAction.PutAfter; + + // We expect to upload 7 volumes, and fail on the last one + var count = 7; + var failed = false; + + // Fail when uploading the dlist file + DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) => + { + if (failOnLastDblock) + { + if (action == operation && remotename.Contains(".dblock.")) + { + if (Interlocked.Decrement(ref count) == 0) + { + // Allow others to progress + Thread.Sleep(2000); + failed = true; + return true; + } + } + } + else + { + if (action == operation && remotename.Contains(".dlist.")) + { + failed = true; + return true; + } + } + + return false; + }; + + using (var c = new Controller(failtarget, testopts, null)) + Assert.Throws(() => c.Backup(new string[] { DATAFOLDER })); + + Assert.That(failed, Is.True, "Failed to fail the upload"); + + // Make spacing for the next backup + Thread.Sleep(3000); + + // Issue repair + using (var c = new Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Repair()); + + // Verify that all is in order + using (var c = new Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null)) + TestUtils.AssertResults(c.Test(long.MaxValue)); + + // Make a new backup, should continue as normal + using (var c = new Controller("file://" + TARGETFOLDER, testopts, null)) + { + var backupResults = c.Backup(new string[] { DATAFOLDER }); + TestUtils.AssertResults(backupResults); + Assert.AreEqual(2, c.List().Filesets.Count()); } // Test that we can recreate @@ -841,20 +877,15 @@ namespace Duplicati.UnitTest testopts["dbpath"] = recreatedDatabaseFile; - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) - { - IRepairResults repairResults = c.Repair(); - Assert.AreEqual(0, repairResults.Errors.Count()); - Assert.AreEqual(0, repairResults.Warnings.Count()); - } + using (var c = new Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Repair()); - // Check that we have 3 versions - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + // Check that we have 2 versions + using (var c = new Controller("file://" + TARGETFOLDER, testopts, null)) { - IListResults listResults = c.List(); - Assert.AreEqual(0, listResults.Errors.Count()); - Assert.AreEqual(0, listResults.Warnings.Count()); - Assert.AreEqual(3, listResults.Filesets.Count()); + var listResults = c.List(); + TestUtils.AssertResults(listResults); + Assert.AreEqual(2, listResults.Filesets.Count()); } } @@ -909,7 +940,7 @@ namespace Duplicati.UnitTest return false; }; - using (var c = new Library.Main.Controller(failtarget, testopts, null)) + using (var c = new Controller(failtarget, testopts, null)) Assert.Throws(() => c.Backup(new string[] { DATAFOLDER })); Assert.That(failed, Is.True, "Failed to fail the upload"); @@ -918,22 +949,16 @@ namespace Duplicati.UnitTest Thread.Sleep(3000); // Make a new backup, should continue as normal - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + using (var c = new Controller("file://" + TARGETFOLDER, testopts, null)) { IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER }); - Assert.AreEqual(0, backupResults.Errors.Count()); - Assert.AreEqual(0, backupResults.Warnings.Count()); + TestUtils.AssertResults(backupResults); Assert.AreEqual(2, c.List().Filesets.Count()); } // Verify that all is in order - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null)) - { - var r = c.Test(long.MaxValue); - Assert.AreEqual(0, r.Errors.Count()); - Assert.AreEqual(0, r.Warnings.Count()); - Assert.IsFalse(r.Verifications.Any(p => p.Value.Any())); - } + using (var c = new Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null)) + TestUtils.AssertResults(c.Test(long.MaxValue)); // Test that we can recreate var recreatedDatabaseFile = Path.Combine(BASEFOLDER, "recreated-database.sqlite"); @@ -942,19 +967,14 @@ namespace Duplicati.UnitTest testopts["dbpath"] = recreatedDatabaseFile; - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) - { - IRepairResults repairResults = c.Repair(); - Assert.AreEqual(0, repairResults.Errors.Count()); - Assert.AreEqual(0, repairResults.Warnings.Count()); - } + using (var c = new Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Repair()); // Check that we have 2 versions - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + using (var c = new Controller("file://" + TARGETFOLDER, testopts, null)) { - IListResults listResults = c.List(); - Assert.AreEqual(0, listResults.Errors.Count()); - Assert.AreEqual(0, listResults.Warnings.Count()); + var listResults = c.List(); + TestUtils.AssertResults(listResults); Assert.AreEqual(2, listResults.Filesets.Count()); } } @@ -985,23 +1005,18 @@ namespace Duplicati.UnitTest return false; }; - using (var c = new Library.Main.Controller(failtarget, testopts, null)) + using (var c = new Controller(failtarget, testopts, null)) Assert.Throws(() => c.Backup(new string[] { DATAFOLDER })); // Test that we can repair - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) - { - IRepairResults repairResults = c.Repair(); - Assert.AreEqual(0, repairResults.Errors.Count()); - Assert.AreEqual(0, repairResults.Warnings.Count()); - } + using (var c = new Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Repair()); // Check that we have 1 version - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + using (var c = new Controller("file://" + TARGETFOLDER, testopts, null)) { - IListResults listResults = c.List(); - Assert.AreEqual(0, listResults.Errors.Count()); - Assert.AreEqual(0, listResults.Warnings.Count()); + var listResults = c.List(); + TestUtils.AssertResults(listResults); Assert.AreEqual(1, listResults.Filesets.Count()); } } @@ -1026,11 +1041,7 @@ namespace Duplicati.UnitTest { // Make a regular backup using (var c = new Controller("file://" + TARGETFOLDER, testopts, null)) - { - IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER }); - Assert.AreEqual(0, backupResults.Errors.Count()); - Assert.AreEqual(0, backupResults.Warnings.Count()); - } + TestUtils.AssertResults(c.Backup(new string[] { DATAFOLDER })); } @@ -1056,7 +1067,13 @@ namespace Duplicati.UnitTest ModifySourceFiles(); using (var c = new Controller(failtarget, testopts, null)) - Assert.Throws(() => c.Backup(new string[] { DATAFOLDER })); + try + { + c.Backup(new string[] { DATAFOLDER }); + } + catch (DeterministicErrorBackend.DeterministicErrorBackendException) + { + } // Prevent clashes in timestamps Thread.Sleep(3000); @@ -1068,20 +1085,14 @@ namespace Duplicati.UnitTest // Make a new backup, should continue as normal using (var c = new Controller("file://" + TARGETFOLDER, testopts, null)) { - IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER }); - Assert.AreEqual(0, backupResults.Errors.Count()); - Assert.AreEqual(0, backupResults.Warnings.Count()); + var backupResults = c.Backup(new string[] { DATAFOLDER }); + TestUtils.AssertResults(backupResults); Assert.AreEqual(expectedFilesets, c.List().Filesets.Count()); } // Verify that all is in order - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null)) - { - var r = c.Test(long.MaxValue); - Assert.AreEqual(0, r.Errors.Count()); - Assert.AreEqual(0, r.Warnings.Count()); - Assert.IsFalse(r.Verifications.Any(p => p.Value.Any())); - } + using (var c = new Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null)) + TestUtils.AssertResults(c.Test(long.MaxValue)); // Test that we can recreate var recreatedDatabaseFile = Path.Combine(BASEFOLDER, "recreated-database.sqlite"); @@ -1090,19 +1101,14 @@ namespace Duplicati.UnitTest testopts["dbpath"] = recreatedDatabaseFile; - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) - { - IRepairResults repairResults = c.Repair(); - Assert.AreEqual(0, repairResults.Errors.Count()); - Assert.AreEqual(0, repairResults.Warnings.Count()); - } + using (var c = new Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Repair()); // Check that we have 2 versions - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + using (var c = new Controller("file://" + TARGETFOLDER, testopts, null)) { - IListResults listResults = c.List(); - Assert.AreEqual(0, listResults.Errors.Count()); - Assert.AreEqual(0, listResults.Warnings.Count()); + var listResults = c.List(); + TestUtils.AssertResults(listResults); Assert.AreEqual(expectedFilesets, listResults.Filesets.Count()); } } @@ -1165,12 +1171,8 @@ namespace Duplicati.UnitTest testopts["dblock-size"] = "10mb"; // Make a base backup - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) - { - IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER }); - Assert.AreEqual(0, backupResults.Errors.Count()); - Assert.AreEqual(0, backupResults.Warnings.Count()); - } + using (var c = new Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup(new string[] { DATAFOLDER })); // Make a new backup that fails uploading a dblock file ModifySourceFiles(); @@ -1194,7 +1196,7 @@ namespace Duplicati.UnitTest return false; }; - using (var c = new Library.Main.Controller(failtarget, testopts, null)) + using (var c = new Controller(failtarget, testopts, null)) { var sink = new LogSink(); c.AppendSink(sink); @@ -1214,7 +1216,7 @@ namespace Duplicati.UnitTest failstep = DeterministicErrorBackend.BackendAction.PutAfter; ModifySourceFiles(); - using (var c = new Library.Main.Controller(failtarget, testopts, null)) + using (var c = new Controller(failtarget, testopts, null)) { var sink = new LogSink(); c.AppendSink(sink); @@ -1232,21 +1234,12 @@ namespace Duplicati.UnitTest ModifySourceFiles(); // Create a regular backup - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) - { - IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER }); - Assert.AreEqual(0, backupResults.Errors.Count()); - Assert.AreEqual(0, backupResults.Warnings.Count()); - } + using (var c = new Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup(new string[] { DATAFOLDER })); // Verify that all is in order - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null)) - { - var r = c.Test(long.MaxValue); - Assert.AreEqual(0, r.Errors.Count()); - Assert.AreEqual(0, r.Warnings.Count()); - Assert.IsFalse(r.Verifications.Any(p => p.Value.Any())); - } + using (var c = new Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null)) + TestUtils.AssertResults(c.Test(long.MaxValue)); // Test that we can recreate var recreatedDatabaseFile = Path.Combine(BASEFOLDER, "recreated-database.sqlite"); @@ -1255,19 +1248,14 @@ namespace Duplicati.UnitTest testopts["dbpath"] = recreatedDatabaseFile; - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) - { - IRepairResults repairResults = c.Repair(); - Assert.AreEqual(0, repairResults.Errors.Count()); - Assert.AreEqual(0, repairResults.Warnings.Count()); - } + using (var c = new Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Repair()); // Check that we have 3 versions - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + using (var c = new Controller("file://" + TARGETFOLDER, testopts, null)) { - IListResults listResults = c.List(); - Assert.AreEqual(0, listResults.Errors.Count()); - Assert.AreEqual(0, listResults.Warnings.Count()); + var listResults = c.List(); + TestUtils.AssertResults(listResults); Assert.AreEqual(4, listResults.Filesets.Count()); } } diff --git a/Duplicati/UnitTest/TestUtils.cs b/Duplicati/UnitTest/TestUtils.cs index 596a193f1..b5e1e5e98 100644 --- a/Duplicati/UnitTest/TestUtils.cs +++ b/Duplicati/UnitTest/TestUtils.cs @@ -291,6 +291,9 @@ namespace Duplicati.UnitTest } Assert.AreEqual(0, results.Errors.Count(), "{0} errors:\n{1}", operation, string.Join("\n", results.Errors)); Assert.AreEqual(0, results.Warnings.Count(), "{0} warnings:\n{1}", operation, string.Join("\n", results.Warnings)); + + if (results is ITestResults testResults) + Assert.IsFalse(testResults.Verifications.Any(p => p.Value.Any()), "{0} verifications:\n{1}", operation, string.Join("\n", testResults.Verifications.Select(p => string.Join("\n", p.Value)))); } } } From d053470cf96f4e4e6c9826e0660d5fec71fb1eb8 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 10 Mar 2025 14:40:13 +0100 Subject: [PATCH 335/362] Added additional tests for repair code --- Duplicati/UnitTest/RepairHandlerTests.cs | 126 +++++++++++++++++++++++ 1 file changed, 126 insertions(+) diff --git a/Duplicati/UnitTest/RepairHandlerTests.cs b/Duplicati/UnitTest/RepairHandlerTests.cs index 790d459d4..2b3e3b4db 100644 --- a/Duplicati/UnitTest/RepairHandlerTests.cs +++ b/Duplicati/UnitTest/RepairHandlerTests.cs @@ -43,6 +43,11 @@ namespace Duplicati.UnitTest File.WriteAllBytes(Path.Combine(this.DATAFOLDER, "file"), new byte[] { 0 }); } + public void ModifyFile() + { + File.WriteAllBytes(Path.Combine(this.DATAFOLDER, "file"), new byte[] { 1 }); + } + [Test] [Category("RepairHandler")] public void RepairMissingBlocklistHashes() @@ -429,5 +434,126 @@ namespace Duplicati.UnitTest } } } + + [Test] + [Category("RepairHandler")] + public void RepairMissingDlistFile() + { + // Make two backups + var options = this.TestOptions; + using (var c = new Controller("file://" + this.TARGETFOLDER, options, null)) + TestUtils.AssertResults(c.Backup(new[] { this.DATAFOLDER })); + + ModifyFile(); + using (var c = new Controller("file://" + this.TARGETFOLDER, options, null)) + TestUtils.AssertResults(c.Backup(new[] { this.DATAFOLDER })); + + // Find and delete a dlist file + var dlistFile = Directory.EnumerateFiles(this.TARGETFOLDER, "*.dlist.*").First(); + File.Delete(dlistFile); + + using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) + { + IRepairResults repairResults = c.Repair(); + TestUtils.AssertResults(repairResults); + Assert.AreEqual(2, Directory.EnumerateFiles(this.TARGETFOLDER, "*.dlist.*").Count()); + } + } + + [Test] + [Category("RepairHandler")] + [TestCase("true")] + [TestCase("false")] + public void RepairMissingDlistVolume(bool deleteRemoteFile) + { + // Make two backups + var options = this.TestOptions; + using (var c = new Controller("file://" + this.TARGETFOLDER, options, null)) + TestUtils.AssertResults(c.Backup(new[] { this.DATAFOLDER })); + + ModifyFile(); + using (var c = new Controller("file://" + this.TARGETFOLDER, options, null)) + TestUtils.AssertResults(c.Backup(new[] { this.DATAFOLDER })); + + // Find a dlist file + var dlistFile = Directory.EnumerateFiles(this.TARGETFOLDER, "*.dlist.*").First(); + if (deleteRemoteFile) + File.Delete(dlistFile); + using (var con = SQLiteLoader.LoadConnection(options["dbpath"])) + using (var cmd = con.CreateCommand()) + Assert.AreEqual(1, cmd.ExecuteNonQuery("DELETE FROM RemoteVolume WHERE Name = ?", Path.GetFileName(dlistFile))); + + // Should catch this in validation + using (var c = new Controller("file://" + this.TARGETFOLDER, options, null)) + Assert.Throws(() => c.Backup(new[] { this.DATAFOLDER })); + + using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) + { + IRepairResults repairResults = c.Repair(); + TestUtils.AssertResults(repairResults); + Assert.AreEqual(2, Directory.EnumerateFiles(this.TARGETFOLDER, "*.dlist.*").Count()); + Assert.AreEqual(2, c.List().Filesets.Count()); + } + + // Check that the entry was recreated + using (var con = SQLiteLoader.LoadConnection(options["dbpath"])) + using (var cmd = con.CreateCommand()) + Assert.AreEqual(2, cmd.ExecuteScalarInt64("SELECT COUNT(*) FROM RemoteVolume WHERE Name LIKE '%.dlist.%' AND State != ?", -1, RemoteVolumeState.Deleted.ToString())); + + // Delete the database and check that the result is correct + File.Delete(options["dbpath"]); + + using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) + { + IRepairResults repairResults = c.Repair(); + TestUtils.AssertResults(repairResults); + Assert.AreEqual(2, c.List().Filesets.Count()); + + var r = c.Test(long.MaxValue); + Assert.AreEqual(0, r.Errors.Count()); + Assert.AreEqual(0, r.Warnings.Count()); + Assert.IsFalse(r.Verifications.Any(p => p.Value.Any())); + } + } + + [Test] + [Category("RepairHandler")] + public void RepairMissingFilesetVolume() + { + // Make two backups + var options = this.TestOptions; + using (var c = new Controller("file://" + this.TARGETFOLDER, options, null)) + TestUtils.AssertResults(c.Backup(new[] { this.DATAFOLDER })); + + ModifyFile(); + using (var c = new Controller("file://" + this.TARGETFOLDER, options, null)) + TestUtils.AssertResults(c.Backup(new[] { this.DATAFOLDER })); + + // Remove a fileset + using (var con = SQLiteLoader.LoadConnection(options["dbpath"])) + using (var cmd = con.CreateCommand()) + { + var filesetId = cmd.ExecuteScalarInt64("SELECT Id FROM Fileset ORDER BY Id DESC LIMIT 1"); + Assert.AreEqual(1, cmd.ExecuteNonQuery("DELETE FROM Fileset WHERE Id = ?", filesetId)); + cmd.ExecuteNonQuery("DELETE FROM FilesetEntry WHERE FilesetId = ?", filesetId); + } + + // Should catch this in validation + using (var c = new Controller("file://" + this.TARGETFOLDER, options, null)) + Assert.Throws(() => c.Backup(new[] { this.DATAFOLDER })); + + using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) + { + IRepairResults repairResults = c.Repair(); + TestUtils.AssertResults(repairResults); + Assert.AreEqual(2, Directory.EnumerateFiles(this.TARGETFOLDER, "*.dlist.*").Count()); + Assert.AreEqual(2, c.List().Filesets.Count()); + } + + // Check that entry was recreated + using (var con = SQLiteLoader.LoadConnection(options["dbpath"])) + using (var cmd = con.CreateCommand()) + Assert.AreEqual(2, cmd.ExecuteScalarInt64("SELECT COUNT(*) FROM Fileset")); + } } } \ No newline at end of file From 55fc5d2e6c498ccd2ced3a32b594a6d1531a50d3 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 10 Mar 2025 14:41:12 +0100 Subject: [PATCH 336/362] Added support for recreating filesets and remote lists in some scenarios --- .../Main/Database/LocalRepairDatabase.cs | 214 ++++++++++-------- .../Main/Operation/RecreateDatabaseHandler.cs | 153 +++++++------ .../Library/Main/Operation/RepairHandler.cs | 68 +++++- 3 files changed, 258 insertions(+), 177 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs index 421115a75..600cdc4a0 100644 --- a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -37,17 +37,17 @@ namespace Duplicati.Library.Main.Database public LocalRepairDatabase(string path) : base(path, "Repair", true) { - + } - + public long GetFilesetIdFromRemotename(string filelist) { - using(var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand()) { var filesetid = cmd.ExecuteScalarInt64(@"SELECT ""Fileset"".""ID"" FROM ""Fileset"",""RemoteVolume"" WHERE ""Fileset"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""RemoteVolume"".""Name"" = ?", -1, filelist); if (filesetid == -1) throw new Exception(string.Format("No such remote file: {0}", filelist)); - + return filesetid; } } @@ -57,48 +57,48 @@ namespace Duplicati.Library.Main.Database string File { get; } long Offset { get; } } - - public interface IBlockWithSources : LocalBackupDatabase.IBlock + + public interface IBlockWithSources : LocalBackupDatabase.IBlock { IEnumerable Sources { get; } } - + private class BlockWithSources : LocalBackupDatabase.Block, IBlockWithSources { private class BlockSource : IBlockSource { public string File { get; private set; } public long Offset { get; private set; } - + public BlockSource(string file, long offset) { this.File = file; this.Offset = offset; - + } } private readonly System.Data.IDataReader m_rd; public bool Done { get; private set; } - + public BlockWithSources(System.Data.IDataReader rd) : base(rd.GetString(0), rd.GetInt64(1)) { m_rd = rd; Done = !m_rd.Read(); } - + public IEnumerable Sources { get { if (Done) yield break; - + var cur = new BlockSource(m_rd.GetString(2), m_rd.GetInt64(3)); var file = cur.File; - - while(!Done && cur.File == file) + + while (!Done && cur.File == file) { yield return cur; Done = m_rd.Read(); @@ -108,13 +108,13 @@ namespace Duplicati.Library.Main.Database } } } - + private class RemoteVolume : IRemoteVolume { public string Name { get; private set; } public string Hash { get; private set; } public long Size { get; private set; } - + public RemoteVolume(string name, string hash, long size) { this.Name = name; @@ -122,14 +122,14 @@ namespace Duplicati.Library.Main.Database this.Size = size; } } - + public IEnumerable GetBlockVolumesFromIndexName(string name) { - using(var cmd = m_connection.CreateCommand()) - foreach(var rd in cmd.ExecuteReaderEnumerable(@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""ID"" IN (SELECT ""BlockVolumeID"" FROM ""IndexBlockLink"" WHERE ""IndexVolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" = ?))", name)) + using (var cmd = m_connection.CreateCommand()) + foreach (var rd in cmd.ExecuteReaderEnumerable(@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""ID"" IN (SELECT ""BlockVolumeID"" FROM ""IndexBlockLink"" WHERE ""IndexVolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" = ?))", name)) yield return new RemoteVolume(rd.GetString(0), rd.ConvertValueToString(1), rd.ConvertValueToInt64(2)); } - + public interface IMissingBlockList : IDisposable { bool SetBlockRestored(string hash, long size); @@ -138,7 +138,7 @@ namespace Duplicati.Library.Main.Database IEnumerable GetFilesetsUsingMissingBlocks(); IEnumerable GetMissingBlockSources(); } - + private class MissingBlockList : IMissingBlockList { private readonly System.Data.IDbConnection m_connection; @@ -146,19 +146,19 @@ namespace Duplicati.Library.Main.Database private System.Data.IDbCommand m_insertCommand; private string m_tablename; private readonly string m_volumename; - + public MissingBlockList(string volumename, System.Data.IDbConnection connection, System.Data.IDbTransaction transaction) { m_connection = connection; m_transaction = new TemporaryTransactionWrapper(m_connection, transaction); m_volumename = volumename; var tablename = "MissingBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - using(var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand()) { cmd.Transaction = m_transaction.Parent; cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Restored"" INTEGER NOT NULL) ", tablename)); m_tablename = tablename; - + var blockCount = cmd.ExecuteNonQuery(string.Format(@"INSERT INTO ""{0}"" (""Hash"", ""Size"", ""Restored"") SELECT DISTINCT ""Block"".""Hash"", ""Block"".""Size"", 0 AS ""Restored"" FROM ""Block"",""Remotevolume"" WHERE ""Block"".""VolumeID"" = ""Remotevolume"".""ID"" AND ""Remotevolume"".""Name"" = ? ", m_tablename), volumename); if (blockCount == 0) throw new Exception(string.Format("Unexpected empty block volume: {0}", volumename)); @@ -171,7 +171,7 @@ namespace Duplicati.Library.Main.Database m_insertCommand.CommandText = string.Format(@"UPDATE ""{0}"" SET ""Restored"" = ? WHERE ""Hash"" = ? AND ""Size"" = ? AND ""Restored"" = ? ", tablename); m_insertCommand.AddParameters(4); } - + public bool SetBlockRestored(string hash, long size) { m_insertCommand.SetParameterValue(0, 1); @@ -180,65 +180,65 @@ namespace Duplicati.Library.Main.Database m_insertCommand.SetParameterValue(3, 0); return m_insertCommand.ExecuteNonQuery() == 1; } - + public IEnumerable GetSourceFilesWithBlocks(long blocksize) { - using(var cmd = m_connection.CreateCommand(m_transaction.Parent)) - using(var rd = cmd.ExecuteReader(string.Format(@"SELECT DISTINCT ""{0}"".""Hash"", ""{0}"".""Size"", ""File"".""Path"", ""BlocksetEntry"".""Index"" * {1} FROM ""{0}"", ""Block"", ""BlocksetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""{0}"".""Hash"" = ""Block"".""Hash"" AND ""{0}"".""Size"" = ""Block"".""Size"" AND ""{0}"".""Restored"" = ? ", m_tablename, blocksize), 0)) - if (rd.Read()) - { - var bs = new BlockWithSources(rd); - while (!bs.Done) - yield return bs; - } + using (var cmd = m_connection.CreateCommand(m_transaction.Parent)) + using (var rd = cmd.ExecuteReader(string.Format(@"SELECT DISTINCT ""{0}"".""Hash"", ""{0}"".""Size"", ""File"".""Path"", ""BlocksetEntry"".""Index"" * {1} FROM ""{0}"", ""Block"", ""BlocksetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""{0}"".""Hash"" = ""Block"".""Hash"" AND ""{0}"".""Size"" = ""Block"".""Size"" AND ""{0}"".""Restored"" = ? ", m_tablename, blocksize), 0)) + if (rd.Read()) + { + var bs = new BlockWithSources(rd); + while (!bs.Done) + yield return bs; + } } - + public IEnumerable> GetMissingBlocks() { - using(var cmd = m_connection.CreateCommand(m_transaction.Parent)) - foreach(var rd in cmd.ExecuteReaderEnumerable(string.Format(@"SELECT ""{0}"".""Hash"", ""{0}"".""Size"" FROM ""{0}"" WHERE ""{0}"".""Restored"" = ? ", m_tablename), 0)) + using (var cmd = m_connection.CreateCommand(m_transaction.Parent)) + foreach (var rd in cmd.ExecuteReaderEnumerable(string.Format(@"SELECT ""{0}"".""Hash"", ""{0}"".""Size"" FROM ""{0}"" WHERE ""{0}"".""Restored"" = ? ", m_tablename), 0)) yield return new KeyValuePair(rd.ConvertValueToString(0), rd.ConvertValueToInt64(1)); } - + public IEnumerable GetFilesetsUsingMissingBlocks() { var blocks = @"SELECT DISTINCT ""FileLookup"".""ID"" AS ID FROM ""{0}"", ""Block"", ""Blockset"", ""BlocksetEntry"", ""FileLookup"" WHERE ""Block"".""Hash"" = ""{0}"".""Hash"" AND ""Block"".""Size"" = ""{0}"".""Size"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" "; var blocklists = @"SELECT DISTINCT ""FileLookup"".""ID"" AS ID FROM ""{0}"", ""Block"", ""Blockset"", ""BlocklistHash"", ""FileLookup"" WHERE ""Block"".""Hash"" = ""{0}"".""Hash"" AND ""Block"".""Size"" = ""{0}"".""Size"" AND ""BlocklistHash"".""Hash"" = ""Block"".""Hash"" AND ""BlocklistHash"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" "; - + var cmdtxt = @"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""FilesetEntry"", ""Fileset"" WHERE ""RemoteVolume"".""ID"" = ""Fileset"".""VolumeID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""RemoteVolume"".""Type"" = ? AND ""FilesetEntry"".""FileID"" IN (SELECT DISTINCT ""ID"" FROM ( " + blocks + " UNION " + blocklists + " ))"; - - using(var cmd = m_connection.CreateCommand(m_transaction.Parent)) - foreach(var rd in cmd.ExecuteReaderEnumerable(string.Format(cmdtxt, m_tablename), RemoteVolumeType.Files.ToString())) + + using (var cmd = m_connection.CreateCommand(m_transaction.Parent)) + foreach (var rd in cmd.ExecuteReaderEnumerable(string.Format(cmdtxt, m_tablename), RemoteVolumeType.Files.ToString())) yield return new RemoteVolume(rd.GetString(0), rd.ConvertValueToString(1), rd.ConvertValueToInt64(2)); } - + public IEnumerable GetMissingBlockSources() { - using(var cmd = m_connection.CreateCommand(m_transaction.Parent)) - foreach(var rd in cmd.ExecuteReaderEnumerable(string.Format(@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""Block"", ""{0}"" WHERE ""Block"".""Hash"" = ""{0}"".""Hash"" AND ""Block"".""Size"" = ""{0}"".""Size"" AND ""Block"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""Remotevolume"".""Name"" != ? ", m_tablename), m_volumename)) + using (var cmd = m_connection.CreateCommand(m_transaction.Parent)) + foreach (var rd in cmd.ExecuteReaderEnumerable(string.Format(@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""Block"", ""{0}"" WHERE ""Block"".""Hash"" = ""{0}"".""Hash"" AND ""Block"".""Size"" = ""{0}"".""Size"" AND ""Block"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""Remotevolume"".""Name"" != ? ", m_tablename), m_volumename)) yield return new RemoteVolume(rd.GetString(0), rd.ConvertValueToString(1), rd.ConvertValueToInt64(2)); } - + public void Dispose() { if (m_tablename != null) { try { - using(var cmd = m_connection.CreateCommand(m_transaction.Parent)) + using (var cmd = m_connection.CreateCommand(m_transaction.Parent)) cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", m_transaction)); } catch { } finally { m_tablename = null; } } - + if (m_insertCommand != null) try { m_insertCommand.Dispose(); } - catch {} + catch { } finally { m_insertCommand = null; } } } - + public IMissingBlockList CreateBlockList(string volumename, System.Data.IDbTransaction transaction = null) { return new MissingBlockList(volumename, m_connection, transaction); @@ -246,12 +246,12 @@ namespace Duplicati.Library.Main.Database public void FixDuplicateMetahash() { - using(var tr = m_connection.BeginTransaction()) - using(var cmd = m_connection.CreateCommand(tr)) + using (var tr = m_connection.BeginTransaction()) + using (var cmd = m_connection.CreateCommand(tr)) { cmd.Transaction = tr; - var sql_count = + var sql_count = @"SELECT COUNT(*) FROM (" + @" SELECT DISTINCT c1 FROM (" + @"SELECT COUNT(*) AS ""C1"" FROM (SELECT DISTINCT ""BlocksetID"" FROM ""Metadataset"") UNION SELECT COUNT(*) AS ""C1"" FROM ""Metadataset"" " + @@ -269,11 +269,11 @@ namespace Duplicati.Library.Main.Database var sql = @"SELECT ""A"".""ID"", ""B"".""BlocksetID"" FROM (SELECT MIN(""ID"") AS ""ID"", COUNT(""ID"") AS ""Duplicates"" FROM ""Metadataset"" GROUP BY ""BlocksetID"") ""A"", ""Metadataset"" ""B"" WHERE ""A"".""Duplicates"" > 1 AND ""A"".""ID"" = ""B"".""ID"""; - using(var c2 = m_connection.CreateCommand(tr)) + using (var c2 = m_connection.CreateCommand(tr)) { c2.CommandText = string.Format(@"UPDATE ""{0}"" SET ""MetadataID"" = ? WHERE ""MetadataID"" IN (SELECT ""ID"" FROM ""Metadataset"" WHERE ""BlocksetID"" = ?)", tablename); c2.CommandText += @"; DELETE FROM ""Metadataset"" WHERE ""BlocksetID"" = ? AND ""ID"" != ?"; - using(var rd = cmd.ExecuteReader(sql)) + using (var rd = cmd.ExecuteReader(sql)) while (rd.Read()) c2.ExecuteNonQuery(null, rd.GetValue(0), rd.GetValue(1), rd.GetValue(1), rd.GetValue(0)); } @@ -282,12 +282,12 @@ namespace Duplicati.Library.Main.Database SELECT MIN(""ID"") AS ""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Entries"" FROM ""{0}"" GROUP BY ""Path"", ""BlocksetID"", ""MetadataID"") WHERE ""Entries"" > 1 ORDER BY ""ID""", tablename); - using(var c2 = m_connection.CreateCommand()) + using (var c2 = m_connection.CreateCommand()) { c2.Transaction = tr; c2.CommandText = string.Format(@"UPDATE ""FilesetEntry"" SET ""FileID"" = ? WHERE ""FileID"" IN (SELECT ""ID"" FROM ""{0}"" WHERE ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ?)", tablename); c2.CommandText += string.Format(@"; DELETE FROM ""{0}"" WHERE ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ? AND ""ID"" != ?", tablename); - foreach(var rd in cmd.ExecuteReaderEnumerable(sql)) + foreach (var rd in cmd.ExecuteReaderEnumerable(sql)) c2.ExecuteNonQuery(null, rd.GetValue(0), rd.GetValue(1), rd.GetValue(2), rd.GetValue(3), rd.GetValue(1), rd.GetValue(2), rd.GetValue(3), rd.GetValue(0)); } @@ -309,8 +309,8 @@ namespace Duplicati.Library.Main.Database public void FixDuplicateFileentries() { - using(var tr = m_connection.BeginTransaction()) - using(var cmd = m_connection.CreateCommand(tr)) + using (var tr = m_connection.BeginTransaction()) + using (var cmd = m_connection.CreateCommand(tr)) { var sql_count = @"SELECT COUNT(*) FROM (SELECT ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Duplicates"" FROM ""FileLookup"" GROUP BY ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"") WHERE ""Duplicates"" > 1"; @@ -323,11 +323,11 @@ namespace Duplicati.Library.Main.Database SELECT MIN(""ID"") AS ""ID"", ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Entries"" FROM ""FileLookup"" GROUP BY ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"") WHERE ""Entries"" > 1 ORDER BY ""ID"""; - using(var c2 = m_connection.CreateCommand(tr)) + using (var c2 = m_connection.CreateCommand(tr)) { c2.CommandText = @"UPDATE ""FilesetEntry"" SET ""FileID"" = ? WHERE ""FileID"" IN (SELECT ""ID"" FROM ""FileLookup"" WHERE ""PrefixID"" = ? AND ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ?)"; c2.CommandText += @"; DELETE FROM ""FileLookup"" WHERE ""PrefixID"" = ? AND ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ? AND ""ID"" != ?"; - foreach(var rd in cmd.ExecuteReaderEnumerable(sql)) + foreach (var rd in cmd.ExecuteReaderEnumerable(sql)) c2.ExecuteNonQuery(null, rd.GetValue(0), rd.GetValue(1), rd.GetValue(2), rd.GetValue(3), rd.GetValue(4), rd.GetValue(1), rd.GetValue(2), rd.GetValue(3), rd.GetValue(4), rd.GetValue(0)); } @@ -347,9 +347,9 @@ namespace Duplicati.Library.Main.Database { var blocklistbuffer = new byte[blocksize]; - using(var tr = m_connection.BeginTransaction()) - using(var cmd = m_connection.CreateCommand(tr)) - using(var blockhasher = HashFactory.CreateHasher(blockhashalgorithm)) + using (var tr = m_connection.BeginTransaction()) + using (var cmd = m_connection.CreateCommand(tr)) + using (var blockhasher = HashFactory.CreateHasher(blockhashalgorithm)) { var hashsize = blockhasher.HashSize / 8; @@ -360,18 +360,18 @@ namespace Duplicati.Library.Main.Database if (itemswithnoblocklisthash != 0) { Logging.Log.WriteInformationMessage(LOGTAG, "MissingBlocklistHashes", "Found {0} missing blocklisthash entries, repairing", itemswithnoblocklisthash); - using(var c2 = m_connection.CreateCommand(tr)) - using(var c3 = m_connection.CreateCommand(tr)) - using(var c4 = m_connection.CreateCommand(tr)) - using(var c5 = m_connection.CreateCommand(tr)) - using(var c6 = m_connection.CreateCommand(tr)) + using (var c2 = m_connection.CreateCommand(tr)) + using (var c3 = m_connection.CreateCommand(tr)) + using (var c4 = m_connection.CreateCommand(tr)) + using (var c5 = m_connection.CreateCommand(tr)) + using (var c6 = m_connection.CreateCommand(tr)) { c3.CommandText = @"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (?, ?, ?) "; c4.CommandText = @"SELECT COUNT(*) FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ?"; c5.CommandText = @"SELECT ""ID"" FROM ""DeletedBlock"" WHERE ""Hash"" = ? AND ""Size"" = ? AND ""VolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = ? AND (""State"" = ? OR ""State"" = ?))"; c6.CommandText = @"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") SELECT ""Hash"", ""Size"", ""VolumeID"" FROM ""DeletedBlock"" WHERE ""ID"" = ? LIMIT 1; DELETE FROM ""DeletedBlock"" WHERE ""ID"" = ?;"; - foreach(var e in cmd.ExecuteReaderEnumerable(sql)) + foreach (var e in cmd.ExecuteReaderEnumerable(sql)) { var blocksetid = e.ConvertValueToInt64(0); var ix = 0L; @@ -379,7 +379,7 @@ namespace Duplicati.Library.Main.Database c2.ExecuteNonQuery(@"DELETE FROM ""BlocklistHash"" WHERE ""BlocksetID"" = ?", blocksetid); - foreach(var h in c2.ExecuteReaderEnumerable(@"SELECT ""A"".""Hash"" FROM ""Block"" ""A"", ""BlocksetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""BlockID"" AND ""B"".""BlocksetID"" = ? ORDER BY ""B"".""Index""", blocksetid)) + foreach (var h in c2.ExecuteReaderEnumerable(@"SELECT ""A"".""Hash"" FROM ""Block"" ""A"", ""BlocksetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""BlockID"" AND ""B"".""BlocksetID"" = ? ORDER BY ""B"".""Index""", blocksetid)) { var tmp = Convert.FromBase64String(h.GetString(0)); if (blocklistbuffer.Length - blocklistoffset < tmp.Length) @@ -449,8 +449,8 @@ namespace Duplicati.Library.Main.Database public void FixDuplicateBlocklistHashes(long blocksize, long hashsize) { - using(var tr = m_connection.BeginTransaction()) - using(var cmd = m_connection.CreateCommand(tr)) + using (var tr = m_connection.BeginTransaction()) + using (var cmd = m_connection.CreateCommand(tr)) { var dup_sql = @"SELECT * FROM (SELECT ""BlocksetID"", ""Index"", COUNT(*) AS ""EC"" FROM ""BlocklistHash"" GROUP BY ""BlocksetID"", ""Index"") WHERE ""EC"" > 1"; @@ -463,10 +463,10 @@ namespace Duplicati.Library.Main.Database var unique_count = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM (SELECT DISTINCT ""BlocksetID"", ""Index"" FROM ""BlocklistHash"")", 0); - using(var c2 = m_connection.CreateCommand(tr)) + using (var c2 = m_connection.CreateCommand(tr)) { c2.CommandText = @"DELETE FROM ""BlocklistHash"" WHERE rowid IN (SELECT rowid FROM ""BlocklistHash"" WHERE ""BlocksetID"" = ? AND ""Index"" = ? LIMIT ?)"; - foreach(var rd in cmd.ExecuteReaderEnumerable(dup_sql)) + foreach (var rd in cmd.ExecuteReaderEnumerable(dup_sql)) { var expected = rd.GetInt32(2) - 1; var actual = c2.ExecuteNonQuery(null, rd.GetValue(0), rd.GetValue(1), expected); @@ -489,7 +489,7 @@ namespace Duplicati.Library.Main.Database { VerifyConsistency(blocksize, hashsize, true, tr); } - catch(Exception ex) + catch (Exception ex) { throw new Duplicati.Library.Interface.UserInformationException("Repaired blocklisthashes, but the database was broken afterwards, rolled back changes", "DuplicateBlocklistHashesRepairFailed", ex); } @@ -502,8 +502,8 @@ namespace Duplicati.Library.Main.Database public void CheckAllBlocksAreInVolume(string filename, IEnumerable> blocks) { - using(var tr = m_connection.BeginTransaction()) - using(var cmd = m_connection.CreateCommand(tr)) + using (var tr = m_connection.BeginTransaction()) + using (var cmd = m_connection.CreateCommand(tr)) { var tablename = "ProbeBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); @@ -511,7 +511,7 @@ namespace Duplicati.Library.Main.Database cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Hash"", ""Size"") VALUES (?, ?)", tablename); cmd.AddParameters(2); - foreach(var kp in blocks) + foreach (var kp in blocks) { cmd.SetParameterValue(0, kp.Key); cmd.SetParameterValue(1, kp.Value); @@ -530,7 +530,7 @@ namespace Duplicati.Library.Main.Database public void CheckBlocklistCorrect(string hash, long length, IEnumerable blocklist, long blocksize, long blockhashlength) { - using(var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand()) { var query = string.Format(@" SELECT @@ -565,7 +565,7 @@ ORDER BY ""A"".""Index"" " - ,blocksize, blockhashlength); + , blocksize, blockhashlength); using (var en = blocklist.GetEnumerator()) { @@ -582,6 +582,20 @@ ORDER BY } } } + + public IEnumerable MissingLocalFilesets() + { + using (var cmd = m_connection.CreateCommand()) + foreach (var rd in cmd.ExecuteReaderEnumerable(@"SELECT ""Name"" FROM ""RemoteVolume"" WHERE ""Type"" = ? AND ""State"" NOT IN (?, ?) AND ""ID"" NOT IN (SELECT ""VolumeID"" FROM ""Fileset"")", RemoteVolumeType.Files.ToString(), RemoteVolumeState.Deleting.ToString(), RemoteVolumeState.Deleted.ToString())) + yield return rd.ConvertValueToString(0); + } + + public IEnumerable<(long FilesetID, DateTime Timestamp, bool IsFull)> MissingRemoteFilesets() + { + using (var cmd = m_connection.CreateCommand()) + foreach (var rd in cmd.ExecuteReaderEnumerable(@"SELECT ""ID"", ""Timestamp"", ""IsFullBackup"" FROM ""Fileset"" WHERE ""VolumeID"" NOT IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = ? AND ""State"" NOT IN (?, ?))", RemoteVolumeType.Files.ToString(), RemoteVolumeState.Deleting.ToString(), RemoteVolumeState.Deleted.ToString())) + yield return (rd.ConvertValueToInt64(0), ParseFromEpochSeconds(rd.ConvertValueToInt64(1)), rd.ConvertValueToInt64(2) == BackupType.FULL_BACKUP); + } } } diff --git a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs index 66481fb30..7849e343d 100644 --- a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs +++ b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs @@ -20,6 +20,7 @@ // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; +using System.Data; using System.IO; using System.Linq; using System.Threading; @@ -205,9 +206,6 @@ namespace Duplicati.Library.Main.Operation } var isFirstFilelist = true; - var blocksize = m_options.Blocksize; - var hashes_pr_block = blocksize / m_options.BlockhashSize; - await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(filelistWork, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { var entry = new RemoteVolume(name, hash, size); @@ -249,83 +247,12 @@ namespace Duplicati.Library.Main.Operation { VolumeReaderBase.UpdateOptionsFromManifest(compressor, m_options); hasUpdatedOptions = true; - // Recompute the cached sizes - blocksize = m_options.Blocksize; - hashes_pr_block = blocksize / m_options.BlockhashSize; } // Create timestamped operations based on the file timestamp var filesetid = restoredb.CreateFileset(volumeIds[entry.Name], parsed.Time, tr); - // retrieve fileset data from dlist - var filesetData = VolumeReaderBase.GetFilesetData(compressor, m_options); - - // update fileset using filesetData - restoredb.UpdateFullBackupStateInFileset(filesetid, filesetData.IsFullBackup); - - using (var filelistreader = new FilesetVolumeReader(compressor, m_options)) - foreach (var fe in filelistreader.Files.Where(x => Library.Utility.FilterExpression.Matches(filter, x.Path))) - { - try - { - var expectedmetablocks = (fe.Metasize + blocksize - 1) / blocksize; - var expectedmetablocklisthashes = (expectedmetablocks + hashes_pr_block - 1) / hashes_pr_block; - if (expectedmetablocks <= 1) expectedmetablocklisthashes = 0; - - var metadataid = long.MinValue; - var split = Database.LocalDatabase.SplitIntoPrefixAndName(fe.Path); - var prefixid = restoredb.GetOrCreatePathPrefix(split.Key, tr); - - switch (fe.Type) - { - case FilelistEntryType.Folder: - metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, tr); - restoredb.AddDirectoryEntry(filesetid, prefixid, split.Value, fe.Time, metadataid, tr); - break; - case FilelistEntryType.File: - var expectedblocks = (fe.Size + blocksize - 1) / blocksize; - var expectedblocklisthashes = (expectedblocks + hashes_pr_block - 1) / hashes_pr_block; - if (expectedblocks <= 1) expectedblocklisthashes = 0; - - var blocksetid = restoredb.AddBlockset(fe.Hash, fe.Size, fe.BlocklistHashes, expectedblocklisthashes, tr); - metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, tr); - restoredb.AddFileEntry(filesetid, prefixid, split.Value, fe.Time, blocksetid, metadataid, tr); - - if (fe.Size <= blocksize) - { - if (!string.IsNullOrWhiteSpace(fe.Blockhash)) - restoredb.AddSmallBlocksetLink(fe.Hash, fe.Blockhash, fe.Blocksize, tr); - else if (m_options.BlockHashAlgorithm == m_options.FileHashAlgorithm) - restoredb.AddSmallBlocksetLink(fe.Hash, fe.Hash, fe.Size, tr); - else if (fe.Size > 0) - Logging.Log.WriteWarningMessage(LOGTAG, "MissingBlockHash", null, "No block hash found for file: {0}", fe.Path); - } - - break; - case FilelistEntryType.Symlink: - metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, tr); - restoredb.AddSymlinkEntry(filesetid, prefixid, split.Value, fe.Time, metadataid, tr); - break; - default: - Logging.Log.WriteWarningMessage(LOGTAG, "SkippingUnknownFileEntry", null, "Skipping file-entry with unknown type {0}: {1} ", fe.Type, fe.Path); - break; - } - - if (fe.Metasize <= blocksize && (fe.Type == FilelistEntryType.Folder || fe.Type == FilelistEntryType.File || fe.Type == FilelistEntryType.Symlink)) - { - if (!string.IsNullOrWhiteSpace(fe.Metablockhash)) - restoredb.AddSmallBlocksetLink(fe.Metahash, fe.Metablockhash, fe.Metasize, tr); - else if (m_options.BlockHashAlgorithm == m_options.FileHashAlgorithm) - restoredb.AddSmallBlocksetLink(fe.Metahash, fe.Metahash, fe.Metasize, tr); - else - Logging.Log.WriteWarningMessage(LOGTAG, "MissingMetadataBlockHash", null, "No block hash found for file metadata: {0}", fe.Path); - } - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "FileEntryProcessingFailed", ex, "Failed to process file-entry: {0}", fe.Path); - } - } + RecreateFilesetFromRemoteList(restoredb, tr, compressor, filesetid, m_options, filter); } } catch (Exception ex) @@ -633,6 +560,82 @@ namespace Duplicati.Library.Main.Operation } } + public static void RecreateFilesetFromRemoteList(LocalRecreateDatabase restoredb, IDbTransaction transaction, ICompression compressor, long filesetid, Options options, IFilter filter) + { + var blocksize = options.Blocksize; + var hashes_pr_block = blocksize / options.BlockhashSize; + + // retrieve fileset data from dlist + var filesetData = VolumeReaderBase.GetFilesetData(compressor, options); + + // update fileset using filesetData + restoredb.UpdateFullBackupStateInFileset(filesetid, filesetData.IsFullBackup, transaction); + + using (var filelistreader = new FilesetVolumeReader(compressor, options)) + foreach (var fe in filelistreader.Files.Where(x => Library.Utility.FilterExpression.Matches(filter, x.Path))) + { + try + { + var expectedmetablocks = (fe.Metasize + blocksize - 1) / blocksize; + var expectedmetablocklisthashes = (expectedmetablocks + hashes_pr_block - 1) / hashes_pr_block; + if (expectedmetablocks <= 1) expectedmetablocklisthashes = 0; + + var metadataid = long.MinValue; + var split = LocalDatabase.SplitIntoPrefixAndName(fe.Path); + var prefixid = restoredb.GetOrCreatePathPrefix(split.Key, transaction); + + switch (fe.Type) + { + case FilelistEntryType.Folder: + metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, transaction); + restoredb.AddDirectoryEntry(filesetid, prefixid, split.Value, fe.Time, metadataid, transaction); + break; + case FilelistEntryType.File: + var expectedblocks = (fe.Size + blocksize - 1) / blocksize; + var expectedblocklisthashes = (expectedblocks + hashes_pr_block - 1) / hashes_pr_block; + if (expectedblocks <= 1) expectedblocklisthashes = 0; + + var blocksetid = restoredb.AddBlockset(fe.Hash, fe.Size, fe.BlocklistHashes, expectedblocklisthashes, transaction); + metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, transaction); + restoredb.AddFileEntry(filesetid, prefixid, split.Value, fe.Time, blocksetid, metadataid, transaction); + + if (fe.Size <= blocksize) + { + if (!string.IsNullOrWhiteSpace(fe.Blockhash)) + restoredb.AddSmallBlocksetLink(fe.Hash, fe.Blockhash, fe.Blocksize, transaction); + else if (options.BlockHashAlgorithm == options.FileHashAlgorithm) + restoredb.AddSmallBlocksetLink(fe.Hash, fe.Hash, fe.Size, transaction); + else if (fe.Size > 0) + Logging.Log.WriteWarningMessage(LOGTAG, "MissingBlockHash", null, "No block hash found for file: {0}", fe.Path); + } + + break; + case FilelistEntryType.Symlink: + metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, transaction); + restoredb.AddSymlinkEntry(filesetid, prefixid, split.Value, fe.Time, metadataid, transaction); + break; + default: + Logging.Log.WriteWarningMessage(LOGTAG, "SkippingUnknownFileEntry", null, "Skipping file-entry with unknown type {0}: {1} ", fe.Type, fe.Path); + break; + } + + if (fe.Metasize <= blocksize && (fe.Type == FilelistEntryType.Folder || fe.Type == FilelistEntryType.File || fe.Type == FilelistEntryType.Symlink)) + { + if (!string.IsNullOrWhiteSpace(fe.Metablockhash)) + restoredb.AddSmallBlocksetLink(fe.Metahash, fe.Metablockhash, fe.Metasize, transaction); + else if (options.BlockHashAlgorithm == options.FileHashAlgorithm) + restoredb.AddSmallBlocksetLink(fe.Metahash, fe.Metahash, fe.Metasize, transaction); + else + Logging.Log.WriteWarningMessage(LOGTAG, "MissingMetadataBlockHash", null, "No block hash found for file metadata: {0}", fe.Path); + } + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "FileEntryProcessingFailed", ex, "Failed to process file-entry: {0}", fe.Path); + } + } + } + /// /// Look in the database for filenames similar to the current filename, but with a different compression and encryption module /// diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 4b280b1f6..9609b0b43 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -20,6 +20,7 @@ // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -141,8 +142,11 @@ namespace Duplicati.Library.Main.Operation var buffer = new byte[m_options.Blocksize]; var hashsize = HashFactory.HashSizeBytes(m_options.BlockHashAlgorithm); + var missingRemoteFilesets = db.MissingRemoteFilesets().ToList(); + var missingLocalFilesets = db.MissingLocalFilesets().ToList(); + var progress = 0; - var targetProgess = tp.ExtraVolumes.Count() + tp.MissingVolumes.Count() + tp.VerificationRequiredVolumes.Count(); + var targetProgess = tp.ExtraVolumes.Count() + tp.MissingVolumes.Count() + tp.VerificationRequiredVolumes.Count() + missingRemoteFilesets.Count + missingLocalFilesets.Count; var mostRecentLocal = db.FilesetTimes.Select(x => x.Value).Append(DateTime.MinValue).Max(); var mostRecentRemote = tp.ParsedVolumes.Select(x => x.Time).Append(DateTime.MinValue).Max(); @@ -166,7 +170,7 @@ namespace Duplicati.Library.Main.Operation } } - if (tp.ExtraVolumes.Any() || tp.MissingVolumes.Any() || tp.VerificationRequiredVolumes.Any()) + if (tp.ExtraVolumes.Any() || tp.MissingVolumes.Any() || tp.VerificationRequiredVolumes.Any() || missingRemoteFilesets.Any() || missingLocalFilesets.Any()) { if (tp.VerificationRequiredVolumes.Any()) { @@ -300,6 +304,66 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException($"The backup storage destination is missing data files. You can either enable `--rebuild-missing-dblock-files` or run the purge command to remove these files. The following files are missing: {string.Join(", ", missingDblocks.Select(x => x.Name))}", "MissingDblockFiles"); } + var anyDlistUploads = false; + foreach (var (filesetId, timestamp, isfull) in missingRemoteFilesets) + { + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + { + await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false); + return; + } + + progress++; + m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess); + var fileTime = FilesetVolumeWriter.ProbeUnusedFilenameName(db, m_options, timestamp); + + var fsw = new FilesetVolumeWriter(m_options, fileTime); + Logging.Log.WriteInformationMessage(LOGTAG, "ReuploadingFileset", "Re-uploading fileset {0} from {1} as remote volume registration is missing, new filename: {2}", filesetId, timestamp, fsw.RemoteFilename); + + if (!string.IsNullOrEmpty(m_options.ControlFiles)) + foreach (var p in m_options.ControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries)) + fsw.AddControlFile(p, m_options.GetCompressionHintFromFilename(p)); + + fsw.CreateFilesetFile(isfull); + db.WriteFileset(fsw, filesetId, null); + fsw.Close(); + + if (m_options.Dryrun) + { + fsw.Dispose(); + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadFileset", "would re-upload fileset {0}", fsw.RemoteFilename); + continue; + } + + fsw.VolumeID = db.RegisterRemoteVolume(fsw.RemoteFilename, RemoteVolumeType.Files, -1, RemoteVolumeState.Temporary); + db.LinkFilesetToVolume(filesetId, fsw.VolumeID, null); + await backendManager.PutAsync(fsw, null, null, false, cancellationToken).ConfigureAwait(false); + } + + if (anyDlistUploads) + await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false); + + foreach (var volumename in missingLocalFilesets) + { + var remoteVolume = db.GetRemoteVolume(volumename); + using (var tmpfile = await backendManager.GetAsync(remoteVolume.Name, remoteVolume.Hash, remoteVolume.Size, cancellationToken).ConfigureAwait(false)) + { + var parsed = VolumeBase.ParseFilename(remoteVolume.Name); + using (var stream = new FileStream(tmpfile, FileMode.Open, FileAccess.Read, FileShare.Read)) + using (var compressor = DynamicLoader.CompressionLoader.GetModule(parsed.CompressionModule, stream, ArchiveMode.Read, m_options.RawOptions)) + using (var transaction = db.BeginTransaction()) + using (var recreatedb = new LocalRecreateDatabase(db, m_options)) + { + if (compressor == null) + throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule"); + + var filesetid = db.CreateFileset(remoteVolume.ID, parsed.Time, transaction); + RecreateDatabaseHandler.RecreateFilesetFromRemoteList(recreatedb, transaction, compressor, filesetid, m_options, new FilterExpression()); + transaction.Commit(); + } + } + } + if (!m_options.Dryrun && tp.MissingVolumes.Any()) db.TerminatedWithActiveUploads = true; From d500097e053e157ff9d29ca1df817e7e2d19ee4e Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 10 Mar 2025 14:41:36 +0100 Subject: [PATCH 337/362] Added additional consistency checks to local database --- .../Library/Main/Database/LocalDatabase.cs | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 02edb5c52..4a9e09b08 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -27,6 +27,7 @@ using System.IO; using Duplicati.Library.Modules.Builtin.ResultSerialization; using Duplicati.Library.Utility; using System.Runtime.CompilerServices; +using Duplicati.Library.Interface; // Expose internal classes to UnitTests, so that Database classes can be tested @@ -875,6 +876,28 @@ ON if (cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""FileLookup"" WHERE ""BlocksetID"" != ? AND ""BlocksetID"" != ? AND NOT ""BlocksetID"" IN (SELECT ""ID"" FROM ""Blockset"")", 0, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID) != 0) throw new DatabaseInconsistencyException("Detected files associated with non-existing blocksets!"); + var filesetsMissingVolumes = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""Fileset"" WHERE ""VolumeID"" NOT IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = ? AND ""State"" != ?)", 0, RemoteVolumeType.Files.ToString(), RemoteVolumeState.Deleted.ToString()); + if (filesetsMissingVolumes != 0) + { + if (filesetsMissingVolumes == 1) + using (var reader = cmd.ExecuteReader(@"SELECT ""ID"", ""Timestamp"", ""VolumeID"" FROM ""Fileset"" WHERE ""VolumeID"" NOT IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = ? AND ""State"" != ?)", 0, RemoteVolumeType.Files.ToString(), RemoteVolumeState.Deleted.ToString())) + if (reader.Read()) + throw new DatabaseInconsistencyException($"Detected 1 fileset with missing volume: FilesetId = {reader.ConvertValueToInt64(0)}, Time = ({ParseFromEpochSeconds(reader.ConvertValueToInt64(1))}), unmatched VolumeID {reader.ConvertValueToInt64(2)}"); + + throw new DatabaseInconsistencyException(string.Format("Detected {0} filesets with missing volumes", filesetsMissingVolumes)); + } + + var volumesMissingFilests = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""RemoteVolume"" WHERE ""Type"" = ? AND ""State"" != ? AND ""ID"" NOT IN (SELECT ""VolumeID"" FROM ""Fileset"")", 0, RemoteVolumeType.Files.ToString(), RemoteVolumeState.Deleted.ToString()); + if (volumesMissingFilests != 0) + { + if (volumesMissingFilests == 1) + using (var reader = cmd.ExecuteReader(@"SELECT ""ID"", ""Name"", ""State"" FROM ""RemoteVolume"" WHERE ""Type"" = ? AND ""State"" != ? AND ""ID"" NOT IN (SELECT ""VolumeID"" FROM ""Fileset"")", 0, RemoteVolumeType.Files.ToString(), RemoteVolumeState.Deleted.ToString())) + if (reader.Read()) + throw new DatabaseInconsistencyException($"Detected 1 volume with missing filesets: VolumeId = {reader.ConvertValueToInt64(0)}, Name = {reader.ConvertValueToString(1)}, State = {reader.ConvertValueToString(2)}"); + + throw new DatabaseInconsistencyException(string.Format("Detected {0} volumes with missing filesets", volumesMissingFilests)); + } + if (verifyfilelists) { var anyError = new List(); From 086d98ee4b196a185836a1de1b909dc8b2f49f23 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 10 Mar 2025 21:02:16 +0100 Subject: [PATCH 338/362] Detect non-attached files in filesets --- Duplicati/Library/Main/Database/LocalDatabase.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 4a9e09b08..72489ddcd 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -898,10 +898,15 @@ ON throw new DatabaseInconsistencyException(string.Format("Detected {0} volumes with missing filesets", volumesMissingFilests)); } + var nonAttachedFiles = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FileID"" NOT IN (SELECT ""ID"" FROM ""FileLookup"")"); + if (nonAttachedFiles != 0) + throw new DatabaseInconsistencyException($"Detected {nonAttachedFiles} file(s) in FilesetEntry without corresponding FileLookup entry"); + if (verifyfilelists) { var anyError = new List(); using (var cmd2 = m_connection.CreateCommand(transaction)) + { foreach (var filesetid in cmd.ExecuteReaderEnumerable(@"SELECT ""ID"" FROM ""Fileset"" ").Select(x => x.ConvertValueToInt64(0, -1))) { var expandedCmd = string.Format(@"SELECT COUNT(*) FROM (SELECT DISTINCT ""Path"" FROM ({0}) UNION SELECT DISTINCT ""Path"" FROM ({1}))", LocalDatabase.LIST_FILESETS, LocalDatabase.LIST_FOLDERS_AND_SYMLINKS); @@ -918,6 +923,7 @@ ON anyError.Add(string.Format("Unexpected difference in fileset {0}, found {1} entries, but expected {2}", filesetname, expandedlist, storedlist)); } } + } if (anyError.Any()) { throw new DatabaseInconsistencyException(string.Join("\n\r", anyError), "FilesetDifferences"); From ceae46f395b19d90347d44f11ce885ff9cd4a1da Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 10 Mar 2025 22:01:14 +0100 Subject: [PATCH 339/362] Fixed a long-standing bug that could cause errors on delete due to pending delete of filesets --- .../Main/Database/LocalDeleteDatabase.cs | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs index 2dfc6cb67..e3e265413 100644 --- a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -103,8 +103,8 @@ namespace Duplicati.Library.Main.Database cmd.ExecuteNonQuery(@"INSERT INTO ""DeletedBlock"" (""Hash"", ""Size"", ""VolumeID"") SELECT ""Hash"", ""Size"", ""VolumeID"" FROM ""Block"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""BlockID"" AS ""BlockID"" FROM ""BlocksetEntry"" UNION SELECT DISTINCT ""ID"" FROM ""Block"", ""BlocklistHash"" WHERE ""Block"".""Hash"" = ""BlocklistHash"".""Hash"") "); cmd.ExecuteNonQuery(@"DELETE FROM ""Block"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""BlockID"" FROM ""BlocksetEntry"" UNION SELECT DISTINCT ""ID"" FROM ""Block"", ""BlocklistHash"" WHERE ""Block"".""Hash"" = ""BlocklistHash"".""Hash"") "); - //Find all remote filesets that are no longer required, and mark them as delete - var updated = cmd.ExecuteNonQuery(@"UPDATE ""RemoteVolume"" SET ""State"" = ? WHERE ""Type"" = ? AND ""State"" IN (?, ?, ?) AND ""ID"" NOT IN (SELECT ""VolumeID"" FROM ""Fileset"") ", RemoteVolumeState.Deleting.ToString(), RemoteVolumeType.Files.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString(), RemoteVolumeState.Temporary.ToString()); + //Find all remote filesets that are no longer required, and mark them as deleting + var updated = cmd.ExecuteNonQuery(@"UPDATE ""RemoteVolume"" SET ""State"" = ? WHERE ""Type"" = ? AND ""State"" IN (?, ?, ?, ?) AND ""ID"" NOT IN (SELECT ""VolumeID"" FROM ""Fileset"") ", RemoteVolumeState.Deleting.ToString(), RemoteVolumeType.Files.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString(), RemoteVolumeState.Temporary.ToString(), RemoteVolumeState.Deleting.ToString()); if (deleted != updated) throw new Exception(string.Format("Unexpected number of remote volumes marked as deleted. Found {0} filesets, but {1} volumes", deleted, updated)); From 5d45baacbfe2832903a2abaa21c32fec43ff7bcc Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 10 Mar 2025 22:04:48 +0100 Subject: [PATCH 340/362] Renamed tests away from "Failed" to make searching for failed tests easier. --- Duplicati/UnitTest/DisruptionTests.cs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Duplicati/UnitTest/DisruptionTests.cs b/Duplicati/UnitTest/DisruptionTests.cs index 8c1b4702e..766655ea4 100644 --- a/Duplicati/UnitTest/DisruptionTests.cs +++ b/Duplicati/UnitTest/DisruptionTests.cs @@ -715,7 +715,7 @@ namespace Duplicati.UnitTest [Test] [Category("Disruption")] - public void TestFailedUploadWithNoRetries() + public void TestUploadExceptionWithNoRetries() { var testopts = TestOptions; testopts["number-of-retries"] = "0"; @@ -801,7 +801,7 @@ namespace Duplicati.UnitTest [TestCase(false, false)] [TestCase(true, true)] [TestCase(false, true)] - public void TestFailedUploadOfFirstDlistWithRepair(bool before, bool failOnLastDblock) + public void TestUploadExceptionOnFirstDlistWithRepair(bool before, bool failOnLastDblock) { var testopts = TestOptions; testopts["number-of-retries"] = "0"; @@ -895,7 +895,7 @@ namespace Duplicati.UnitTest [TestCase(false, false)] [TestCase(true, true)] [TestCase(false, true)] - public void TestFailedUploadOfFirstDlist(bool before, bool failOnLastDblock) + public void TestUploadExceptionOnFirstDlist(bool before, bool failOnLastDblock) { var testopts = TestOptions; testopts["number-of-retries"] = "0"; @@ -1030,7 +1030,7 @@ namespace Duplicati.UnitTest [TestCase(false, true, 3, false)] [TestCase(true, true, 3, true)] [TestCase(false, true, 3, true)] - public void TestMultiFailedDlist(bool before, bool modifyInBetween, int runs, bool withBase) + public void TestMultiExceptionOnDlist(bool before, bool modifyInBetween, int runs, bool withBase) { var testopts = TestOptions; testopts["number-of-retries"] = "0"; @@ -1144,27 +1144,27 @@ namespace Duplicati.UnitTest [Test] [Category("Disruption")] - public void TestFailedDblockUploadWithOperationCancellation() + public void TestDblockUploadWithOperationCancellation() { - TestFailedUploadWithOperationCancellation(".dblock."); + TestUploadWithOperationCancellation(".dblock."); } [Test] [Category("Disruption")] - public void TestFailedDindexUploadWithOperationCancellation() + public void TestDindexUploadWithOperationCancellation() { - TestFailedUploadWithOperationCancellation(".dindex."); + TestUploadWithOperationCancellation(".dindex."); } [Test] [Category("Disruption")] - public void TestFailedDlistUploadWithOperationCancellation() + public void TestDlistUploadWithOperationCancellation() { - TestFailedUploadWithOperationCancellation(".dlist."); + TestUploadWithOperationCancellation(".dlist."); } - private void TestFailedUploadWithOperationCancellation(string filefragment) + private void TestUploadWithOperationCancellation(string filefragment) { var testopts = TestOptions; testopts["number-of-retries"] = "1"; From 0274fc2b6e270c04268e81d998e3dbe9b0625e64 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 11 Mar 2025 08:08:28 +0100 Subject: [PATCH 341/362] Added sleeps to ensure timestamps have space for repairs --- Duplicati/UnitTest/IssueTests.cs | 7 +++++-- Duplicati/UnitTest/RepairHandlerTests.cs | 4 ++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/Duplicati/UnitTest/IssueTests.cs b/Duplicati/UnitTest/IssueTests.cs index a0302b149..33ffebdf6 100644 --- a/Duplicati/UnitTest/IssueTests.cs +++ b/Duplicati/UnitTest/IssueTests.cs @@ -113,11 +113,14 @@ namespace Duplicati.UnitTest if (compactBeforeRecreate) { + // Make room for a backup + Thread.Sleep(2000); + // Delete f2 and compact File.Delete(file2); - using (var c = new Library.Main.Controller(target, testopts, null)) + using (var c = new Controller(target, testopts, null)) { - IBackupResults backupResults = c.Backup(new[] { DATAFOLDER }); + var backupResults = c.Backup(new[] { DATAFOLDER }); TestUtils.AssertResults(backupResults); ICompactResults compactResults = c.Compact(); TestUtils.AssertResults(compactResults); diff --git a/Duplicati/UnitTest/RepairHandlerTests.cs b/Duplicati/UnitTest/RepairHandlerTests.cs index 2b3e3b4db..457826083 100644 --- a/Duplicati/UnitTest/RepairHandlerTests.cs +++ b/Duplicati/UnitTest/RepairHandlerTests.cs @@ -24,6 +24,7 @@ using System.Data; using System.IO; using System.IO.Compression; using System.Linq; +using System.Threading; using Duplicati.Library.Interface; using Duplicati.Library.Main; using Duplicati.Library.Main.Database; @@ -525,6 +526,9 @@ namespace Duplicati.UnitTest using (var c = new Controller("file://" + this.TARGETFOLDER, options, null)) TestUtils.AssertResults(c.Backup(new[] { this.DATAFOLDER })); + // Make room for a new backup + Thread.Sleep(2000); + ModifyFile(); using (var c = new Controller("file://" + this.TARGETFOLDER, options, null)) TestUtils.AssertResults(c.Backup(new[] { this.DATAFOLDER })); From a0c1676bca9d6700e39e93d230d310eb407b888d Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 11 Mar 2025 10:45:22 +0100 Subject: [PATCH 342/362] Added option to force repair with an older database. --- Duplicati/Library/Main/Operation/RepairHandler.cs | 7 ++++++- Duplicati/Library/Main/Options.cs | 10 ++++++++++ Duplicati/Library/Main/Strings.cs | 2 ++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 9609b0b43..929b318fa 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -153,7 +153,12 @@ namespace Duplicati.Library.Main.Operation if (mostRecentLocal < DateTime.UnixEpoch) throw new UserInformationException("The local database has no fileset times. Consider deleting the local database and run the repair operation again.", "LocalDatabaseHasNoFilesetTimes"); if (mostRecentRemote > mostRecentLocal) - throw new UserInformationException($"The remote files are newer ({mostRecentRemote}) than the local database ({mostRecentLocal}), this is likely because the database is outdated. Consider deleting the local database and run the repair operation again.", "RemoteFilesNewerThanLocalDatabase"); + { + if (m_options.RepairIgnoreOutdatedDatabase) + Logging.Log.WriteWarningMessage(LOGTAG, "RemoteFilesNewerThanLocalDatabase", null, "The remote files are newer ({0}) than the local database ({1}), this is likely because the database is outdated. Continuing as the options force ignoring this.", mostRecentRemote, mostRecentLocal); + else + throw new UserInformationException($"The remote files are newer ({mostRecentRemote}) than the local database ({mostRecentLocal}), this is likely because the database is outdated. Consider deleting the local database and run the repair operation again. If this is expected, set the option \"--repair-ignore-outdated-database\" ", "RemoteFilesNewerThanLocalDatabase"); + } if (m_options.Dryrun) { diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index f0590cd4f..ea7ba72e2 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -446,6 +446,7 @@ namespace Duplicati.Library.Main new CommandLineArgument("repair-only-paths", CommandLineArgument.ArgumentType.Boolean, Strings.Options.RepaironlypathsShort, Strings.Options.RepaironlypathsLong, "false"), new CommandLineArgument("repair-force-block-use", CommandLineArgument.ArgumentType.Boolean, Strings.Options.RepaironlypathsShort, Strings.Options.RepaironlypathsLong, "false"), + new CommandLineArgument("repair-ignore-outdated-database", CommandLineArgument.ArgumentType.Boolean, Strings.Options.RepairignoreoutdateddatabaseShort, Strings.Options.RepairignoreoutdateddatabaseLong, "false"), new CommandLineArgument("force-locale", CommandLineArgument.ArgumentType.String, Strings.Options.ForcelocaleShort, Strings.Options.ForcelocaleLong), new CommandLineArgument("force-actual-date", CommandLineArgument.ArgumentType.Boolean, Strings.Options.ForceActualDateShort, Strings.Options.ForceActualDateLong, "false"), @@ -1845,6 +1846,15 @@ namespace Duplicati.Library.Main get { return Library.Utility.Utility.ParseBoolOption(m_options, "repair-only-paths"); } } + /// + /// Gets a flag indicating if the repair process will ignore outdated database + /// + /// true if repair process will ignore outdated database; otherwise, false. + public bool RepairIgnoreOutdatedDatabase + { + get { return Library.Utility.Utility.ParseBoolOption(m_options, "repair-ignore-outdated-database"); } + } + /// /// Gets a flag indicating if the repair process will always use blocks /// diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs index 9ba7b35c5..e4ba2fa38 100644 --- a/Duplicati/Library/Main/Strings.cs +++ b/Duplicati/Library/Main/Strings.cs @@ -260,6 +260,8 @@ namespace Duplicati.Library.Main.Strings public static string LogretentionShort { get { return LC.L(@"Clean up old log data"); } } public static string RepaironlypathsLong { get { return LC.L(@"Use this option to build a searchable local database which only contains path information. This option is usable for quickly building a database to locate certain content without needing to reconstruct all information. The resulting database can be searched, but cannot be used to restore data with."); } } public static string RepaironlypathsShort { get { return LC.L(@"Repair database with paths"); } } + public static string RepairignoreoutdateddatabaseLong { get { return LC.L(@"Use this option to ignore that the remote destination contains newer contents than the database. This should only be used when the database is known to be correct, but the remote destination is not, as data can be lost otherwise."); } } + public static string RepairignoreoutdateddatabaseShort { get { return LC.L(@"Ignore outdated database files"); } } public static string ForcelocaleLong { get { return LC.L(@"By default, your system locale and culture settings will be used. In some cases you may prefer to run with another locale, for example to get messages in another language. Use this option to set the locale. Supply a blank string to choose the ""Invariant Culture""."); } } public static string ForcelocaleShort { get { return LC.L(@"Force the locale setting"); } } public static string ForceActualDateLong { get { return LC.L(@"By default, dates are displayed in the calendar format, meaning ""Today"" or ""Last Thursday"". By setting this option, only the actual dates are displayed, ""Nov 12, 2018, 8:01 AM"" for example."); } } From 351fb89509339f26f991c725975409c0942f2f85 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 11 Mar 2025 10:45:44 +0100 Subject: [PATCH 343/362] Ignoring timestamps in with dirty database test --- Duplicati/UnitTest/RepairHandlerTests.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Duplicati/UnitTest/RepairHandlerTests.cs b/Duplicati/UnitTest/RepairHandlerTests.cs index 457826083..44ebd656f 100644 --- a/Duplicati/UnitTest/RepairHandlerTests.cs +++ b/Duplicati/UnitTest/RepairHandlerTests.cs @@ -546,7 +546,9 @@ namespace Duplicati.UnitTest using (var c = new Controller("file://" + this.TARGETFOLDER, options, null)) Assert.Throws(() => c.Backup(new[] { this.DATAFOLDER })); - using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) + // Since we have removed the entry that is being checked here, we need to disable the check + var repairOptions = new Dictionary(options) { ["repair-ignore-outdated-database"] = "true" }; + using (Controller c = new Controller("file://" + this.TARGETFOLDER, repairOptions, null)) { IRepairResults repairResults = c.Repair(); TestUtils.AssertResults(repairResults); From b61d594180ae79039e99d1983278a4b03e7e20e8 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 11 Mar 2025 14:22:49 +0100 Subject: [PATCH 344/362] Added higher resolution internal profiling timers for the BlockManager --- .../Main/Operation/Restore/BlockManager.cs | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs index 9294b0e1d..1f99980ab 100644 --- a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs @@ -90,6 +90,19 @@ namespace Duplicati.Library.Main.Operation.Restore /// Internal stopwatch for profiling setting up the waiters. /// private readonly Stopwatch sw_get_wait; + /// + /// Internal stopwatch for profiling setting a block in the cache. + /// + private readonly Stopwatch sw_set_set; + /// + /// Internal stopwatch for profiling getting a waiter to notify that the block is available. + /// + private readonly Stopwatch sw_set_wake_get; + /// + /// Internal stopwatch for profiling for waking up the waiting request. + /// + private readonly Stopwatch sw_set_wake_set; + /// /// Dictionary for keeping track of how many times each block is requested. Used to determine when a block is no longer needed. /// @@ -155,6 +168,9 @@ namespace Duplicati.Library.Main.Operation.Restore sw_cacheevict = options.InternalProfiling ? new() : null; sw_checkcounts = options.InternalProfiling ? new() : null; sw_get_wait = options.InternalProfiling ? new() : null; + sw_set_set = options.InternalProfiling ? new() : null; + sw_set_wake_get = options.InternalProfiling ? new() : null; + sw_set_wake_set = options.InternalProfiling ? new() : null; foreach (var (block_id, volume_id) in db.GetBlocksAndVolumeIDs(options.SkipMetadata)) { @@ -304,13 +320,20 @@ namespace Duplicati.Library.Main.Operation.Restore { Logging.Log.WriteExplicitMessage(LOGTAG, "BlockCacheSet", "Setting block {0} in cache", blockRequest.BlockID); + sw_set_set?.Start(); m_block_cache.Set(blockRequest.BlockID, value); + sw_set_set?.Stop(); // Notify any waiters that the block is available. + sw_set_wake_get?.Start(); if (m_waiters.TryRemove(blockRequest.BlockID, out var tcs)) { + sw_set_wake_get?.Stop(); + sw_set_wake_set?.Start(); tcs.SetResult(value); + sw_set_wake_set?.Stop(); } + sw_set_wake_get?.Stop(); sw_cacheevict?.Start(); if (m_block_cache.Count > m_options.RestoreCacheMax) @@ -365,7 +388,7 @@ namespace Duplicati.Library.Main.Operation.Restore if (m_options.InternalProfiling) { - Logging.Log.WriteProfilingMessage(LOGTAG, "InternalTimings", $"Sleepable dictionary - CheckCounts: {sw_checkcounts.ElapsedMilliseconds}ms, Get wait: {sw_get_wait.ElapsedMilliseconds}ms, Cache evict: {sw_cacheevict.ElapsedMilliseconds}ms"); + Logging.Log.WriteProfilingMessage(LOGTAG, "InternalTimings", $"Sleepable dictionary - CheckCounts: {sw_checkcounts.ElapsedMilliseconds}ms, Get wait: {sw_get_wait.ElapsedMilliseconds}ms, Set set: {sw_set_set.ElapsedMilliseconds}ms, Set wake get: {sw_set_wake_get.ElapsedMilliseconds}ms, Set wake set: {sw_set_wake_set.ElapsedMilliseconds}ms, Cache evict: {sw_cacheevict.ElapsedMilliseconds}ms"); } if (m_block_cache.Count > 0) From ebb04ddd087dfe0afdcab65ff5c4842530d53c26 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 11 Mar 2025 14:23:37 +0100 Subject: [PATCH 345/362] Made waking up a waiting block request on the BlockManagers cache async to lower time spent by the volume consumer task --- Duplicati/Library/Main/Operation/Restore/BlockManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs index 1f99980ab..af8a34d0a 100644 --- a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs @@ -281,7 +281,7 @@ namespace Duplicati.Library.Main.Operation.Restore // If the block is not in the cache, request it from the volume. sw_get_wait?.Start(); - var tcs = new TaskCompletionSource(); + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var new_tcs = m_waiters.GetOrAdd(block_request.BlockID, tcs); if (tcs == new_tcs) { From b63869448a907fa3593c117e80a0acd4a2304640 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 11 Mar 2025 17:00:14 +0100 Subject: [PATCH 346/362] Ignore warning when reparing broken database --- Duplicati/UnitTest/RepairHandlerTests.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Duplicati/UnitTest/RepairHandlerTests.cs b/Duplicati/UnitTest/RepairHandlerTests.cs index 44ebd656f..308596c2c 100644 --- a/Duplicati/UnitTest/RepairHandlerTests.cs +++ b/Duplicati/UnitTest/RepairHandlerTests.cs @@ -551,7 +551,8 @@ namespace Duplicati.UnitTest using (Controller c = new Controller("file://" + this.TARGETFOLDER, repairOptions, null)) { IRepairResults repairResults = c.Repair(); - TestUtils.AssertResults(repairResults); + Assert.AreEqual(0, repairResults.Errors.Count()); + Assert.AreEqual(0, repairResults.Warnings.Where(x => x.IndexOf("RemoteFilesNewerThanLocalDatabase", StringComparison.OrdinalIgnoreCase) < 0).Count()); Assert.AreEqual(2, Directory.EnumerateFiles(this.TARGETFOLDER, "*.dlist.*").Count()); Assert.AreEqual(2, c.List().Filesets.Count()); } From b035c69e4d96b1301293b120202ef3aa1233adb2 Mon Sep 17 00:00:00 2001 From: "Marcelo C." Date: Tue, 11 Mar 2025 16:24:15 -0300 Subject: [PATCH 347/362] Integrating the IPv6 & IPv4 detection It should be noted that I was unable to completely remove IPv4 on Windows 11 to a state where it would not provide the 127.0.0.1 loopback, even when disabling Ipv4 in various ways, including with registry flags. On ubuntu/linux, after completely disabling ipv4 the ipv6 loopback was returned. --- Duplicati/CommandLine/ServerUtil/Settings.cs | 3 +- .../CommandLine/ServerUtil/SettingsBinder.cs | 2 +- .../GUI/Duplicati.GUI.TrayIcon/Program.cs | 4 +-- Duplicati/Library/Utility/Utility.cs | 34 +++++++++++++++---- 4 files changed, 33 insertions(+), 10 deletions(-) diff --git a/Duplicati/CommandLine/ServerUtil/Settings.cs b/Duplicati/CommandLine/ServerUtil/Settings.cs index 5743a4c27..1771f524c 100644 --- a/Duplicati/CommandLine/ServerUtil/Settings.cs +++ b/Duplicati/CommandLine/ServerUtil/Settings.cs @@ -25,6 +25,7 @@ using Duplicati.Library.Encryption; using Duplicati.Library.Interface; using Duplicati.Library.Main; using Duplicati.Library.Utility; +using Utility = Duplicati.Library.Main.Utility; namespace Duplicati.CommandLine.ServerUtil; @@ -88,7 +89,7 @@ public sealed record Settings( /// The loaded settings public static Settings Load(string? password, System.Uri? hostUrl, string settingsFile, bool insecure, string? settingsPassphrase, string? secretProvider, SecretProviderHelper.CachingLevel secretProviderCache, string secretProviderPattern, string? acceptedHostCertificate) { - hostUrl ??= new System.Uri("http://localhost:8200"); + hostUrl ??= new System.Uri($"http://{Library.Utility.Utility.IpVersionCompatibleLoopback}:8200"); ISecretProvider? secretInstance = null; if (!string.IsNullOrWhiteSpace(secretProvider)) diff --git a/Duplicati/CommandLine/ServerUtil/SettingsBinder.cs b/Duplicati/CommandLine/ServerUtil/SettingsBinder.cs index e27483d7e..b97b7378b 100644 --- a/Duplicati/CommandLine/ServerUtil/SettingsBinder.cs +++ b/Duplicati/CommandLine/ServerUtil/SettingsBinder.cs @@ -37,7 +37,7 @@ public class SettingsBinder : BinderBase /// /// The host URL option. /// - public static readonly Option hostUrlOption = new Option("--hosturl", description: "The host URL to use", getDefaultValue: () => new Uri("http://localhost:8200")); + public static readonly Option hostUrlOption = new Option("--hosturl", description: "The host URL to use", getDefaultValue: () => new Uri($"http://{Library.Utility.Utility.IpVersionCompatibleLoopback}:8200")); /// /// The server datafolder option. /// diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs index 2529f79f4..c1cc97bdd 100644 --- a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs +++ b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs @@ -66,12 +66,12 @@ namespace Duplicati.GUI.TrayIcon private const string DETACHED_PROCESS = "detached-process"; private const string BROWSER_COMMAND_OPTION = "browser-command"; - private const string DEFAULT_HOSTURL = "http://localhost:8200"; + private static string DEFAULT_HOSTURL => $"http://{Library.Utility.Utility.IpVersionCompatibleLoopback}:8200"; private static string _browser_command = null; private static bool disableTrayIconLogin = false; private static bool openui = false; - private static Uri serverURL = new Uri(DEFAULT_HOSTURL); + private static Uri serverURL = new(DEFAULT_HOSTURL); public static string BrowserCommand { get { return _browser_command; } } public static Server.Database.Connection databaseConnection = null; diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index 5843328db..071a65ec5 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -23,17 +23,20 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.IO; using System.Linq; -using System.Threading; -using System.Threading.Tasks; +using System.Net; +using System.Net.NetworkInformation; +using System.Net.Sockets; +using System.Runtime.Versioning; +using System.Security.Cryptography; +using System.Security.Cryptography.X509Certificates; using System.Text; using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; using Duplicati.Library.Common.IO; -using System.Globalization; -using System.Runtime.Versioning; -using System.Security.Cryptography.X509Certificates; -using System.Security.Cryptography; namespace Duplicati.Library.Utility { @@ -1543,6 +1546,25 @@ namespace Duplicati.Library.Utility collection.Import(pfxPath, password, X509KeyStorageFlags.Exportable | X509KeyStorageFlags.PersistKeySet); return collection; } + + /// + /// Probes the system for the presence of a loopback address on IPv4 + /// + public static bool HasIPv4Loopback => + NetworkInterface.GetAllNetworkInterfaces() + .Where(ni => ni.OperationalStatus == OperationalStatus.Up) + .SelectMany(ni => ni.GetIPProperties().UnicastAddresses) + .Any(addr => addr.Address.AddressFamily == AddressFamily.InterNetwork + && addr.Address.Equals(IPAddress.Loopback)); + + /// + /// On systems that have IPV4 and IPV6, the method will return the default loopback ( 127, 0, 0, 1) + /// On systems with IPV6 only, the method will return the IPV6 loopback (::1) + /// + /// + public static string IpVersionCompatibleLoopback => + HasIPv4Loopback ? IPAddress.Loopback.ToString() : $"[{IPAddress.IPv6Loopback.ToString()}]"; + /// /// Flattens an exception and its inner exceptions From 64d40ef0ea9f552d1fab026c8336342b013cbc81 Mon Sep 17 00:00:00 2001 From: "Marcelo C." Date: Wed, 12 Mar 2025 10:40:20 -0300 Subject: [PATCH 348/362] Fix to StopNow tests Since the core changes in the cancelation process where now cancellationTokens are observed rather than a thread.abort attempt, the expected exception is now OperationCanceledException when calling .Abort on the controller. Also eliminated the time based awaiting to abort in favor of monitoring the backup phase to ensure the .Abort happens after the controller has had time to initiate the backend and backend is at Backup_ProcessingFiles state. --- Duplicati/UnitTest/DisruptionTests.cs | 39 ++++++++++++++++++--------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/Duplicati/UnitTest/DisruptionTests.cs b/Duplicati/UnitTest/DisruptionTests.cs index 6e27637bf..b3b49d605 100644 --- a/Duplicati/UnitTest/DisruptionTests.cs +++ b/Duplicati/UnitTest/DisruptionTests.cs @@ -708,23 +708,38 @@ namespace Duplicati.UnitTest this.ModifySourceFiles(); using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { - // ReSharper disable once AccessToDisposedClosure - Task backupTask = Task.Run(() => c.Backup(new[] { this.DATAFOLDER })); - - // Block for a small amount of time to allow the ITaskControl to be associated - // with the Controller. Otherwise, the call to Stop will simply be a no-op. - Thread.Sleep(1000); - - c.Abort(); - try + var startedTcs = new TaskCompletionSource(); + c.OnOperationStarted += results => { - await backupTask.ConfigureAwait(false); + ((BackupResults)results).OperationProgressUpdater.PhaseChanged += (phase, previousPhase) => + { + if (phase == OperationPhase.Backup_ProcessingFiles) + startedTcs.SetResult(true); + }; + }; + + Task backupTask = Task.Run(() => c.Backup([DATAFOLDER])); + + Task completedTask = await Task.WhenAny(startedTcs.Task, Task.Delay(5000)); + if (completedTask == startedTcs.Task) + { + c.Abort(); + try + { + await backupTask.ConfigureAwait(false); + Assert.Fail("Expected OperationCanceledException but none was thrown."); + } + catch (OperationCanceledException) + { + // Success, exception thrown as expected + } } - catch (TaskCanceledException) + else { + Assert.Fail("Operation did not reach ready state within timeout."); } } - + // The next backup should proceed without issues. using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { From 5edfcc5c6ab0cac779c00c96f59b0f0c0a741831 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 13 Mar 2025 12:09:57 +0100 Subject: [PATCH 349/362] Prevent blocksize near remote volume size If the user selects the blocksize to be the same as the remote volume size, this will cause each block to be in a separate volume and will trigger repeated compact calls. This commit adds a check that ensures the volume size is large enough to hold at least two blocks. Also fixed an issue where the compact could create volumes larger than the set limit (at most one block larger). --- Duplicati/Library/Main/Controller.cs | 3 +++ Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs | 2 +- Duplicati/Library/Main/Operation/CompactHandler.cs | 2 +- Duplicati/UnitTest/BorderTests.cs | 1 + 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Main/Controller.cs b/Duplicati/Library/Main/Controller.cs index 5104e2282..3486d7226 100644 --- a/Duplicati/Library/Main/Controller.cs +++ b/Duplicati/Library/Main/Controller.cs @@ -749,6 +749,9 @@ namespace Duplicati.Library.Main if (!string.IsNullOrWhiteSpace(m_options.Prefix) && m_options.Prefix.Contains("-")) throw new Interface.UserInformationException("The prefix cannot contain hyphens (-)", "PrefixCannotContainHyphens"); + if (m_options.VolumeSize <= m_options.Blocksize * 2) + throw new Interface.UserInformationException("The volume size must be at least twice the block size", "VolumeSizeTooSmall"); + //Check validity of retention-policy option value try { diff --git a/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs b/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs index 7b43c4d36..670f3a809 100644 --- a/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs +++ b/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs @@ -119,7 +119,7 @@ namespace Duplicati.Library.Main.Operation.Backup indexvolume.AddBlock(b.HashKey, b.Size); // If the volume is full, send to upload - if (blockvolume.Filesize > options.VolumeSize - options.Blocksize) + if (blockvolume.Filesize > (options.VolumeSize - options.Blocksize)) { //When uploading a new volume, we register the volumes and then flush the transaction // this ensures that the local database and remote storage are as closely related as possible diff --git a/Duplicati/Library/Main/Operation/CompactHandler.cs b/Duplicati/Library/Main/Operation/CompactHandler.cs index 5144211dc..da38fa8a1 100644 --- a/Duplicati/Library/Main/Operation/CompactHandler.cs +++ b/Duplicati/Library/Main/Operation/CompactHandler.cs @@ -195,7 +195,7 @@ namespace Duplicati.Library.Main.Operation db.RegisterDuplicatedBlock(e.Key, e.Value, newvol.VolumeID, transaction); blocksInVolume++; - if (newvol.Filesize > m_options.VolumeSize) + if (newvol.Filesize > (m_options.VolumeSize - m_options.Blocksize)) { FinishVolumeAndUpload(db, backendManager, newvol, newvolindex, uploadedVolumes); diff --git a/Duplicati/UnitTest/BorderTests.cs b/Duplicati/UnitTest/BorderTests.cs index 9bdd5b098..e03f8b10f 100644 --- a/Duplicati/UnitTest/BorderTests.cs +++ b/Duplicati/UnitTest/BorderTests.cs @@ -54,6 +54,7 @@ namespace Duplicati.UnitTest RunCommands(1024 * 10, modifyOptions: opts => { opts["blocksize"] = "10mb"; + opts["dblock-size"] = "25mb"; }); } From 30ec9754691ad00378e6b8fd54ebe9793f4ef884 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 13 Mar 2025 13:34:55 +0100 Subject: [PATCH 350/362] Relaxed size requirement to better work with tests --- Duplicati/Library/Main/Controller.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Controller.cs b/Duplicati/Library/Main/Controller.cs index 3486d7226..5fc7d8554 100644 --- a/Duplicati/Library/Main/Controller.cs +++ b/Duplicati/Library/Main/Controller.cs @@ -749,7 +749,7 @@ namespace Duplicati.Library.Main if (!string.IsNullOrWhiteSpace(m_options.Prefix) && m_options.Prefix.Contains("-")) throw new Interface.UserInformationException("The prefix cannot contain hyphens (-)", "PrefixCannotContainHyphens"); - if (m_options.VolumeSize <= m_options.Blocksize * 2) + if (m_options.VolumeSize < m_options.Blocksize * 2) throw new Interface.UserInformationException("The volume size must be at least twice the block size", "VolumeSizeTooSmall"); //Check validity of retention-policy option value From f342d70819d92f4cf5275da9325e3b08c0ad5e62 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 13 Mar 2025 23:06:32 +0100 Subject: [PATCH 351/362] Implemented forced invariant locale on all SQL strings. This fixes #6023 --- .../Main/Database/LocalBackupDatabase.cs | 42 ++--- .../Main/Database/LocalBugReportDatabase.cs | 54 +++--- .../Library/Main/Database/LocalDatabase.cs | 131 +++++++------ .../Main/Database/LocalDeleteDatabase.cs | 22 +-- .../Database/LocalListAffectedDatabase.cs | 46 ++--- .../Database/LocalListBrokenFilesDatabase.cs | 62 +++--- .../Main/Database/LocalListChangesDatabase.cs | 84 ++++----- .../Main/Database/LocalListDatabase.cs | 93 +++++---- .../Main/Database/LocalPurgeDatabase.cs | 64 +++---- .../Main/Database/LocalRecreateDatabase.cs | 72 +++---- .../Main/Database/LocalRepairDatabase.cs | 74 ++++---- .../Main/Database/LocalRestoreDatabase.cs | 176 +++++++++--------- .../Main/Database/LocalTestDatabase.cs | 62 +++--- Duplicati/Library/Utility/Utility.cs | 17 ++ 14 files changed, 516 insertions(+), 483 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index 510bab3dd..c5db46e6e 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -185,7 +185,7 @@ namespace Duplicati.Library.Main.Database @" WHERE ""A"".""ID"" = ""B"".""FileID"" " + @" AND ""B"".""FilesetID"" = ? "; - m_findfileCommand.CommandText = string.Format( + m_findfileCommand.CommandText = FormatInvariant( @"SELECT ""C"".""ID"" AS ""FileID"", ""C"".""LastModified"", ""D"".""Length"", ""E"".""FullHash"" as ""Metahash"", ""E"".""Length"" AS ""Metasize"" " + @" FROM " + @" ({0}) AS ""C"", ""Blockset"" AS ""D"", ""Blockset"" AS ""E"", ""Metadataset"" ""F"" " + @@ -343,12 +343,12 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist { var bid = cmd.ExecuteScalarInt64(@"SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = ?", -1, h); if (bid == -1) - throw new Exception(string.Format("Could not find any blocks with the given hash: {0}", h)); + throw new Exception($"Could not find any blocks with the given hash: {h}"); foreach (var rd in cmd.ExecuteReaderEnumerable(@"SELECT ""Size"" FROM ""Block"" WHERE ""Hash"" = ?", h)) Logging.Log.WriteErrorMessage(LOGTAG, "FoundIssue1400Error", null, "Found block with ID {0} and hash {1} and size {2}", bid, h, rd.ConvertValueToInt64(0, -1)); } - throw new Exception(string.Format("Unexpected result count: {0}, expected {1}, check log for more messages", c, 1)); + throw new Exception($"Unexpected result count: {c}, expected {1}, check log for more messages"); } ix++; @@ -616,22 +616,22 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist { var subqueryFiles = @"SELECT ""File"".""Path"" AS ""Path"", ""A"".""Fullhash"" AS ""Filehash"", ""B"".""Fullhash"" AS ""Metahash"" FROM ""File"", ""FilesetEntry"", ""Blockset"" A, ""Blockset"" B, ""Metadataset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""A"".""ID"" = ""File"".""BlocksetID"" AND ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""MetadataID"" = ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""B"".""ID"" "; - cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS " + subqueryFiles, tmpName1), lastFilesetId); - cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS " + subqueryFiles, tmpName2), m_filesetId); - cmd.ExecuteNonQuery(string.Format(@"CREATE INDEX ""nn_tmpName1"" ON ""{0}"" (""Path"")", tmpName1)); - cmd.ExecuteNonQuery(string.Format(@"CREATE INDEX ""nn_tmpName2"" ON ""{0}"" (""Path"")", tmpName2)); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" AS " + subqueryFiles, tmpName1), lastFilesetId); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" AS " + subqueryFiles, tmpName2), m_filesetId); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE INDEX ""nn_tmpName1"" ON ""{0}"" (""Path"")", tmpName1)); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE INDEX ""nn_tmpName2"" ON ""{0}"" (""Path"")", tmpName2)); - results.AddedFiles = cmd.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""BlocksetID"" != ? AND ""File"".""BlocksetID"" != ? AND NOT ""File"".""Path"" IN (SELECT ""Path"" FROM ""{0}"")", tmpName1), 0, m_filesetId, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID); - results.DeletedFiles = cmd.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM ""{0}"" WHERE ""{0}"".""Path"" NOT IN (SELECT ""Path"" FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ?)", tmpName1), 0, m_filesetId); - results.ModifiedFiles = cmd.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM ""{0}"" A, ""{1}"" B WHERE ""A"".""Path"" = ""B"".""Path"" AND (""A"".""Filehash"" != ""B"".""Filehash"" OR ""A"".""Metahash"" != ""B"".""Metahash"")", tmpName1, tmpName2), 0); + results.AddedFiles = cmd.ExecuteScalarInt64(FormatInvariant(@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""BlocksetID"" != ? AND ""File"".""BlocksetID"" != ? AND NOT ""File"".""Path"" IN (SELECT ""Path"" FROM ""{0}"")", tmpName1), 0, m_filesetId, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID); + results.DeletedFiles = cmd.ExecuteScalarInt64(FormatInvariant(@"SELECT COUNT(*) FROM ""{0}"" WHERE ""{0}"".""Path"" NOT IN (SELECT ""Path"" FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ?)", tmpName1), 0, m_filesetId); + results.ModifiedFiles = cmd.ExecuteScalarInt64(FormatInvariant(@"SELECT COUNT(*) FROM ""{0}"" A, ""{1}"" B WHERE ""A"".""Path"" = ""B"".""Path"" AND (""A"".""Filehash"" != ""B"".""Filehash"" OR ""A"".""Metahash"" != ""B"".""Metahash"")", tmpName1, tmpName2), 0); } finally { - try { cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"";", tmpName1)); } + try { cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"";", tmpName1)); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "DisposeError", ex, "Dispose temp table error"); } - try { cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"";", tmpName2)); } + try { cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"";", tmpName2)); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "DisposeError", ex, "Dispose temp table error"); } } } @@ -726,21 +726,21 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist // copy entries from previous file set into a temporary table, except those file IDs already added by the current backup var tempFileSetTable = "FilesetEntry-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); cmd.Transaction = tr.Parent; - cmd.ExecuteNonQuery($@"CREATE TEMPORARY TABLE ""{tempFileSetTable}"" AS SELECT ""FileID"", ""Lastmodified"" FROM (SELECT DISTINCT ""FilesetID"", ""FileID"", ""Lastmodified"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = ? AND ""FileID"" NOT IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = ?))", lastFilesetId, fileSetId); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tempFileSetTable}"" AS SELECT ""FileID"", ""Lastmodified"" FROM (SELECT DISTINCT ""FilesetID"", ""FileID"", ""Lastmodified"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = ? AND ""FileID"" NOT IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = ?))"), lastFilesetId, fileSetId); // now we need to remove, from the above, any entries that were enumerated by the // UNC-driven backup cmdDelete.Transaction = tr.Parent; - cmdDelete.CommandText = $@"DELETE FROM ""{tempFileSetTable}"" WHERE ""FileID"" = ?"; + cmdDelete.CommandText = FormatInvariant($@"DELETE FROM ""{tempFileSetTable}"" WHERE ""FileID"" = ?"); cmdDelete.AddParameters(1); // enumerate files from new temporary file set, and remove any entries handled by UNC cmd.Transaction = tr.Parent; - foreach (var row in cmd.ExecuteReaderEnumerable( + foreach (var row in cmd.ExecuteReaderEnumerable(FormatInvariant( $@"SELECT f.""Path"", fs.""FileID"", fs.""Lastmodified"", COALESCE(bs.""Length"", -1) FROM (SELECT DISTINCT ""FileID"", ""Lastmodified"" FROM ""{tempFileSetTable}"") AS fs LEFT JOIN ""File"" AS f ON fs.""FileID"" = f.""ID"" - LEFT JOIN ""Blockset"" AS bs ON f.""BlocksetID"" = bs.""ID"";")) + LEFT JOIN ""Blockset"" AS bs ON f.""BlocksetID"" = bs.""ID"";"))) { var path = row.GetString(0); var size = row.GetInt64(3); @@ -753,8 +753,8 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist } // now copy the temporary table into the FileSetEntry table - cmd.ExecuteNonQuery($@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") - SELECT ?, ""FileID"", ""Lastmodified"" FROM ""{tempFileSetTable}""", fileSetId); + cmd.ExecuteNonQuery(FormatInvariant($@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") + SELECT ?, ""FileID"", ""Lastmodified"" FROM ""{tempFileSetTable}"""), fileSetId); tr.Commit(); } @@ -803,7 +803,7 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist cmd.Transaction = transaction; var c = cmd.ExecuteNonQuery(@"UPDATE ""Block"" SET ""VolumeID"" = ? WHERE ""Hash"" = ? AND ""Size"" = ? AND ""VolumeID"" = ? ", targetvolumeid, blockkey, size, sourcevolumeid); if (c != 1) - throw new Exception(string.Format("Failed to move block {0}:{1} from volume {2}, count: {3}", blockkey, size, sourcevolumeid, c)); + throw new Exception($"Failed to move block {blockkey}:{size} from volume {sourcevolumeid}, count: {c}"); } } @@ -815,7 +815,7 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist { var c = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""Block"" WHERE ""VolumeID"" = ? ", -1, volumeid); if (c != 0) - throw new Exception(string.Format("Failed to safe-delete volume {0}, blocks: {1}", name, c)); + throw new Exception($"Failed to safe-delete volume {name}, blocks: {c}"); RemoveRemoteVolume(name, transaction); } @@ -838,7 +838,7 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist { using (var cmd = m_connection.CreateCommand()) { - cmd.CommandText = @$"SELECT ""Path"" FROM ""File"" ORDER BY LENGTH(""Path"") DESC LIMIT 1"; + cmd.CommandText = @"SELECT ""Path"" FROM ""File"" ORDER BY LENGTH(""Path"") DESC LIMIT 1"; var v0 = cmd.ExecuteScalar(); if (v0 == null || v0 == DBNull.Value) return null; diff --git a/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs b/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs index 604479906..f155b0784 100644 --- a/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -52,10 +52,10 @@ namespace Duplicati.Library.Main.Database { upcmd.Transaction = tr; - upcmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""ID"" INTEGER PRIMARY KEY, ""RealPath"" TEXT NOT NULL, ""Obfuscated"" TEXT NULL)", tablename)); - upcmd.ExecuteNonQuery(string.Format(@"INSERT INTO ""{0}"" (""RealPath"") SELECT DISTINCT ""Path"" FROM ""File"" ORDER BY ""Path"" ", tablename)); - upcmd.ExecuteNonQuery(string.Format(@"UPDATE ""{0}"" SET ""Obfuscated"" = ? || length(""RealPath"") || ? || ""ID"" || (CASE WHEN substr(""RealPath"", length(""RealPath"")) = ? THEN ? ELSE ? END) ", tablename), !OperatingSystem.IsWindows() ? "/" : "X:\\", Util.DirectorySeparatorString, Util.DirectorySeparatorString, Util.DirectorySeparatorString, ".bin"); - + upcmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (""ID"" INTEGER PRIMARY KEY, ""RealPath"" TEXT NOT NULL, ""Obfuscated"" TEXT NULL)", tablename)); + upcmd.ExecuteNonQuery(FormatInvariant(@"INSERT INTO ""{0}"" (""RealPath"") SELECT DISTINCT ""Path"" FROM ""File"" ORDER BY ""Path"" ", tablename)); + upcmd.ExecuteNonQuery(FormatInvariant(@"UPDATE ""{0}"" SET ""Obfuscated"" = ? || length(""RealPath"") || ? || ""ID"" || (CASE WHEN substr(""RealPath"", length(""RealPath"")) = ? THEN ? ELSE ? END) ", tablename), !OperatingSystem.IsWindows() ? "/" : "X:\\", Util.DirectorySeparatorString, Util.DirectorySeparatorString, Util.DirectorySeparatorString, ".bin"); + /*long id = 1; using(var rd = cmd.ExecuteReader(string.Format(@"SELECT ""RealPath"", ""Obfuscated"" FROM ""{0}"" ", tablename))) while(rd.Read()) @@ -66,17 +66,17 @@ namespace Duplicati.Library.Main.Database */ } - cmd.ExecuteNonQuery(@"UPDATE ""LogData"" SET ""Message"" = 'ERASED!' WHERE ""Message"" LIKE '%/%' OR ""Message"" LIKE '%:\%' "); - cmd.ExecuteNonQuery(@"UPDATE ""LogData"" SET ""Exception"" = 'ERASED!' WHERE ""Exception"" LIKE '%/%' OR ""Exception"" LIKE '%:\%' "); + cmd.ExecuteNonQuery(@"UPDATE ""LogData"" SET ""Message"" = 'ERASED!' WHERE ""Message"" LIKE '%/%' OR ""Message"" LIKE '%:\%' "); + cmd.ExecuteNonQuery(@"UPDATE ""LogData"" SET ""Exception"" = 'ERASED!' WHERE ""Exception"" LIKE '%/%' OR ""Exception"" LIKE '%:\%' "); cmd.ExecuteNonQuery(@"UPDATE ""Configuration"" SET ""Value"" = 'ERASED!' WHERE ""Key"" = 'passphrase' "); - cmd.ExecuteNonQuery(string.Format(@"CREATE TABLE ""FixedFile"" AS SELECT ""B"".""ID"" AS ""ID"", ""A"".""Obfuscated"" AS ""Path"", ""B"".""BlocksetID"" AS ""BlocksetID"", ""B"".""MetadataID"" AS ""MetadataID"" FROM ""{0}"" ""A"", ""File"" ""B"" WHERE ""A"".""RealPath"" = ""B"".""Path"" ", tablename)); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TABLE ""FixedFile"" AS SELECT ""B"".""ID"" AS ""ID"", ""A"".""Obfuscated"" AS ""Path"", ""B"".""BlocksetID"" AS ""BlocksetID"", ""B"".""MetadataID"" AS ""MetadataID"" FROM ""{0}"" ""A"", ""File"" ""B"" WHERE ""A"".""RealPath"" = ""B"".""Path"" ", tablename)); cmd.ExecuteNonQuery(@"DROP VIEW ""File"" "); cmd.ExecuteNonQuery(@"DROP TABLE ""FileLookup"" "); cmd.ExecuteNonQuery(@"DROP TABLE ""PathPrefix"" "); - cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", tablename)); + cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", tablename)); using(new Logging.Timer(LOGTAG, "CommitUpdateBugReport", "CommitUpdateBugReport")) tr.Commit(); diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 72489ddcd..332eb2b44 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -97,6 +97,23 @@ namespace Duplicati.Library.Main.Database return c; } + /// + /// Formats a string using the invariant culture + /// + /// The format string + /// The arguments to format + /// The formatted string + public static string FormatInvariant(string format, params object[] args) + => Library.Utility.Utility.FormatInvariant(format, args); + + /// + /// Formats the string using the invariant culture + /// + /// The formattable string + /// The formatted string + public static string FormatInvariant(FormattableString formattable) + => Library.Utility.Utility.FormatInvariant(formattable); + public static bool Exists(string path) { return File.Exists(path); @@ -187,14 +204,14 @@ namespace Duplicati.Library.Main.Database m_selectremotevolumesCommand.CommandText = @"SELECT ""ID"", ""Name"", ""Type"", ""Size"", ""Hash"", ""State"", ""DeleteGraceTime"" FROM ""Remotevolume"""; m_selectremotevolumeCommand.CommandText = m_selectremotevolumesCommand.CommandText + @" WHERE ""Name"" = ?"; - m_selectduplicateRemoteVolumesCommand.CommandText = string.Format(@"SELECT DISTINCT ""Name"", ""State"" FROM ""Remotevolume"" WHERE ""Name"" IN (SELECT ""Name"" FROM ""Remotevolume"" WHERE ""State"" IN ('{0}', '{1}')) AND NOT ""State"" IN ('{0}', '{1}')", RemoteVolumeState.Deleted.ToString(), RemoteVolumeState.Deleting.ToString()); + m_selectduplicateRemoteVolumesCommand.CommandText = FormatInvariant(@"SELECT DISTINCT ""Name"", ""State"" FROM ""Remotevolume"" WHERE ""Name"" IN (SELECT ""Name"" FROM ""Remotevolume"" WHERE ""State"" IN ('{0}', '{1}')) AND NOT ""State"" IN ('{0}', '{1}')", RemoteVolumeState.Deleted.ToString(), RemoteVolumeState.Deleting.ToString()); m_selectremotevolumeCommand.AddParameter(); m_removeremotevolumeCommand.CommandText = @"DELETE FROM ""Remotevolume"" WHERE ""Name"" = ? AND (""DeleteGraceTime"" < ? OR ""State"" != ?)"; m_removeremotevolumeCommand.AddParameters(3); - m_removedeletedremotevolumeCommand.CommandText = $@"DELETE FROM ""Remotevolume"" WHERE ""State"" == '{RemoteVolumeState.Deleted.ToString()}' AND (""DeleteGraceTime"" < ? OR LENGTH(""DeleteGraceTime"") > 12) "; // >12 is to handle removal of old records that were in ticks + m_removedeletedremotevolumeCommand.CommandText = FormatInvariant($@"DELETE FROM ""Remotevolume"" WHERE ""State"" == '{RemoteVolumeState.Deleted.ToString()}' AND (""DeleteGraceTime"" < ? OR LENGTH(""DeleteGraceTime"") > 12) "); // >12 is to handle removal of old records that were in ticks m_removedeletedremotevolumeCommand.AddParameter(); m_selectremotevolumeIdCommand.CommandText = @"SELECT ""ID"" FROM ""Remotevolume"" WHERE ""Name"" = ?"; @@ -446,7 +463,7 @@ namespace Duplicati.Library.Main.Database cmd.Transaction = tr.Parent; var c = cmd.ExecuteNonQuery(@"DELETE FROM ""RemoteVolume"" WHERE ""Name"" = ? AND ""State"" = ? ", name, state.ToString()); if (c != 1) - throw new Exception(string.Format("Unexpected number of remote volumes deleted: {0}, expected {1}", c, 1)); + throw new Exception($"Unexpected number of remote volumes deleted: {c}, expected {1}"); tr.Commit(); } @@ -471,8 +488,8 @@ namespace Duplicati.Library.Main.Database var blocksetidstable = "DelBlockSetIds-" + temptransguid; // Create and fill a temp table with the volids to delete. We avoid using too many parameters that way. - deletecmd.ExecuteNonQuery(string.Format(@"CREATE TEMP TABLE ""{0}"" (""ID"" INTEGER PRIMARY KEY)", volidstable)); - deletecmd.CommandText = string.Format(@"INSERT OR IGNORE INTO ""{0}"" (""ID"") VALUES (?)", volidstable); + deletecmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMP TABLE ""{0}"" (""ID"" INTEGER PRIMARY KEY)", volidstable)); + deletecmd.CommandText = FormatInvariant(@"INSERT OR IGNORE INTO ""{0}"" (""ID"") VALUES (?)", volidstable); deletecmd.Parameters.Clear(); deletecmd.AddParameters(1); foreach (var name in names) @@ -481,11 +498,11 @@ namespace Duplicati.Library.Main.Database deletecmd.SetParameterValue(0, volumeid); deletecmd.ExecuteNonQuery(); } - var volIdsSubQuery = string.Format(@"SELECT ""ID"" FROM ""{0}"" ", volidstable); + var volIdsSubQuery = FormatInvariant(@"SELECT ""ID"" FROM ""{0}"" ", volidstable); deletecmd.Parameters.Clear(); - var bsIdsSubQuery = string.Format( + var bsIdsSubQuery = FormatInvariant( @"SELECT DISTINCT ""BlocksetEntry"".""BlocksetID"" FROM ""BlocksetEntry"", ""Block""" + @" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" IN ({0}) " + @"UNION ALL " @@ -494,69 +511,69 @@ namespace Duplicati.Library.Main.Database , volIdsSubQuery); // Create a temporary table to cache subquery result, as it might take long (SQLite does not cache at all). - deletecmd.ExecuteNonQuery(string.Format(@"CREATE TEMP TABLE ""{0}"" (""ID"" INTEGER PRIMARY KEY)", blocksetidstable)); - deletecmd.ExecuteNonQuery(string.Format(@"INSERT OR IGNORE INTO ""{0}"" (""ID"") {1}", blocksetidstable, bsIdsSubQuery)); - bsIdsSubQuery = string.Format(@"SELECT DISTINCT ""ID"" FROM ""{0}"" ", blocksetidstable); + deletecmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMP TABLE ""{0}"" (""ID"" INTEGER PRIMARY KEY)", blocksetidstable)); + deletecmd.ExecuteNonQuery(FormatInvariant(@"INSERT OR IGNORE INTO ""{0}"" (""ID"") {1}", blocksetidstable, bsIdsSubQuery)); + bsIdsSubQuery = FormatInvariant(@"SELECT DISTINCT ""ID"" FROM ""{0}"" ", blocksetidstable); deletecmd.Parameters.Clear(); // Create a temp table to associate metadata that is being deleted to a fileset - var metadataFilesetQuery = $@"SELECT Metadataset.ID, FilesetEntry.FilesetID + var metadataFilesetQuery = FormatInvariant($@"SELECT Metadataset.ID, FilesetEntry.FilesetID FROM Metadataset INNER JOIN FileLookup ON FileLookup.MetadataID = Metadataset.ID INNER JOIN FilesetEntry ON FilesetEntry.FileID = FileLookup.ID WHERE Metadataset.BlocksetID IN ({bsIdsSubQuery}) -OR Metadataset.ID IN (SELECT MetadataID FROM FileLookup WHERE BlocksetID IN ({bsIdsSubQuery}))"; +OR Metadataset.ID IN (SELECT MetadataID FROM FileLookup WHERE BlocksetID IN ({bsIdsSubQuery}))"); var metadataFilesetTable = @"DelMetadataFilesetIds-" + temptransguid; - deletecmd.ExecuteNonQuery($@"CREATE TEMP TABLE ""{metadataFilesetTable}"" (MetadataID INTEGER PRIMARY KEY, FilesetID INTEGER)"); - deletecmd.ExecuteNonQuery($@"INSERT OR IGNORE INTO ""{metadataFilesetTable}"" (MetadataID, FilesetID) {metadataFilesetQuery}"); + deletecmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMP TABLE ""{metadataFilesetTable}"" (MetadataID INTEGER PRIMARY KEY, FilesetID INTEGER)")); + deletecmd.ExecuteNonQuery(FormatInvariant($@"INSERT OR IGNORE INTO ""{metadataFilesetTable}"" (MetadataID, FilesetID) {metadataFilesetQuery}")); // Delete FilesetEntry rows that had their metadata deleted - deletecmd.ExecuteNonQuery($@"DELETE FROM FilesetEntry + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM FilesetEntry WHERE FilesetEntry.FilesetID IN (SELECT DISTINCT FilesetID FROM ""{metadataFilesetTable}"") AND FilesetEntry.FileID IN ( SELECT FilesetEntry.FileID FROM FilesetEntry INNER JOIN FileLookup ON FileLookup.ID = FilesetEntry.FileID - WHERE FileLookup.MetadataID IN (SELECT MetadataID FROM ""{metadataFilesetTable}""))"); + WHERE FileLookup.MetadataID IN (SELECT MetadataID FROM ""{metadataFilesetTable}""))")); // Delete FilesetEntry rows that had their blocks deleted - deletecmd.ExecuteNonQuery($@"DELETE FROM FilesetEntry WHERE FilesetEntry.FileID IN ( + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM FilesetEntry WHERE FilesetEntry.FileID IN ( SELECT ID FROM FileLookup -WHERE FileLookup.BlocksetID IN ({bsIdsSubQuery}))"); - deletecmd.ExecuteNonQuery($@"DELETE FROM FileLookup WHERE FileLookup.MetadataID IN (SELECT MetadataID FROM ""{metadataFilesetTable}"")"); +WHERE FileLookup.BlocksetID IN ({bsIdsSubQuery}))")); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM FileLookup WHERE FileLookup.MetadataID IN (SELECT MetadataID FROM ""{metadataFilesetTable}"")")); - deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""Metadataset"" WHERE ""BlocksetID"" IN ({0})", bsIdsSubQuery)); - deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""FileLookup"" WHERE ""BlocksetID"" IN ({0})", bsIdsSubQuery)); - deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""Blockset"" WHERE ""ID"" IN ({0})", bsIdsSubQuery)); - deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ({0})", bsIdsSubQuery)); - deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""BlocklistHash"" WHERE ""BlocklistHash"".""BlocksetID"" IN ({0})", bsIdsSubQuery)); + deletecmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""Metadataset"" WHERE ""BlocksetID"" IN ({0})", bsIdsSubQuery)); + deletecmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""FileLookup"" WHERE ""BlocksetID"" IN ({0})", bsIdsSubQuery)); + deletecmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""Blockset"" WHERE ""ID"" IN ({0})", bsIdsSubQuery)); + deletecmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ({0})", bsIdsSubQuery)); + deletecmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""BlocklistHash"" WHERE ""BlocklistHash"".""BlocksetID"" IN ({0})", bsIdsSubQuery)); // If the volume is a block or index volume, this will update the crosslink table, otherwise nothing will happen - deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN ({0}) OR ""IndexVolumeID"" IN ({0})", volIdsSubQuery)); - deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""Block"" WHERE ""VolumeID"" IN ({0})", volIdsSubQuery)); - deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""DeletedBlock"" WHERE ""VolumeID"" IN ({0})", volIdsSubQuery)); - deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""ChangeJournalData"" WHERE ""FilesetID"" IN (SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ({0}))", volIdsSubQuery)); - deletecmd.ExecuteNonQuery($@"DELETE FROM FilesetEntry WHERE FilesetID IN (SELECT ID FROM Fileset WHERE VolumeID IN ({volIdsSubQuery}))"); - deletecmd.ExecuteNonQuery($@"DELETE FROM Fileset WHERE VolumeID IN ({volIdsSubQuery})"); + deletecmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN ({0}) OR ""IndexVolumeID"" IN ({0})", volIdsSubQuery)); + deletecmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""Block"" WHERE ""VolumeID"" IN ({0})", volIdsSubQuery)); + deletecmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""DeletedBlock"" WHERE ""VolumeID"" IN ({0})", volIdsSubQuery)); + deletecmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""ChangeJournalData"" WHERE ""FilesetID"" IN (SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ({0}))", volIdsSubQuery)); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM FilesetEntry WHERE FilesetID IN (SELECT ID FROM Fileset WHERE VolumeID IN ({volIdsSubQuery}))")); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM Fileset WHERE VolumeID IN ({volIdsSubQuery})")); // Delete from Fileset if FilesetEntry rows were deleted by related metadata and there are no references in FilesetEntry anymore - deletecmd.ExecuteNonQuery($@"DELETE FROM Fileset WHERE Fileset.ID IN + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM Fileset WHERE Fileset.ID IN (SELECT DISTINCT FilesetID FROM ""{metadataFilesetTable}"") AND Fileset.ID NOT IN - (SELECT DISTINCT FilesetID FROM FilesetEntry)"); + (SELECT DISTINCT FilesetID FROM FilesetEntry)")); // Clean up temp tables for subqueries. We truncate content and then try to delete. // Drop in try-block, as it fails in nested transactions (SQLite problem) // System.Data.SQLite.SQLiteException (0x80004005): database table is locked - deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""{0}"" ", blocksetidstable)); - deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""{0}"" ", volidstable)); + deletecmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""{0}"" ", blocksetidstable)); + deletecmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""{0}"" ", volidstable)); try { deletecmd.CommandTimeout = 2; - deletecmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", blocksetidstable)); - deletecmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", volidstable)); - deletecmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", metadataFilesetTable)); + deletecmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", blocksetidstable)); + deletecmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", volidstable)); + deletecmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", metadataFilesetTable)); } catch { /* Ignore, will be deleted on close anyway. */ } @@ -673,7 +690,7 @@ AND Fileset.ID NOT IN { using (var cmd = m_connection.CreateCommand()) { - using (var rd = cmd.ExecuteReader($@"SELECT ""IsFullBackup"" FROM ""Fileset"" WHERE ""Timestamp"" = {Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(filesetTime)}")) + using (var rd = cmd.ExecuteReader(FormatInvariant($@"SELECT ""IsFullBackup"" FROM ""Fileset"" WHERE ""Timestamp"" = {Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(filesetTime)}"))) { if (!rd.Read()) { @@ -864,11 +881,11 @@ ON var unique_count = cmd.ExecuteScalarInt64(@"SELECT Count(*) FROM (SELECT DISTINCT ""BlocksetID"", ""Index"" FROM ""BlocklistHash"")", 0); if (real_count != unique_count) - throw new DatabaseInconsistencyException(string.Format("Found {0} blocklist hashes, but there should be {1}. Run repair to fix it.", real_count, unique_count)); + throw new DatabaseInconsistencyException($"Found {real_count} blocklist hashes, but there should be {unique_count}. Run repair to fix it."); - var itemswithnoblocklisthash = cmd.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM (SELECT * FROM (SELECT ""N"".""BlocksetID"", ((""N"".""BlockCount"" + {0} - 1) / {0}) AS ""BlocklistHashCountExpected"", CASE WHEN ""G"".""BlocklistHashCount"" IS NULL THEN 0 ELSE ""G"".""BlocklistHashCount"" END AS ""BlocklistHashCountActual"" FROM (SELECT ""BlocksetID"", COUNT(*) AS ""BlockCount"" FROM ""BlocksetEntry"" GROUP BY ""BlocksetID"") ""N"" LEFT OUTER JOIN (SELECT ""BlocksetID"", COUNT(*) AS ""BlocklistHashCount"" FROM ""BlocklistHash"" GROUP BY ""BlocksetID"") ""G"" ON ""N"".""BlocksetID"" = ""G"".""BlocksetID"" WHERE ""N"".""BlockCount"" > 1) WHERE ""BlocklistHashCountExpected"" != ""BlocklistHashCountActual"")", blocksize / hashsize), 0); + var itemswithnoblocklisthash = cmd.ExecuteScalarInt64(FormatInvariant(@"SELECT COUNT(*) FROM (SELECT * FROM (SELECT ""N"".""BlocksetID"", ((""N"".""BlockCount"" + {0} - 1) / {0}) AS ""BlocklistHashCountExpected"", CASE WHEN ""G"".""BlocklistHashCount"" IS NULL THEN 0 ELSE ""G"".""BlocklistHashCount"" END AS ""BlocklistHashCountActual"" FROM (SELECT ""BlocksetID"", COUNT(*) AS ""BlockCount"" FROM ""BlocksetEntry"" GROUP BY ""BlocksetID"") ""N"" LEFT OUTER JOIN (SELECT ""BlocksetID"", COUNT(*) AS ""BlocklistHashCount"" FROM ""BlocklistHash"" GROUP BY ""BlocksetID"") ""G"" ON ""N"".""BlocksetID"" = ""G"".""BlocksetID"" WHERE ""N"".""BlockCount"" > 1) WHERE ""BlocklistHashCountExpected"" != ""BlocklistHashCountActual"")", blocksize / hashsize), 0); if (itemswithnoblocklisthash != 0) - throw new DatabaseInconsistencyException(string.Format("Found {0} file(s) with missing blocklist hashes", itemswithnoblocklisthash)); + throw new DatabaseInconsistencyException($"Found {itemswithnoblocklisthash} file(s) with missing blocklist hashes"); if (cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""Blockset"" WHERE ""Length"" > 0 AND ""ID"" NOT IN (SELECT ""BlocksetId"" FROM ""BlocksetEntry"")") != 0) throw new DatabaseInconsistencyException("Detected non-empty blocksets with no associated blocks!"); @@ -884,7 +901,7 @@ ON if (reader.Read()) throw new DatabaseInconsistencyException($"Detected 1 fileset with missing volume: FilesetId = {reader.ConvertValueToInt64(0)}, Time = ({ParseFromEpochSeconds(reader.ConvertValueToInt64(1))}), unmatched VolumeID {reader.ConvertValueToInt64(2)}"); - throw new DatabaseInconsistencyException(string.Format("Detected {0} filesets with missing volumes", filesetsMissingVolumes)); + throw new DatabaseInconsistencyException($"Detected {filesetsMissingVolumes} filesets with missing volumes"); } var volumesMissingFilests = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""RemoteVolume"" WHERE ""Type"" = ? AND ""State"" != ? AND ""ID"" NOT IN (SELECT ""VolumeID"" FROM ""Fileset"")", 0, RemoteVolumeType.Files.ToString(), RemoteVolumeState.Deleted.ToString()); @@ -895,7 +912,7 @@ ON if (reader.Read()) throw new DatabaseInconsistencyException($"Detected 1 volume with missing filesets: VolumeId = {reader.ConvertValueToInt64(0)}, Name = {reader.ConvertValueToString(1)}, State = {reader.ConvertValueToString(2)}"); - throw new DatabaseInconsistencyException(string.Format("Detected {0} volumes with missing filesets", volumesMissingFilests)); + throw new DatabaseInconsistencyException($"Detected {volumesMissingFilests} volumes with missing filesets"); } var nonAttachedFiles = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FileID"" NOT IN (SELECT ""ID"" FROM ""FileLookup"")"); @@ -909,9 +926,9 @@ ON { foreach (var filesetid in cmd.ExecuteReaderEnumerable(@"SELECT ""ID"" FROM ""Fileset"" ").Select(x => x.ConvertValueToInt64(0, -1))) { - var expandedCmd = string.Format(@"SELECT COUNT(*) FROM (SELECT DISTINCT ""Path"" FROM ({0}) UNION SELECT DISTINCT ""Path"" FROM ({1}))", LocalDatabase.LIST_FILESETS, LocalDatabase.LIST_FOLDERS_AND_SYMLINKS); + var expandedCmd = FormatInvariant(@"SELECT COUNT(*) FROM (SELECT DISTINCT ""Path"" FROM ({0}) UNION SELECT DISTINCT ""Path"" FROM ({1}))", LocalDatabase.LIST_FILESETS, LocalDatabase.LIST_FOLDERS_AND_SYMLINKS); var expandedlist = cmd2.ExecuteScalarInt64(expandedCmd, 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, filesetid); - //var storedfilelist = cmd2.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM ""FilesetEntry"", ""FileLookup"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FileLookup"".""BlocksetID"" != ? AND ""FileLookup"".""BlocksetID"" != ?"), 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID); + //var storedfilelist = cmd2.ExecuteScalarInt64(FormatInvariant(@"SELECT COUNT(*) FROM ""FilesetEntry"", ""FileLookup"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FileLookup"".""BlocksetID"" != ? AND ""FileLookup"".""BlocksetID"" != ?"), 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID); var storedlist = cmd2.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetEntry"".""FilesetID"" = ?", 0, filesetid); if (expandedlist != storedlist) @@ -919,8 +936,8 @@ ON var filesetname = filesetid.ToString(); var fileset = FilesetTimes.Zip(Enumerable.Range(0, FilesetTimes.Count()), (a, b) => new Tuple(b, a.Key, a.Value)).FirstOrDefault(x => x.Item2 == filesetid); if (fileset != null) - filesetname = string.Format("version {0}: {1} (database id: {2})", fileset.Item1, fileset.Item3, fileset.Item2); - anyError.Add(string.Format("Unexpected difference in fileset {0}, found {1} entries, but expected {2}", filesetname, expandedlist, storedlist)); + filesetname = $"version {fileset.Item1}: {fileset.Item3} (database id: {fileset.Item2})"; + anyError.Add($"Unexpected difference in fileset {filesetname}, found {expandedlist} entries, but expected {storedlist}"); } } } @@ -1256,7 +1273,7 @@ ORDER BY cmd.Transaction = transaction; var c = cmd.ExecuteNonQuery(@"UPDATE ""Fileset"" SET ""VolumeID"" = ? WHERE ""ID"" = ?", volumeid, filesetid); if (c != 1) - throw new Exception(string.Format("Failed to link filesetid {0} to volumeid {1}", filesetid, volumeid)); + throw new Exception($"Failed to link filesetid {filesetid} to volumeid {volumeid}"); } } @@ -1302,7 +1319,7 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != ? ORDER BY Timest using (var cmd = m_connection.CreateCommand()) { cmd.Transaction = transaction; - cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS SELECT DISTINCT ""Path"" FROM ""File"" ", Tablename)); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" AS SELECT DISTINCT ""Path"" FROM ""File"" ", Tablename)); return; } } @@ -1314,10 +1331,10 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != ? ORDER BY Timest // TODO: Optimize this to not rely on the "File" view, and not instantiate the paths in full cmd.Transaction = transaction; - cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL)", Tablename)); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL)", Tablename)); using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) { - cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"") VALUES (?)", Tablename); + cmd.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" (""Path"") VALUES (?)", Tablename); cmd.AddParameter(); cmd.Transaction = tr.Parent; using (var c2 = m_connection.CreateCommand()) @@ -1361,8 +1378,8 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != ? ORDER BY Timest using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) { cmd.Transaction = tr.Parent; - cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL)", Tablename)); - cmd.ExecuteNonQuery(string.Format(@"INSERT INTO ""{0}"" SELECT DISTINCT ""Path"" FROM ""File"" WHERE " + sb, Tablename), args.ToArray()); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL)", Tablename)); + cmd.ExecuteNonQuery(FormatInvariant(@"INSERT INTO ""{0}"" SELECT DISTINCT ""Path"" FROM ""File"" WHERE " + sb, Tablename), args.ToArray()); tr.Commit(); } } @@ -1374,7 +1391,7 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != ? ORDER BY Timest try { using (var cmd = m_connection.CreateCommand()) - cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", Tablename)); + cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", Tablename)); } catch { } finally { Tablename = null; } @@ -1391,7 +1408,7 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != ? ORDER BY Timest //Rename the old entry, to preserve ID links var c = cmd.ExecuteNonQuery(@"UPDATE ""Remotevolume"" SET ""Name"" = ? WHERE ""Name"" = ?", newname, oldname); if (c != 1) - throw new Exception(string.Format("Unexpected result from renaming \"{0}\" to \"{1}\", expected {2} got {3}", oldname, newname, 1, c)); + throw new Exception($"Unexpected result from renaming \"{oldname}\" to \"{newname}\", expected {1} got {c}"); // Grab the type of entry var type = (RemoteVolumeType)Enum.Parse(typeof(RemoteVolumeType), cmd.ExecuteScalar(@"SELECT ""Type"" FROM ""Remotevolume"" WHERE ""Name"" = ?", newname).ToString(), true); @@ -1443,7 +1460,7 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != ? ORDER BY Timest using (var cmd = m_connection.CreateCommand(transaction)) { // Group subquery by hash to ensure that each blocklist hash appears only once in the result - var sql = string.Format(@"SELECT ""A"".""Hash"", ""C"".""Hash"" FROM " + + var sql = FormatInvariant(@"SELECT ""A"".""Hash"", ""C"".""Hash"" FROM " + @"(SELECT ""BlocklistHash"".""BlocksetID"", ""Block"".""Hash"", ""BlocklistHash"".""Index"" FROM ""BlocklistHash"",""Block"" WHERE ""BlocklistHash"".""Hash"" = ""Block"".""Hash"" AND ""Block"".""VolumeID"" = ? GROUP BY ""Block"".""Hash"", ""Block"".""Size"") A, " + @" ""BlocksetEntry"" B, ""Block"" C WHERE ""B"".""BlocksetID"" = ""A"".""BlocksetID"" AND " + @" ""B"".""Index"" >= (""A"".""Index"" * {0}) AND ""B"".""Index"" < ((""A"".""Index"" + 1) * {0}) AND ""C"".""ID"" = ""B"".""BlockID"" " + @@ -1520,7 +1537,7 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != ? ORDER BY Timest { using (var cmd = m_connection.CreateCommand(transaction)) { - using (var rd = cmd.ExecuteReader(@$"SELECT DISTINCT ""Fileset"".""ID"", ""Fileset"".""Timestamp"" FROM ""Fileset"", ""RemoteVolume"" WHERE ""RemoteVolume"".""ID"" = ""Fileset"".""VolumeID"" AND ""Fileset"".""ID"" IN (SELECT ""FilesetID"" FROM ""FilesetEntry"") AND (""RemoteVolume"".""State"" = '{RemoteVolumeState.Uploading}' OR ""RemoteVolume"".""State"" = '{RemoteVolumeState.Temporary}')")) + using (var rd = cmd.ExecuteReader(FormatInvariant(@$"SELECT DISTINCT ""Fileset"".""ID"", ""Fileset"".""Timestamp"" FROM ""Fileset"", ""RemoteVolume"" WHERE ""RemoteVolume"".""ID"" = ""Fileset"".""VolumeID"" AND ""Fileset"".""ID"" IN (SELECT ""FilesetID"" FROM ""FilesetEntry"") AND (""RemoteVolume"".""State"" = '{RemoteVolumeState.Uploading}' OR ""RemoteVolume"".""State"" = '{RemoteVolumeState.Temporary}')"))) while (rd.Read()) { yield return new KeyValuePair( diff --git a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs index e3e265413..fb5f5ae6e 100644 --- a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs @@ -88,7 +88,7 @@ namespace Duplicati.Library.Main.Database } if (deleted != toDelete.Length) - throw new Exception(string.Format("Unexpected number of deleted filesets {0} vs {1}", deleted, toDelete.Length)); + throw new Exception($"Unexpected number of deleted filesets {deleted} vs {toDelete.Length}"); //Then we delete anything that is no longer being referenced cmd.ExecuteNonQuery(@"DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" NOT IN (SELECT DISTINCT ""ID"" FROM ""Fileset"")"); @@ -107,7 +107,7 @@ namespace Duplicati.Library.Main.Database var updated = cmd.ExecuteNonQuery(@"UPDATE ""RemoteVolume"" SET ""State"" = ? WHERE ""Type"" = ? AND ""State"" IN (?, ?, ?, ?) AND ""ID"" NOT IN (SELECT ""VolumeID"" FROM ""Fileset"") ", RemoteVolumeState.Deleting.ToString(), RemoteVolumeType.Files.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString(), RemoteVolumeState.Temporary.ToString(), RemoteVolumeState.Deleting.ToString()); if (deleted != updated) - throw new Exception(string.Format("Unexpected number of remote volumes marked as deleted. Found {0} filesets, but {1} volumes", deleted, updated)); + throw new Exception($"Unexpected number of remote volumes marked as deleted. Found {deleted} filesets, but {updated} volumes"); using (var rd = cmd.ExecuteReader(@"SELECT ""Name"", ""Size"" FROM ""RemoteVolume"" WHERE ""Type"" = ? AND ""State"" = ? ", RemoteVolumeType.Files.ToString(), RemoteVolumeState.Deleting.ToString())) { @@ -185,7 +185,7 @@ namespace Duplicati.Library.Main.Database var combined = active + " UNION " + inactive + " UNION " + empty; var collected = @"SELECT ""VolumeID"" AS ""VolumeID"", SUM(""ActiveSize"") AS ""ActiveSize"", SUM(""InactiveSize"") AS ""InactiveSize"", MAX(""Sorttime"") AS ""Sorttime"" FROM (" + combined + @") GROUP BY ""VolumeID"" "; - var createtable = @$"CREATE {TEMPORARY} TABLE ""{tmptablename}"" AS " + collected; + var createtable = FormatInvariant(@$"CREATE {TEMPORARY} TABLE ""{tmptablename}"" AS " + collected); using (var cmd = m_connection.CreateCommand()) { @@ -193,13 +193,13 @@ namespace Duplicati.Library.Main.Database try { cmd.ExecuteNonQuery(createtable, RemoteVolumeType.Blocks.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()); - using (var rd = cmd.ExecuteReader(string.Format(@"SELECT ""A"".""Name"", ""B"".""ActiveSize"", ""B"".""InactiveSize"", ""A"".""Size"" FROM ""Remotevolume"" A, ""{0}"" B WHERE ""A"".""ID"" = ""B"".""VolumeID"" ORDER BY ""B"".""Sorttime"" ASC ", tmptablename))) + using (var rd = cmd.ExecuteReader(FormatInvariant(@"SELECT ""A"".""Name"", ""B"".""ActiveSize"", ""B"".""InactiveSize"", ""A"".""Size"" FROM ""Remotevolume"" A, ""{0}"" B WHERE ""A"".""ID"" = ""B"".""VolumeID"" ORDER BY ""B"".""Sorttime"" ASC ", tmptablename))) while (rd.Read()) yield return new VolumeUsage(rd.GetValue(0).ToString(), rd.ConvertValueToInt64(1, 0) + rd.ConvertValueToInt64(2, 0), rd.ConvertValueToInt64(2, 0), rd.ConvertValueToInt64(3, 0)); } finally { - try { cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", tmptablename)); } + try { cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", tmptablename)); } catch { } } } @@ -388,20 +388,20 @@ namespace Duplicati.Library.Main.Database var replacementBlocks = "ReplacementBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); try { - cmd.ExecuteNonQuery(@$"CREATE {TEMPORARY} TABLE """ + updatedBlocks + @""" AS SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" = ? ", deletedVolume.ID); - cmd.ExecuteNonQuery(@$"CREATE {TEMPORARY} TABLE """ + replacementBlocks + @""" AS SELECT ""BlockID"", MAX(""VolumeID"") AS ""VolumeID"" FROM ""DuplicateBlock"" WHERE ""VolumeID"" NOT IN (?) AND ""BlockID"" IN (SELECT ""ID"" FROM """ + updatedBlocks + @""") GROUP BY ""BlockID"" ", volumeIdsToBeRemoved.ToArray()); + cmd.ExecuteNonQuery(FormatInvariant(@$"CREATE {TEMPORARY} TABLE """ + updatedBlocks + @""" AS SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" = ? "), deletedVolume.ID); + cmd.ExecuteNonQuery(FormatInvariant(@$"CREATE {TEMPORARY} TABLE """ + replacementBlocks + @""" AS SELECT ""BlockID"", MAX(""VolumeID"") AS ""VolumeID"" FROM ""DuplicateBlock"" WHERE ""VolumeID"" NOT IN (?) AND ""BlockID"" IN (SELECT ""ID"" FROM """ + updatedBlocks + @""") GROUP BY ""BlockID"" "), volumeIdsToBeRemoved.ToArray()); var targetCount = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM """ + updatedBlocks + @""" "); if (targetCount == 0) return; var replacementCount = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM """ + replacementBlocks + @""" "); - var updateCount = cmd.ExecuteNonQuery(@$"UPDATE ""Block"" SET ""VolumeID"" = (SELECT ""VolumeID"" FROM ""{replacementBlocks}"" WHERE ""{replacementBlocks}"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" = ?) WHERE ""Block"".""VolumeID"" = ? ", deletedVolume.ID, deletedVolume.ID); - var deleteCount = cmd.ExecuteNonQuery(@$"DELETE FROM ""DuplicateBlock"" WHERE (""DuplicateBlock"".""BlockID"" || ':' || ""DuplicateBlock"".""VolumeID"") IN (SELECT ""RB"".""BlockID"" || ':' || ""RB"".""VolumeID"" FROM ""{replacementBlocks}"" RB)"); + var updateCount = cmd.ExecuteNonQuery(FormatInvariant(@$"UPDATE ""Block"" SET ""VolumeID"" = (SELECT ""VolumeID"" FROM ""{replacementBlocks}"" WHERE ""{replacementBlocks}"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" = ?) WHERE ""Block"".""VolumeID"" = ? "), deletedVolume.ID, deletedVolume.ID); + var deleteCount = cmd.ExecuteNonQuery(FormatInvariant(@$"DELETE FROM ""DuplicateBlock"" WHERE (""DuplicateBlock"".""BlockID"" || ':' || ""DuplicateBlock"".""VolumeID"") IN (SELECT ""RB"".""BlockID"" || ':' || ""RB"".""VolumeID"" FROM ""{replacementBlocks}"" RB)")); if (targetCount != updateCount || replacementCount != deleteCount || updateCount != deleteCount) - throw new Exception(string.Format($"Unexpected number of rows updated. Expected {targetCount} but got updated {updateCount}, deleted {deleteCount}, and replaced {replacementCount}")); + throw new Exception($"Unexpected number of rows updated. Expected {targetCount} but got updated {updateCount}, deleted {deleteCount}, and replaced {replacementCount}"); // Remove knowledge of any old blocks - cmd.ExecuteNonQuery(@"DELETE FROM ""DuplicateBlock"" WHERE ""VolumeID"" = ?", deletedVolume.ID); + cmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""DuplicateBlock"" WHERE ""VolumeID"" = ?"), deletedVolume.ID); } finally { diff --git a/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs b/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs index a51dba614..4317fa11b 100644 --- a/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -66,7 +66,7 @@ namespace Duplicati.Library.Main.Database for(var i = 0; i < filesets.Length; i++) dict[filesets[i].Key] = i; - var sql = string.Format( + var sql = FormatInvariant( @"SELECT DISTINCT ""FilesetID"" FROM (" + @"SELECT ""FilesetID"" FROM ""FilesetEntry"" WHERE ""FileID"" IN ( SELECT ""ID"" FROM ""FileLookup"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN ( SELECT ""ID"" From ""Block"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN ({0})))))" + " UNION " + @@ -92,7 +92,7 @@ namespace Duplicati.Library.Main.Database public IEnumerable GetFiles(IEnumerable items) { - var sql = string.Format( + var sql = FormatInvariant( @"SELECT DISTINCT ""Path"" FROM (" + @"SELECT ""Path"" FROM ""File"" WHERE ""BlocksetID"" IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN (SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" IN (SELECT ""ID"" from ""RemoteVolume"" WHERE ""Name"" IN ({0}))))" + @" UNION " + @@ -118,7 +118,7 @@ namespace Duplicati.Library.Main.Database public IEnumerable GetLogLines(IEnumerable items) { - var sql = string.Format( + var sql = FormatInvariant( @"SELECT ""TimeStamp"", ""Message"" || ' ' || CASE WHEN ""Exception"" IS NULL THEN '' ELSE ""Exception"" END FROM ""LogData"" WHERE {0}" + @" UNION " + @"SELECT ""Timestamp"", ""Data"" FROM ""RemoteOperation"" WHERE ""Path"" IN ({1})", @@ -140,7 +140,7 @@ namespace Duplicati.Library.Main.Database public IEnumerable GetVolumes(IEnumerable items) { - var sql = string.Format( + var sql = FormatInvariant( @"SELECT DISTINCT ""Name"" FROM ( " + @" SELECT ""Name"" FROM ""Remotevolume"" WHERE ""ID"" IN ( SELECT ""VolumeID"" FROM ""Block"" WHERE ""ID"" IN ( SELECT ""BlockID"" FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""FileLookup"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" IN ( SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN ({0}))))))) " + @" UNION " + diff --git a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs index cf6e07aad..a72ba7df5 100644 --- a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -82,7 +82,7 @@ SELECT ""A"".""Path"", ""B"".""Length"" FROM ""File"" A, ""Blockset"" B WHERE "" public IEnumerable> GetBrokenFilesets(DateTime time, long[] versions, System.Data.IDbTransaction transaction) { - var query = string.Format(BROKEN_FILE_SETS, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, RemoteVolumeType.Blocks.ToString()); + var query = FormatInvariant(BROKEN_FILE_SETS, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, RemoteVolumeType.Blocks.ToString()); var clause = GetFilelistWhereClause(time, versions); if (!string.IsNullOrWhiteSpace(clause.Item1)) query += @" AND ""A"".""FilesetID"" IN (SELECT ""ID"" FROM ""Fileset"" " + clause.Item1 + ")"; @@ -98,7 +98,7 @@ SELECT ""A"".""Path"", ""B"".""Length"" FROM ""File"" A, ""Blockset"" B WHERE "" public IEnumerable> GetBrokenFilenames(long filesetid, System.Data.IDbTransaction transaction) { using (var cmd = Connection.CreateCommand(transaction)) - foreach (var rd in cmd.ExecuteReaderEnumerable(string.Format(BROKEN_FILE_NAMES, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, RemoteVolumeType.Blocks.ToString()), filesetid)) + foreach (var rd in cmd.ExecuteReaderEnumerable(FormatInvariant(BROKEN_FILE_NAMES, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, RemoteVolumeType.Blocks.ToString()), filesetid)) if (!rd.IsDBNull(0)) yield return new Tuple(rd.ConvertValueToString(0), rd.ConvertValueToInt64(1)); } @@ -106,7 +106,7 @@ SELECT ""A"".""Path"", ""B"".""Length"" FROM ""File"" A, ""Blockset"" B WHERE "" public void InsertBrokenFileIDsIntoTable(long filesetid, string tablename, string IDfieldname, System.Data.IDbTransaction transaction) { using (var cmd = Connection.CreateCommand(transaction)) - cmd.ExecuteNonQuery(string.Format(INSERT_BROKEN_IDS, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, RemoteVolumeType.Blocks.ToString(), tablename, IDfieldname), filesetid); + cmd.ExecuteNonQuery(FormatInvariant(INSERT_BROKEN_IDS, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, RemoteVolumeType.Blocks.ToString(), tablename, IDfieldname), filesetid); } public void RemoveMissingBlocks(IEnumerable names, System.Data.IDbTransaction transaction) @@ -121,8 +121,8 @@ SELECT ""A"".""Path"", ""B"".""Length"" FROM ""File"" A, ""Blockset"" B WHERE "" var volidstable = "DelVolSetIds-" + temptransguid; // Create and fill a temp table with the volids to delete. We avoid using too many parameters that way. - deletecmd.ExecuteNonQuery(string.Format(@"CREATE TEMP TABLE ""{0}"" (""ID"" INTEGER PRIMARY KEY)", volidstable)); - deletecmd.CommandText = string.Format(@"INSERT OR IGNORE INTO ""{0}"" (""ID"") VALUES (?)", volidstable); + deletecmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMP TABLE ""{0}"" (""ID"" INTEGER PRIMARY KEY)", volidstable)); + deletecmd.CommandText = FormatInvariant(@"INSERT OR IGNORE INTO ""{0}"" (""ID"") VALUES (?)", volidstable); deletecmd.Parameters.Clear(); deletecmd.AddParameters(1); foreach (var name in names) @@ -131,22 +131,22 @@ SELECT ""A"".""Path"", ""B"".""Length"" FROM ""File"" A, ""Blockset"" B WHERE "" deletecmd.SetParameterValue(0, volumeid); deletecmd.ExecuteNonQuery(); } - var volIdsSubQuery = string.Format(@"SELECT ""ID"" FROM ""{0}"" ", volidstable); + var volIdsSubQuery = FormatInvariant(@"SELECT ""ID"" FROM ""{0}"" ", volidstable); deletecmd.Parameters.Clear(); - deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN ({0}) OR ""IndexVolumeID"" IN ({0})", volIdsSubQuery)); - deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""Block"" WHERE ""VolumeID"" IN ({0})", volIdsSubQuery)); - deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""DeletedBlock"" WHERE ""VolumeID"" IN ({0})", volIdsSubQuery)); - deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""DuplicateBlock"" WHERE ""VolumeID"" IN ({0})", volIdsSubQuery)); + deletecmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN ({0}) OR ""IndexVolumeID"" IN ({0})", volIdsSubQuery)); + deletecmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""Block"" WHERE ""VolumeID"" IN ({0})", volIdsSubQuery)); + deletecmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""DeletedBlock"" WHERE ""VolumeID"" IN ({0})", volIdsSubQuery)); + deletecmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""DuplicateBlock"" WHERE ""VolumeID"" IN ({0})", volIdsSubQuery)); // Clean up temp tables for subqueries. We truncate content and then try to delete. // Drop in try-block, as it fails in nested transactions (SQLite problem) // System.Data.SQLite.SQLiteException (0x80004005): database table is locked - deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""{0}"" ", volidstable)); + deletecmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""{0}"" ", volidstable)); try { deletecmd.CommandTimeout = 2; - deletecmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", volidstable)); + deletecmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", volidstable)); } catch { /* Ignore, will be deleted on close anyway. */ } } diff --git a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs index bd84a6073..031d7c3cc 100644 --- a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -115,18 +115,18 @@ namespace Duplicati.Library.Main.Database { cmd.Transaction = m_transaction; - cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) ", m_previousTable)); - cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) ", m_currentTable)); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) ", m_previousTable)); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) ", m_currentTable)); } m_insertPreviousElementCommand = m_connection.CreateCommand(); m_insertPreviousElementCommand.Transaction = m_transaction; - m_insertPreviousElementCommand.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (?,?,?,?,?)", m_previousTable); + m_insertPreviousElementCommand.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (?,?,?,?,?)", m_previousTable); m_insertPreviousElementCommand.AddParameters(5); m_insertCurrentElementCommand = m_connection.CreateCommand(); m_insertCurrentElementCommand.Transaction = m_transaction; - m_insertCurrentElementCommand.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (?,?,?,?,?)", m_currentTable); + m_insertCurrentElementCommand.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (?,?,?,?,?)", m_currentTable); m_insertCurrentElementCommand.AddParameters(5); } @@ -134,9 +134,9 @@ namespace Duplicati.Library.Main.Database { var tablename = asNew ? m_currentTable : m_previousTable; - var folders = string.Format(@"SELECT ""File"".""Path"" AS ""Path"", NULL AS ""FileHash"", ""Blockset"".""Fullhash"" AS ""MetaHash"", -1 AS ""Size"", {0} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" = -100 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" ", (int)Library.Interface.ListChangesElementType.Folder); - var symlinks = string.Format(@"SELECT ""File"".""Path"" AS ""Path"", NULL AS ""FileHash"", ""Blockset"".""Fullhash"" AS ""MetaHash"", -1 AS ""Size"", {0} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" = -200 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" ", (int)Library.Interface.ListChangesElementType.Symlink); - var files = string.Format(@"SELECT ""File"".""Path"" AS ""Path"", ""FB"".""FullHash"" AS ""FileHash"", ""MB"".""Fullhash"" AS ""MetaHash"", ""FB"".""Length"" AS ""Size"", {0} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" MB, ""Blockset"" FB WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" >= 0 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""MB"".""ID"" AND ""File"".""BlocksetID"" = ""FB"".""ID"" ", (int)Library.Interface.ListChangesElementType.File); + var folders = FormatInvariant(@"SELECT ""File"".""Path"" AS ""Path"", NULL AS ""FileHash"", ""Blockset"".""Fullhash"" AS ""MetaHash"", -1 AS ""Size"", {0} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" = -100 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" ", (int)Library.Interface.ListChangesElementType.Folder); + var symlinks = FormatInvariant(@"SELECT ""File"".""Path"" AS ""Path"", NULL AS ""FileHash"", ""Blockset"".""Fullhash"" AS ""MetaHash"", -1 AS ""Size"", {0} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" = -200 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" ", (int)Library.Interface.ListChangesElementType.Symlink); + var files = FormatInvariant(@"SELECT ""File"".""Path"" AS ""Path"", ""FB"".""FullHash"" AS ""FileHash"", ""MB"".""Fullhash"" AS ""MetaHash"", ""FB"".""Length"" AS ""Size"", {0} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" MB, ""Blockset"" FB WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" >= 0 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""MB"".""ID"" AND ""File"".""BlocksetID"" = ""FB"".""ID"" ", (int)Library.Interface.ListChangesElementType.File); var combined = "(" + folders + " UNION " + symlinks + " UNION " + files + ")"; @@ -146,7 +146,7 @@ namespace Duplicati.Library.Main.Database if (filter == null || filter.Empty) { // Simple case, select everything - cmd.ExecuteNonQuery(string.Format(@"INSERT INTO ""{0}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") SELECT ""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"" FROM {1} A WHERE ""A"".""FilesetID"" = ? ", tablename, combined), filesetId); + cmd.ExecuteNonQuery(FormatInvariant(@"INSERT INTO ""{0}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") SELECT ""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"" FROM {1} A WHERE ""A"".""FilesetID"" = ? ", tablename, combined), filesetId); } else if (Library.Utility.Utility.IsFSCaseSensitive && filter is FilterExpression expression && expression.Type == Duplicati.Library.Utility.FilterType.Simple) { @@ -155,8 +155,8 @@ namespace Duplicati.Library.Main.Database // SQLite only supports ASCII compares var p = expression.GetSimpleList(); var filenamestable = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL) ", filenamestable)); - cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"") VALUES (?)", filenamestable); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL) ", filenamestable)); + cmd.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" (""Path"") VALUES (?)", filenamestable); cmd.AddParameter(); foreach(var s in p) @@ -169,15 +169,15 @@ namespace Duplicati.Library.Main.Database if (expression.Result) { // Include filter - whereClause = string.Format(@"""A"".""FilesetID"" = ? AND ""A"".""Path"" IN (SELECT DISTINCT ""Path"" FROM ""{0}"")", filenamestable); + whereClause = FormatInvariant(@"""A"".""FilesetID"" = ? AND ""A"".""Path"" IN (SELECT DISTINCT ""Path"" FROM ""{0}"")", filenamestable); } else { // Exclude filter - whereClause = string.Format(@"""A"".""FilesetID"" = ? AND ""A"".""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""{0}"")", filenamestable); + whereClause = FormatInvariant(@"""A"".""FilesetID"" = ? AND ""A"".""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""{0}"")", filenamestable); } - cmd.ExecuteNonQuery(string.Format(@"INSERT INTO ""{0}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") SELECT ""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"" FROM {1} A WHERE {2} ", tablename, combined, whereClause), filesetId); - cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", filenamestable)); + cmd.ExecuteNonQuery(FormatInvariant(@"INSERT INTO ""{0}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") SELECT ""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"" FROM {1} A WHERE {2} ", tablename, combined, whereClause), filesetId); + cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", filenamestable)); } else { @@ -185,11 +185,11 @@ namespace Duplicati.Library.Main.Database object[] values = new object[5]; using(var cmd2 = m_connection.CreateCommand()) { - cmd2.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (?,?,?,?,?)", tablename); + cmd2.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (?,?,?,?,?)", tablename); cmd2.AddParameters(5); cmd2.Transaction = m_transaction; - using(var rd = cmd.ExecuteReader(string.Format(@"SELECT ""A"".""Path"", ""A"".""FileHash"", ""A"".""MetaHash"", ""A"".""Size"", ""A"".""Type"" FROM {0} A WHERE ""A"".""FilesetID"" = ?", combined), filesetId)) + using(var rd = cmd.ExecuteReader(FormatInvariant(@"SELECT ""A"".""Path"", ""A"".""FileHash"", ""A"".""MetaHash"", ""A"".""Size"", ""A"".""Type"" FROM {0} A WHERE ""A"".""FilesetID"" = ?", combined), filesetId)) while (rd.Read()) { rd.GetValues(values); @@ -235,9 +235,9 @@ namespace Duplicati.Library.Main.Database private Tuple GetSqls(bool allTypes) { return new Tuple( - string.Format(@"SELECT ""Path"" FROM ""{0}"" WHERE {2} ""{0}"".""Path"" NOT IN (SELECT ""Path"" FROM ""{1}"")", m_currentTable, m_previousTable, string.Format(allTypes ? "" : @" ""{0}"".""Type"" = ? AND ", m_currentTable)), - string.Format(@"SELECT ""Path"" FROM ""{0}"" WHERE {2} ""{0}"".""Path"" NOT IN (SELECT ""Path"" FROM ""{1}"")", m_previousTable, m_currentTable, string.Format(allTypes ? "" : @" ""{0}"".""Type"" = ? AND ", m_previousTable)), - string.Format(@"SELECT ""{0}"".""Path"" FROM ""{0}"",""{1}"" WHERE {2} ""{0}"".""Path"" = ""{1}"".""Path"" AND (""{0}"".""FileHash"" != ""{1}"".""FileHash"" OR ""{0}"".""MetaHash"" != ""{1}"".""MetaHash"" OR ""{0}"".""Type"" != ""{1}"".""Type"") ", m_currentTable, m_previousTable, string.Format(allTypes ? "" : @" ""{0}"".""Type"" = ? AND ", m_currentTable)) + FormatInvariant(@"SELECT ""Path"" FROM ""{0}"" WHERE {2} ""{0}"".""Path"" NOT IN (SELECT ""Path"" FROM ""{1}"")", m_currentTable, m_previousTable, FormatInvariant(allTypes ? "" : @" ""{0}"".""Type"" = ? AND ", m_currentTable)), + FormatInvariant(@"SELECT ""Path"" FROM ""{0}"" WHERE {2} ""{0}"".""Path"" NOT IN (SELECT ""Path"" FROM ""{1}"")", m_previousTable, m_currentTable, FormatInvariant(allTypes ? "" : @" ""{0}"".""Type"" = ? AND ", m_previousTable)), + FormatInvariant(@"SELECT ""{0}"".""Path"" FROM ""{0}"",""{1}"" WHERE {2} ""{0}"".""Path"" = ""{1}"".""Path"" AND (""{0}"".""FileHash"" != ""{1}"".""FileHash"" OR ""{0}"".""MetaHash"" != ""{1}"".""MetaHash"" OR ""{0}"".""Type"" != ""{1}"".""Type"") ", m_currentTable, m_previousTable, FormatInvariant(allTypes ? "" : @" ""{0}"".""Type"" = ? AND ", m_currentTable)) ); } @@ -254,11 +254,11 @@ namespace Duplicati.Library.Main.Database var result = new ChangeSizeReport(); - result.PreviousSize = cmd.ExecuteScalarInt64(string.Format(@"SELECT SUM(""Size"") FROM ""{0}"" ", m_previousTable), 0); - result.CurrentSize = cmd.ExecuteScalarInt64(string.Format(@"SELECT SUM(""Size"") FROM ""{0}"" ", m_currentTable), 0); + result.PreviousSize = cmd.ExecuteScalarInt64(FormatInvariant(@"SELECT SUM(""Size"") FROM ""{0}"" ", m_previousTable), 0); + result.CurrentSize = cmd.ExecuteScalarInt64(FormatInvariant(@"SELECT SUM(""Size"") FROM ""{0}"" ", m_currentTable), 0); - result.AddedSize = cmd.ExecuteScalarInt64(string.Format(@"SELECT SUM(""Size"") FROM ""{0}"" WHERE ""{0}"".""Path"" IN ({1}) ", m_currentTable, added), 0); - result.DeletedSize = cmd.ExecuteScalarInt64(string.Format(@"SELECT SUM(""Size"") FROM ""{0}"" WHERE ""{0}"".""Path"" IN ({1}) ", m_previousTable, deleted), 0); + result.AddedSize = cmd.ExecuteScalarInt64(FormatInvariant(@"SELECT SUM(""Size"") FROM ""{0}"" WHERE ""{0}"".""Path"" IN ({1}) ", m_currentTable, added), 0); + result.DeletedSize = cmd.ExecuteScalarInt64(FormatInvariant(@"SELECT SUM(""Size"") FROM ""{0}"" WHERE ""{0}"".""Path"" IN ({1}) ", m_previousTable, deleted), 0); return result; } diff --git a/Duplicati/Library/Main/Database/LocalListDatabase.cs b/Duplicati/Library/Main/Database/LocalListDatabase.cs index c602ee629..6ab0d1abf 100644 --- a/Duplicati/Library/Main/Database/LocalListDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListDatabase.cs @@ -1,28 +1,27 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Linq; using System.Collections.Generic; -using System.Text; using Duplicati.Library.Common.IO; namespace Duplicati.Library.Main.Database @@ -77,8 +76,8 @@ namespace Duplicati.Library.Main.Database using(var cmd = m_connection.CreateCommand()) { - cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS SELECT DISTINCT ""ID"" AS ""FilesetID"", ""IsFullBackup"" AS ""IsFullBackup"" , ""Timestamp"" AS ""Timestamp"" FROM ""Fileset"" " + query, m_tablename), args); - cmd.ExecuteNonQuery(string.Format(@"CREATE INDEX ""{0}_FilesetIDTimestampIndex"" ON ""{0}"" (""FilesetID"", ""Timestamp"" DESC)", m_tablename)); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" AS SELECT DISTINCT ""ID"" AS ""FilesetID"", ""IsFullBackup"" AS ""IsFullBackup"" , ""Timestamp"" AS ""Timestamp"" FROM ""Fileset"" " + query, m_tablename), args); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE INDEX ""{0}_FilesetIDTimestampIndex"" ON ""{0}"" (""FilesetID"", ""Timestamp"" DESC)", m_tablename)); } } @@ -143,14 +142,14 @@ namespace Duplicati.Library.Main.Database using(var cmd = m_connection.CreateCommand()) { //First we trim the filelist to exclude filenames not found in any of the filesets - cmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""{0}"" WHERE ""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""File"", ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""FilesetEntry"".""FilesetID"" IN (SELECT ""FilesetID"" FROM ""{1}"") ) ", tmpnames.Tablename, m_tablename)); + cmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""{0}"" WHERE ""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""File"", ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""FilesetEntry"".""FilesetID"" IN (SELECT ""FilesetID"" FROM ""{1}"") ) ", tmpnames.Tablename, m_tablename)); //If we have a prefix rule, apply it if (!string.IsNullOrWhiteSpace(prefixrule)) - cmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""{0}"" WHERE SUBSTR(""Path"", 1, {1}) != ?", tmpnames.Tablename, prefixrule.Length), prefixrule); + cmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""{0}"" WHERE SUBSTR(""Path"", 1, {1}) != ?", tmpnames.Tablename, prefixrule.Length), prefixrule); // Then we recursively find the largest prefix - cmd.CommandText = string.Format(@"SELECT ""Path"" FROM ""{0}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1", tmpnames.Tablename); + cmd.CommandText = FormatInvariant(@"SELECT ""Path"" FROM ""{0}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1", tmpnames.Tablename); var v0 = cmd.ExecuteScalar(); string maxpath = ""; if (v0 != null) @@ -158,12 +157,12 @@ namespace Duplicati.Library.Main.Database var dirsep = Util.GuessDirSeparator(maxpath); - cmd.CommandText = string.Format(@"SELECT COUNT(*) FROM ""{0}""", tmpnames.Tablename); + cmd.CommandText = FormatInvariant(@"SELECT COUNT(*) FROM ""{0}""", tmpnames.Tablename); var filecount = cmd.ExecuteScalarInt64(0); long foundfiles = -1; //TODO: Handle FS case-sensitive? - cmd.CommandText = string.Format(@"SELECT COUNT(*) FROM ""{0}"" WHERE SUBSTR(""Path"", 1, ?) = ?", tmpnames.Tablename); + cmd.CommandText = FormatInvariant(@"SELECT COUNT(*) FROM ""{0}"" WHERE SUBSTR(""Path"", 1, ?) = ?", tmpnames.Tablename); cmd.AddParameter(); cmd.AddParameter(); @@ -189,7 +188,7 @@ namespace Duplicati.Library.Main.Database // Special handling for Windows and multi-drive/UNC backups as they do not have a single common root if (string.IsNullOrWhiteSpace(maxpath) && string.IsNullOrWhiteSpace(prefixrule)) { - var paths = cmd.ExecuteReaderEnumerable(string.Format(@"SELECT Path FROM ""{0}""", tmpnames.Tablename)).Select(x => x.ConvertValueToString(0)).ToArray(); + var paths = cmd.ExecuteReaderEnumerable(FormatInvariant(@"SELECT Path FROM ""{0}""", tmpnames.Tablename)).Select(x => x.ConvertValueToString(0)).ToArray(); var roots = paths.Select(x => x.Substring(0, 1)).Distinct().Where(x => x != "\\").ToArray(); //unc path like \\server.domain\ @@ -212,7 +211,7 @@ namespace Duplicati.Library.Main.Database prefix = Util.AppendDirSeparator(prefix, Util.GuessDirSeparator(prefix)); var ppl = prefix.Length; - using(var rd = cmd.ExecuteReader(string.Format(@"SELECT DISTINCT ""Path"" FROM ""{0}"" ", table))) + using(var rd = cmd.ExecuteReader(FormatInvariant(@"SELECT DISTINCT ""Path"" FROM ""{0}"" ", table))) while (rd.Read()) { var s = rd.GetString(0); @@ -251,7 +250,7 @@ namespace Duplicati.Library.Main.Database using(var cmd = m_connection.CreateCommand()) { //First we trim the filelist to exclude filenames not found in any of the filesets - cmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""{0}"" WHERE ""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""File"", ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""FilesetEntry"".""FilesetID"" IN (SELECT ""FilesetID"" FROM ""{1}"") ) ", tmpnames.Tablename, m_tablename)); + cmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""{0}"" WHERE ""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""File"", ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""FilesetEntry"".""FilesetID"" IN (SELECT ""FilesetID"" FROM ""{1}"") ) ", tmpnames.Tablename, m_tablename)); // If we had instr support this would work: /*var distinctPaths = @"SELECT DISTINCT :1 || " + @@ -262,11 +261,11 @@ namespace Duplicati.Library.Main.Database @" FROM (" + cartesianPathFileset + @")";*/ // Instead we manually iterate the paths - cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL)", tbname)); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL)", tbname)); using(var c2 = m_connection.CreateCommand()) { - c2.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"") VALUES (?)", tbname); + c2.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" (""Path"") VALUES (?)", tbname); c2.AddParameter(); foreach(var n in SelectFolderEntries(cmd, pathprefix, tmpnames.Tablename).Distinct()) @@ -275,14 +274,14 @@ namespace Duplicati.Library.Main.Database c2.ExecuteNonQuery(); } - c2.ExecuteNonQuery(string.Format(@"CREATE INDEX ""{0}_PathIndex"" ON ""{0}"" (""Path"")", tbname)); + c2.ExecuteNonQuery(FormatInvariant(@"CREATE INDEX ""{0}_PathIndex"" ON ""{0}"" (""Path"")", tbname)); } //Then we select the matching results - var filesets = string.Format(@"SELECT ""FilesetID"", ""Timestamp"" FROM ""{0}"" ORDER BY ""Timestamp"" DESC", m_tablename); - var cartesianPathFileset = string.Format(@"SELECT ""A"".""Path"", ""B"".""FilesetID"" FROM ""{0}"" A, (" + filesets + @") B ORDER BY ""A"".""Path"" ASC, ""B"".""Timestamp"" DESC", tbname, m_tablename); + var filesets = FormatInvariant(@"SELECT ""FilesetID"", ""Timestamp"" FROM ""{0}"" ORDER BY ""Timestamp"" DESC", m_tablename); + var cartesianPathFileset = FormatInvariant(@"SELECT ""A"".""Path"", ""B"".""FilesetID"" FROM ""{0}"" A, (" + filesets + @") B ORDER BY ""A"".""Path"" ASC, ""B"".""Timestamp"" DESC", tbname, m_tablename); - var filesWithSizes = string.Format(@"SELECT ""Length"", ""FilesetEntry"".""FilesetID"", ""File"".""Path"" FROM ""Blockset"", ""FilesetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND FilesetEntry.""FilesetID"" IN (SELECT DISTINCT ""FilesetID"" FROM ""{0}"") ", m_tablename); + var filesWithSizes = FormatInvariant(@"SELECT ""Length"", ""FilesetEntry"".""FilesetID"", ""File"".""Path"" FROM ""Blockset"", ""FilesetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND FilesetEntry.""FilesetID"" IN (SELECT DISTINCT ""FilesetID"" FROM ""{0}"") ", m_tablename); var query = @"SELECT ""C"".""Path"", ""D"".""Length"", ""C"".""FilesetID"" FROM (" + cartesianPathFileset + @") C LEFT OUTER JOIN (" + filesWithSizes + @") D ON ""C"".""FilesetID"" = ""D"".""FilesetID"" AND ""C"".""Path"" = ""D"".""Path"""; cmd.AddParameter(pathprefix, "1"); @@ -314,7 +313,7 @@ namespace Duplicati.Library.Main.Database try { using(var c = m_connection.CreateCommand()) - c.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}""", tbname)); + c.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}""", tbname)); } catch { @@ -328,12 +327,12 @@ namespace Duplicati.Library.Main.Database using(var cmd = m_connection.CreateCommand()) { //First we trim the filelist to exclude filenames not found in any of the filesets - cmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""{0}"" WHERE ""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""File"", ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""FilesetEntry"".""FilesetID"" IN (SELECT ""FilesetID"" FROM ""{1}"") ) ", tmpnames.Tablename, m_tablename)); + cmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""{0}"" WHERE ""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""File"", ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""FilesetEntry"".""FilesetID"" IN (SELECT ""FilesetID"" FROM ""{1}"") ) ", tmpnames.Tablename, m_tablename)); //Then we select the matching results - var filesets = string.Format(@"SELECT ""FilesetID"", ""Timestamp"" FROM ""{0}"" ORDER BY ""Timestamp"" DESC", m_tablename); - var cartesianPathFileset = string.Format(@"SELECT ""A"".""Path"", ""B"".""FilesetID"" FROM ""{0}"" A, (" + filesets + @") B ORDER BY ""A"".""Path"" ASC, ""B"".""Timestamp"" DESC", tmpnames.Tablename, m_tablename); - var filesWithSizes = string.Format(@"SELECT ""Length"", ""FilesetEntry"".""FilesetID"", ""File"".""Path"" FROM ""Blockset"", ""FilesetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND FilesetEntry.""FilesetID"" IN (SELECT DISTINCT ""FilesetID"" FROM ""{0}"") ", m_tablename); + var filesets = FormatInvariant(@"SELECT ""FilesetID"", ""Timestamp"" FROM ""{0}"" ORDER BY ""Timestamp"" DESC", m_tablename); + var cartesianPathFileset = FormatInvariant(@"SELECT ""A"".""Path"", ""B"".""FilesetID"" FROM ""{0}"" A, (" + filesets + @") B ORDER BY ""A"".""Path"" ASC, ""B"".""Timestamp"" DESC", tmpnames.Tablename, m_tablename); + var filesWithSizes = FormatInvariant(@"SELECT ""Length"", ""FilesetEntry"".""FilesetID"", ""File"".""Path"" FROM ""Blockset"", ""FilesetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND FilesetEntry.""FilesetID"" IN (SELECT DISTINCT ""FilesetID"" FROM ""{0}"") ", m_tablename); var query = @"SELECT ""C"".""Path"", ""D"".""Length"", ""C"".""FilesetID"" FROM (" + cartesianPathFileset + @") C LEFT OUTER JOIN (" + filesWithSizes + @") D ON ""C"".""FilesetID"" = ""D"".""FilesetID"" AND ""C"".""Path"" = ""D"".""Path"""; using(var rd = cmd.ExecuteReader(query)) if(rd.Read()) @@ -352,7 +351,7 @@ namespace Duplicati.Library.Main.Database public void TakeFirst() { using(var cmd = m_connection.CreateCommand()) - cmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""{0}"" WHERE ""FilesetID"" NOT IN (SELECT ""FilesetID"" FROM ""{0}"" ORDER BY ""Timestamp"" DESC LIMIT 1 )", m_tablename)); + cmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""{0}"" WHERE ""FilesetID"" NOT IN (SELECT ""FilesetID"" FROM ""{0}"" ORDER BY ""Timestamp"" DESC LIMIT 1 )", m_tablename)); } public IEnumerable QuickSets @@ -391,13 +390,13 @@ namespace Duplicati.Library.Main.Database dict[m_filesets[i].Key] = i; } - var summation = - $@"SELECT ""A"".""FilesetID"" AS ""FilesetID"", COUNT(*) AS ""FileCount"", SUM(""C"".""Length"") AS ""FileSizes"" FROM ""FilesetEntry"" A, ""File"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" AND ""A"".""FilesetID"" IN (SELECT DISTINCT ""FilesetID"" FROM ""{m_tablename}"") GROUP BY ""A"".""FilesetID"" "; + var summation = FormatInvariant( + $@"SELECT ""A"".""FilesetID"" AS ""FilesetID"", COUNT(*) AS ""FileCount"", SUM(""C"".""Length"") AS ""FileSizes"" FROM ""FilesetEntry"" A, ""File"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" AND ""A"".""FilesetID"" IN (SELECT DISTINCT ""FilesetID"" FROM ""{m_tablename}"") GROUP BY ""A"".""FilesetID"" "); using (var cmd = m_connection.CreateCommand()) { - using (var rd = cmd.ExecuteReader( - $@"SELECT DISTINCT ""A"".""FilesetID"", ""A"".""IsFullBackup"", ""B"".""FileCount"", ""B"".""FileSizes"" FROM ""{m_tablename}"" A LEFT OUTER JOIN ( {summation} ) B ON ""A"".""FilesetID"" = ""B"".""FilesetID"" ORDER BY ""A"".""Timestamp"" DESC ") + using (var rd = cmd.ExecuteReader(FormatInvariant( + $@"SELECT DISTINCT ""A"".""FilesetID"", ""A"".""IsFullBackup"", ""B"".""FileCount"", ""B"".""FileSizes"" FROM ""{m_tablename}"" A LEFT OUTER JOIN ( {summation} ) B ON ""A"".""FilesetID"" = ""B"".""FilesetID"" ORDER BY ""A"".""Timestamp"" DESC ")) ) { while (rd.Read()) @@ -423,7 +422,7 @@ namespace Duplicati.Library.Main.Database { using (var cmd = m_connection.CreateCommand()) { - cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", m_tablename)); + cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", m_tablename)); } } catch {} diff --git a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs index 692f896c3..becb8515d 100644 --- a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -48,7 +48,7 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand(transaction)) using (var rd = cmd.ExecuteReader(@"SELECT ""B"".""Name"" FROM ""Fileset"" A, ""RemoteVolume"" B WHERE ""A"".""VolumeID"" = ""B"".""ID"" AND ""A"".""ID"" = ? ", id)) if (!rd.Read()) - throw new Exception(string.Format("No remote volume found for fileset with id {0}", id)); + throw new Exception($"No remote volume found for fileset with id {id}"); else return rd.ConvertValueToString(0); } @@ -95,7 +95,7 @@ namespace Duplicati.Library.Main.Database m_tablename = "TempDeletedFilesTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); using (var cmd = m_connection.CreateCommand(m_transaction)) - cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""FileID"" INTEGER PRIMARY KEY) ", m_tablename)); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (""FileID"" INTEGER PRIMARY KEY) ", m_tablename)); } public void ApplyFilter(Action filtercommand) @@ -117,8 +117,8 @@ namespace Duplicati.Library.Main.Database var filenamestable = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); using (var cmd = m_connection.CreateCommand(m_transaction)) { - cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL) ", filenamestable)); - cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"") VALUES (?)", filenamestable); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL) ", filenamestable)); + cmd.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" (""Path"") VALUES (?)", filenamestable); cmd.AddParameter(); foreach (var s in p) @@ -127,8 +127,8 @@ namespace Duplicati.Library.Main.Database cmd.ExecuteNonQuery(); } - cmd.ExecuteNonQuery(string.Format(@"INSERT INTO ""{0}"" (""FileID"") SELECT DISTINCT ""A"".""FileID"" FROM ""FilesetEntry"" A, ""File"" B WHERE ""A"".""FilesetID"" = ? AND ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""Path"" IN ""{1}""", m_tablename, filenamestable), ParentID); - cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", filenamestable)); + cmd.ExecuteNonQuery(FormatInvariant(@"INSERT INTO ""{0}"" (""FileID"") SELECT DISTINCT ""A"".""FileID"" FROM ""FilesetEntry"" A, ""File"" B WHERE ""A"".""FilesetID"" = ? AND ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""Path"" IN ""{1}""", m_tablename, filenamestable), ParentID); + cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", filenamestable)); } } else @@ -138,7 +138,7 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand(m_transaction)) using (var cmd2 = m_connection.CreateCommand(m_transaction)) { - cmd2.CommandText = string.Format(@"INSERT INTO ""{0}"" (""FileID"") VALUES (?)", m_tablename); + cmd2.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" (""FileID"") VALUES (?)", m_tablename); cmd2.AddParameters(1); using (var rd = cmd.ExecuteReader(@"SELECT ""B"".""Path"", ""A"".""FileID"" FROM ""FilesetEntry"" A, ""File"" B WHERE ""A"".""FilesetID"" = ? AND ""A"".""FileID"" = ""B"".""ID"" ", ParentID)) @@ -161,11 +161,11 @@ namespace Duplicati.Library.Main.Database { using (var cmd = m_connection.CreateCommand(m_transaction)) { - RemovedFileCount = cmd.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM ""{0}""", m_tablename), 0); - RemovedFileSize = cmd.ExecuteScalarInt64(string.Format(@"SELECT SUM(""C"".""Length"") FROM ""{0}"" A, ""FileLookup"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" ", m_tablename), 0); - var filesetcount = cmd.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetID"" = " + ParentID), 0); + RemovedFileCount = cmd.ExecuteScalarInt64(FormatInvariant(@"SELECT COUNT(*) FROM ""{0}""", m_tablename), 0); + RemovedFileSize = cmd.ExecuteScalarInt64(FormatInvariant(@"SELECT SUM(""C"".""Length"") FROM ""{0}"" A, ""FileLookup"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" ", m_tablename), 0); + var filesetcount = cmd.ExecuteScalarInt64(FormatInvariant(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetID"" = " + ParentID), 0); if (filesetcount == RemovedFileCount) - throw new Duplicati.Library.Interface.UserInformationException(string.Format("Refusing to purge {0} files from fileset with ID {1}, as that would remove the entire fileset.\nTo delete a fileset, use the \"delete\" command.", RemovedFileCount, ParentID), "PurgeWouldRemoveEntireFileset"); + throw new Duplicati.Library.Interface.UserInformationException($"Refusing to purge {RemovedFileCount} files from fileset with ID {ParentID}, as that would remove the entire fileset.\nTo delete a fileset, use the \"delete\" command.", "PurgeWouldRemoveEntireFileset"); } } @@ -176,7 +176,7 @@ namespace Duplicati.Library.Main.Database m_parentdb.UpdateFullBackupStateInFileset(filesetid, isFullBackup); using (var cmd = m_connection.CreateCommand(m_transaction)) - cmd.ExecuteNonQuery(string.Format(@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") SELECT ?, ""FileID"", ""LastModified"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = ? AND ""FileID"" NOT IN ""{0}"" ", m_tablename), filesetid, ParentID); + cmd.ExecuteNonQuery(FormatInvariant(@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") SELECT ?, ""FileID"", ""LastModified"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = ? AND ""FileID"" NOT IN ""{0}"" ", m_tablename), filesetid, ParentID); return new Tuple(remotevolid, filesetid); } @@ -184,7 +184,7 @@ namespace Duplicati.Library.Main.Database public IEnumerable> ListAllDeletedFiles() { using (var cmd = m_connection.CreateCommand(m_transaction)) - using (var rd = cmd.ExecuteReader(string.Format(@"SELECT ""B"".""Path"", ""C"".""Length"" FROM ""{0}"" A, ""File"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" ", m_tablename))) + using (var rd = cmd.ExecuteReader(FormatInvariant(@"SELECT ""B"".""Path"", ""C"".""Length"" FROM ""{0}"" A, ""File"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" ", m_tablename))) while (rd.Read()) yield return new KeyValuePair(rd.ConvertValueToString(0), rd.ConvertValueToInt64(1)); } diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index dc6a890b1..a7e91308a 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -123,12 +123,12 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) { - cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""BlockListHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""Index"" INTEGER NOT NULL)", m_tempblocklist)); - cmd.ExecuteNonQuery(string.Format(@"CREATE INDEX ""Index_{0}"" ON ""{0}"" (""BlockListHash"");", m_tempblocklist)); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (""BlockListHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""Index"" INTEGER NOT NULL)", m_tempblocklist)); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE INDEX ""Index_{0}"" ON ""{0}"" (""BlockListHash"");", m_tempblocklist)); - cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""FileHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""BlockSize"" INTEGER NOT NULL)", m_tempsmalllist)); - cmd.ExecuteNonQuery(string.Format(@"CREATE UNIQUE INDEX ""Index_File_{0}"" ON ""{0}"" (""FileHash"", ""BlockSize"");", m_tempsmalllist)); - cmd.ExecuteNonQuery(string.Format(@"CREATE UNIQUE INDEX ""Index_Block_{0}"" ON ""{0}"" (""BlockHash"", ""BlockSize"");", m_tempsmalllist)); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (""FileHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""BlockSize"" INTEGER NOT NULL)", m_tempsmalllist)); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE UNIQUE INDEX ""Index_File_{0}"" ON ""{0}"" (""FileHash"", ""BlockSize"");", m_tempsmalllist)); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE UNIQUE INDEX ""Index_Block_{0}"" ON ""{0}"" (""BlockHash"", ""BlockSize"");", m_tempsmalllist)); } m_insertFileCommand = m_connection.CreateCommand(); @@ -165,10 +165,10 @@ namespace Duplicati.Library.Main.Database m_updateBlockVolumeCommand.CommandText = @"UPDATE ""Block"" SET ""VolumeID"" = ? WHERE ""Hash"" = ? AND ""Size"" = ?"; m_updateBlockVolumeCommand.AddParameters(3); - m_insertTempBlockListHash.CommandText = string.Format(@"INSERT INTO ""{0}"" (""BlocklistHash"", ""BlockHash"", ""Index"") VALUES (?,?,?) ", m_tempblocklist); + m_insertTempBlockListHash.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" (""BlocklistHash"", ""BlockHash"", ""Index"") VALUES (?,?,?) ", m_tempblocklist); m_insertTempBlockListHash.AddParameters(3); - m_insertSmallBlockset.CommandText = string.Format(@"INSERT OR IGNORE INTO ""{0}"" (""FileHash"", ""BlockHash"", ""BlockSize"") VALUES (?,?,?) ", m_tempsmalllist); + m_insertSmallBlockset.CommandText = FormatInvariant(@"INSERT OR IGNORE INTO ""{0}"" (""FileHash"", ""BlockHash"", ""BlockSize"") VALUES (?,?,?) ", m_tempsmalllist); m_insertSmallBlockset.AddParameters(3); m_findBlocksetCommand.CommandText = @"SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" = ? AND ""FullHash"" = ? "; @@ -180,7 +180,7 @@ namespace Duplicati.Library.Main.Database m_findFilesetCommand.CommandText = @"SELECT ""ID"" FROM ""FileLookup"" WHERE ""PrefixID"" = ? AND ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ? "; m_findFilesetCommand.AddParameters(4); - m_findTempBlockListHashCommand.CommandText = string.Format(@"SELECT DISTINCT ""BlockListHash"" FROM ""{0}"" WHERE ""BlockListHash"" = ? ", m_tempblocklist); + m_findTempBlockListHashCommand.CommandText = FormatInvariant(@"SELECT DISTINCT ""BlockListHash"" FROM ""{0}"" WHERE ""BlockListHash"" = ? ", m_tempblocklist); m_findTempBlockListHashCommand.AddParameters(1); m_findHashBlockCommand.CommandText = @"SELECT ""VolumeID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ? "; @@ -201,9 +201,9 @@ namespace Duplicati.Library.Main.Database //Update all small blocklists and matching blocks - var selectSmallBlocks = string.Format(@"SELECT ""BlockHash"", ""BlockSize"" FROM ""{0}""", m_tempsmalllist); + var selectSmallBlocks = FormatInvariant(@"SELECT ""BlockHash"", ""BlockSize"" FROM ""{0}""", m_tempsmalllist); - var selectBlockHashes = string.Format( + var selectBlockHashes = FormatInvariant( @"SELECT ""BlockHash"" AS ""FullHash"", ""BlockSize"" AS ""Length"" FROM ( " + SELECT_BLOCKLIST_ENTRIES + @" )", @@ -215,7 +215,7 @@ namespace Duplicati.Library.Main.Database var selectAllBlocks = @"SELECT DISTINCT ""FullHash"", ""Length"" FROM (" + selectBlockHashes + " UNION " + selectSmallBlocks + " )"; - var selectNewBlocks = string.Format( + var selectNewBlocks = FormatInvariant( @"SELECT ""FullHash"" AS ""Hash"", ""Length"" AS ""Size"", -1 AS ""VolumeID"" " + @" FROM (SELECT ""A"".""FullHash"", ""A"".""Length"", CASE WHEN ""B"".""Hash"" IS NULL THEN '' ELSE ""B"".""Hash"" END AS ""Hash"", CASE WHEN ""B"".""Size"" is NULL THEN -1 ELSE ""B"".""Size"" END AS ""Size"" FROM ({0}) A" + @" LEFT OUTER JOIN ""Block"" B ON ""B"".""Hash"" = ""A"".""FullHash"" AND ""B"".""Size"" = ""A"".""Length"" )" + @@ -230,7 +230,7 @@ namespace Duplicati.Library.Main.Database // Insert all known blocks into block table with volumeid = -1 cmd.ExecuteNonQuery(insertBlocksCommand); - var selectBlocklistBlocksetEntries = string.Format( + var selectBlocklistBlocksetEntries = FormatInvariant( @"SELECT ""E"".""BlocksetID"" AS ""BlocksetID"", ""D"".""FullIndex"" AS ""Index"", ""F"".""ID"" AS ""BlockID"" FROM ( " + SELECT_BLOCKLIST_ENTRIES + @") D, ""BlocklistHash"" E, ""Block"" F, ""Block"" G WHERE ""D"".""BlocksetID"" = ""E"".""BlocksetID"" AND ""D"".""BlocklistHash"" = ""E"".""Hash"" AND ""D"".""BlocklistSize"" = ""G"".""Size"" AND ""D"".""BlocklistHash"" = ""G"".""Hash"" AND ""D"".""Blockhash"" = ""F"".""Hash"" AND ""D"".""BlockSize"" = ""F"".""Size"" ", @@ -245,7 +245,7 @@ namespace Duplicati.Library.Main.Database blocksize / hashsize ); - var selectBlocksetEntries = string.Format( + var selectBlocksetEntries = FormatInvariant( @"SELECT ""Blockset"".""ID"" AS ""BlocksetID"", 0 AS ""Index"", ""Block"".""ID"" AS ""BlockID"" FROM ""Blockset"", ""Block"", ""{1}"" S WHERE ""Blockset"".""Fullhash"" = ""S"".""FileHash"" AND ""S"".""BlockHash"" = ""Block"".""Hash"" AND ""S"".""BlockSize"" = ""Block"".""Size"" AND ""Blockset"".""Length"" = ""S"".""BlockSize"" AND ""Blockset"".""Length"" <= {0} ", blocksize, m_tempsmalllist @@ -274,8 +274,8 @@ namespace Duplicati.Library.Main.Database using (var fixcmd = m_connection.CreateCommand()) { - fixcmd.ExecuteNonQuery(string.Format(@"CREATE TABLE ""{0}-Failure"" AS SELECT * FROM ""{0}"" ", m_tempblocklist)); - fixcmd.ExecuteNonQuery(string.Format(@"CREATE TABLE ""{0}-Failure"" AS SELECT * FROM ""{0}"" ", m_tempsmalllist)); + fixcmd.ExecuteNonQuery(FormatInvariant(@"CREATE TABLE ""{0}-Failure"" AS SELECT * FROM ""{0}"" ", m_tempblocklist)); + fixcmd.ExecuteNonQuery(FormatInvariant(@"CREATE TABLE ""{0}-Failure"" AS SELECT * FROM ""{0}"" ", m_tempsmalllist)); } throw new Exception("The recreate failed, please create a bug-report from this database and send it to the developers for further analysis"); @@ -306,7 +306,7 @@ namespace Duplicati.Library.Main.Database { cmd.Transaction = transaction; - var insertBlocksCommand = string.Format( + var insertBlocksCommand = FormatInvariant( @"INSERT INTO BLOCK (Hash, Size, VolumeID) " + @"SELECT DISTINCT BlockHash AS Hash, BlockSize AS Size, -1 AS VolumeID FROM " + @"(" + @@ -326,7 +326,7 @@ namespace Duplicati.Library.Main.Database ); if (!hashOnly) { - insertBlocksCommand += string.Format( + insertBlocksCommand += FormatInvariant( @" UNION " + @"" + @"SELECT TS.BlockHash, TS.BlockSize FROM " + @@ -339,7 +339,7 @@ namespace Duplicati.Library.Main.Database ); } - var insertBlocksetEntriesCommand = string.Format( + var insertBlocksetEntriesCommand = FormatInvariant( @"INSERT INTO BlocksetEntry (BlocksetID, ""Index"", BlockID) " + @"SELECT DISTINCT BH.blocksetid, (BH.""Index"" * {0})+TBL.""Index"" as FullIndex, BK.ID AS BlockID " + @"FROM {1} TBL " + @@ -353,7 +353,7 @@ namespace Duplicati.Library.Main.Database ); if (!hashOnly) { - insertBlocksetEntriesCommand += string.Format( + insertBlocksetEntriesCommand += FormatInvariant( @" UNION " + @"SELECT BS.ID AS BlocksetID, 0 AS ""Index"", BL.ID AS BlockID " + @"FROM {1} TS " + @@ -383,8 +383,8 @@ namespace Duplicati.Library.Main.Database using (var fixcmd = m_connection.CreateCommand()) { - fixcmd.ExecuteNonQuery(string.Format(@"CREATE TABLE ""{0}_Failure"" AS SELECT * FROM ""{0}"" ", m_tempblocklist)); - fixcmd.ExecuteNonQuery(string.Format(@"CREATE TABLE ""{0}_Failure"" AS SELECT * FROM ""{0}"" ", m_tempsmalllist)); + fixcmd.ExecuteNonQuery(FormatInvariant(@"CREATE TABLE ""{0}_Failure"" AS SELECT * FROM ""{0}"" ", m_tempblocklist)); + fixcmd.ExecuteNonQuery(FormatInvariant(@"CREATE TABLE ""{0}_Failure"" AS SELECT * FROM ""{0}"" ", m_tempsmalllist)); } throw new Exception("The recreate failed, please create a bug-report from this database and send it to the developers for further analysis"); @@ -531,7 +531,7 @@ namespace Duplicati.Library.Main.Database m_updateBlockVolumeCommand.SetParameterValue(2, size); var c = m_updateBlockVolumeCommand.ExecuteNonQuery(); if (c != 1) - throw new Exception(string.Format("Failed to update table, found {0} entries for key {1} with size {2}", c, hash, size)); + throw new Exception(FormatInvariant("Failed to update table, found {0} entries for key {1} with size {2}", c, hash, size)); return true; } @@ -600,7 +600,7 @@ namespace Duplicati.Library.Main.Database var selectCommand = @"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"", ""RemoteVolume"".""ID"" FROM ""RemoteVolume"""; var missingBlocklistEntries = - string.Format( + FormatInvariant( @"SELECT ""BlocklistHash"".""Hash"" FROM ""BlocklistHash"" LEFT OUTER JOIN ""BlocksetEntry"" ON ""BlocksetEntry"".""Index"" = (""BlocklistHash"".""Index"" * {0}) AND ""BlocksetEntry"".""BlocksetID"" = ""BlocklistHash"".""BlocksetID"" WHERE ""BlocksetEntry"".""BlocksetID"" IS NULL", blocksize / hashsize ); @@ -608,13 +608,13 @@ namespace Duplicati.Library.Main.Database var missingBlockInfo = @"SELECT ""VolumeID"" FROM ""Block"" WHERE ""VolumeID"" < 0 AND SIZE > 0"; - var missingBlocklistVolumes = string.Format( + var missingBlocklistVolumes = FormatInvariant( @"SELECT ""VolumeID"" FROM ""Block"", (" + missingBlocklistEntries + @") A WHERE ""A"".""Hash"" = ""Block"".""Hash"" " ); - var countMissingInformation = string.Format( + var countMissingInformation = FormatInvariant( @"SELECT COUNT(*) FROM (SELECT DISTINCT ""VolumeID"" FROM ({0} UNION {1}))", missingBlockInfo, missingBlocklistVolumes); @@ -623,7 +623,7 @@ namespace Duplicati.Library.Main.Database { // On the first pass, we select all the volumes we know we need, // which may be an empty list - cmd.CommandText = string.Format(selectCommand + @" WHERE ""ID"" IN ({0})", missingBlocklistVolumes); + cmd.CommandText = FormatInvariant(selectCommand + @" WHERE ""ID"" IN ({0})", missingBlocklistVolumes); // Reset the list m_proccessedVolumes = new Dictionary(); @@ -642,7 +642,7 @@ namespace Duplicati.Library.Main.Database var mentionedVolumes = @"SELECT DISTINCT ""VolumeID"" FROM ""Block"" "; - cmd.CommandText = string.Format(selectCommand + @" WHERE ""ID"" NOT IN ({0}) AND ""Type"" = ? ", mentionedVolumes); + cmd.CommandText = FormatInvariant(selectCommand + @" WHERE ""ID"" NOT IN ({0}) AND ""Type"" = ? ", mentionedVolumes); cmd.AddParameter(RemoteVolumeType.Blocks.ToString()); } else @@ -689,7 +689,7 @@ namespace Duplicati.Library.Main.Database // existing ones (as recorded in the DuplicateBlock table) // The second part removes references to the non-present remote files, // and marks the index files that pointed to them, such that they will be removed later on - var sql = $@" + var sql = FormatInvariant($@" CREATE TEMPORARY TABLE ""{tablename}"" AS SELECT ""A"".""ID"" AS ""BlockID"", ""A"".""VolumeID"" AS ""SourceVolumeID"", ""A"".""State"" AS ""SourceVolumeState"", ""B"".""VolumeID"" AS ""TargetVolumeID"", ""B"".""State"" AS ""TargetVolumeState"" @@ -728,12 +728,12 @@ DROP TABLE ""{tablename}""; DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND ""State"" = '{RemoteVolumeState.Temporary}' AND ""ID"" NOT IN (SELECT DISTINCT ""VolumeID"" FROM ""Block"")); DELETE FROM ""DuplicateBlock"" WHERE ""VolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND ""State"" = '{RemoteVolumeState.Temporary}' AND ""ID"" NOT IN (SELECT DISTINCT ""VolumeID"" FROM ""Block"")); DELETE FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND ""State"" = '{RemoteVolumeState.Temporary}' AND ""ID"" NOT IN (SELECT DISTINCT ""VolumeID"" FROM ""Block""); -"; +"); // We could delete these, but we don't have to, so we keep them around until the next compact is done // UPDATE ""RemoteVolume"" SET ""State"" = ""{3}"" WHERE ""Type"" = ""{5}"" AND ""ID"" NOT IN (SELECT ""IndexVolumeID"" FROM ""IndexBlockLink""); - var countsql = $@"SELECT COUNT(*) FROM ""RemoteVolume"" WHERE ""State"" = '{RemoteVolumeState.Temporary}' AND ""Type"" = '{RemoteVolumeType.Blocks}' "; + var countsql = FormatInvariant($@"SELECT COUNT(*) FROM ""RemoteVolume"" WHERE ""State"" = '{RemoteVolumeState.Temporary}' AND ""Type"" = '{RemoteVolumeType.Blocks}' "); using (var cmd = m_connection.CreateCommand()) { @@ -764,15 +764,15 @@ DELETE FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND "" using (var cmd = m_connection.CreateCommand(tr.Parent)) { // 1. Select blocks not used by any file and not as a blocklist into temporary table - cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" AS SELECT ""Block"".""ID"", ""Block"".""Hash"", ""Block"".""Size"", ""Block"".""VolumeID"" FROM ""Block"" WHERE ""Block"".""ID"" NOT IN (SELECT ""BlocksetEntry"".""BlockID"" FROM ""BlocksetEntry"") AND ""Block"".""Hash"" NOT IN (SELECT ""BlocklistHash"".""Hash"" FROM ""BlocklistHash"")", tmptablename)); // 2. Insert blocks into DeletedBlock table - cmd.ExecuteNonQuery(string.Format(@"INSERT INTO ""DeletedBlock"" (""Hash"", ""Size"", ""VolumeID"") SELECT ""Hash"", ""Size"", ""VolumeID"" FROM ""{0}""", tmptablename)); + cmd.ExecuteNonQuery(FormatInvariant(@"INSERT INTO ""DeletedBlock"" (""Hash"", ""Size"", ""VolumeID"") SELECT ""Hash"", ""Size"", ""VolumeID"" FROM ""{0}""", tmptablename)); // 3. Remove blocks from Block table - cmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""Block"" WHERE ""ID"" IN (SELECT ""ID"" FROM ""{0}"")", tmptablename)); - cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}""", tmptablename)); + cmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""Block"" WHERE ""ID"" IN (SELECT ""ID"" FROM ""{0}"")", tmptablename)); + cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}""", tmptablename)); tr.Commit(); } @@ -786,7 +786,7 @@ DELETE FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND "" if (m_tempblocklist != null) try { - cmd.CommandText = string.Format(@"DROP TABLE IF EXISTS ""{0}""", m_tempblocklist); + cmd.CommandText = FormatInvariant(@"DROP TABLE IF EXISTS ""{0}""", m_tempblocklist); cmd.ExecuteNonQuery(); } catch (Exception ex) @@ -801,7 +801,7 @@ DELETE FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND "" if (m_tempsmalllist != null) try { - cmd.CommandText = string.Format(@"DROP TABLE IF EXISTS ""{0}""", m_tempsmalllist); + cmd.CommandText = FormatInvariant(@"DROP TABLE IF EXISTS ""{0}""", m_tempsmalllist); cmd.ExecuteNonQuery(); } catch (Exception ex) diff --git a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs index 600cdc4a0..aadd51747 100644 --- a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs @@ -46,7 +46,7 @@ namespace Duplicati.Library.Main.Database { var filesetid = cmd.ExecuteScalarInt64(@"SELECT ""Fileset"".""ID"" FROM ""Fileset"",""RemoteVolume"" WHERE ""Fileset"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""RemoteVolume"".""Name"" = ?", -1, filelist); if (filesetid == -1) - throw new Exception(string.Format("No such remote file: {0}", filelist)); + throw new Exception(FormatInvariant("No such remote file: {0}", filelist)); return filesetid; } @@ -156,19 +156,19 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) { cmd.Transaction = m_transaction.Parent; - cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Restored"" INTEGER NOT NULL) ", tablename)); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Restored"" INTEGER NOT NULL) ", tablename)); m_tablename = tablename; - var blockCount = cmd.ExecuteNonQuery(string.Format(@"INSERT INTO ""{0}"" (""Hash"", ""Size"", ""Restored"") SELECT DISTINCT ""Block"".""Hash"", ""Block"".""Size"", 0 AS ""Restored"" FROM ""Block"",""Remotevolume"" WHERE ""Block"".""VolumeID"" = ""Remotevolume"".""ID"" AND ""Remotevolume"".""Name"" = ? ", m_tablename), volumename); + var blockCount = cmd.ExecuteNonQuery(FormatInvariant(@"INSERT INTO ""{0}"" (""Hash"", ""Size"", ""Restored"") SELECT DISTINCT ""Block"".""Hash"", ""Block"".""Size"", 0 AS ""Restored"" FROM ""Block"",""Remotevolume"" WHERE ""Block"".""VolumeID"" = ""Remotevolume"".""ID"" AND ""Remotevolume"".""Name"" = ? ", m_tablename), volumename); if (blockCount == 0) - throw new Exception(string.Format("Unexpected empty block volume: {0}", volumename)); + throw new Exception($"Unexpected empty block volume: {0}"); - cmd.ExecuteNonQuery(string.Format(@"CREATE UNIQUE INDEX ""{0}-Ix"" ON ""{0}"" (""Hash"", ""Size"", ""Restored"")", tablename)); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE UNIQUE INDEX ""{0}-Ix"" ON ""{0}"" (""Hash"", ""Size"", ""Restored"")", tablename)); } m_insertCommand = m_connection.CreateCommand(); m_insertCommand.Transaction = m_transaction.Parent; - m_insertCommand.CommandText = string.Format(@"UPDATE ""{0}"" SET ""Restored"" = ? WHERE ""Hash"" = ? AND ""Size"" = ? AND ""Restored"" = ? ", tablename); + m_insertCommand.CommandText = FormatInvariant(@"UPDATE ""{0}"" SET ""Restored"" = ? WHERE ""Hash"" = ? AND ""Size"" = ? AND ""Restored"" = ? ", tablename); m_insertCommand.AddParameters(4); } @@ -184,7 +184,7 @@ namespace Duplicati.Library.Main.Database public IEnumerable GetSourceFilesWithBlocks(long blocksize) { using (var cmd = m_connection.CreateCommand(m_transaction.Parent)) - using (var rd = cmd.ExecuteReader(string.Format(@"SELECT DISTINCT ""{0}"".""Hash"", ""{0}"".""Size"", ""File"".""Path"", ""BlocksetEntry"".""Index"" * {1} FROM ""{0}"", ""Block"", ""BlocksetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""{0}"".""Hash"" = ""Block"".""Hash"" AND ""{0}"".""Size"" = ""Block"".""Size"" AND ""{0}"".""Restored"" = ? ", m_tablename, blocksize), 0)) + using (var rd = cmd.ExecuteReader(FormatInvariant(@"SELECT DISTINCT ""{0}"".""Hash"", ""{0}"".""Size"", ""File"".""Path"", ""BlocksetEntry"".""Index"" * {1} FROM ""{0}"", ""Block"", ""BlocksetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""{0}"".""Hash"" = ""Block"".""Hash"" AND ""{0}"".""Size"" = ""Block"".""Size"" AND ""{0}"".""Restored"" = ? ", m_tablename, blocksize), 0)) if (rd.Read()) { var bs = new BlockWithSources(rd); @@ -196,7 +196,7 @@ namespace Duplicati.Library.Main.Database public IEnumerable> GetMissingBlocks() { using (var cmd = m_connection.CreateCommand(m_transaction.Parent)) - foreach (var rd in cmd.ExecuteReaderEnumerable(string.Format(@"SELECT ""{0}"".""Hash"", ""{0}"".""Size"" FROM ""{0}"" WHERE ""{0}"".""Restored"" = ? ", m_tablename), 0)) + foreach (var rd in cmd.ExecuteReaderEnumerable(FormatInvariant(@"SELECT ""{0}"".""Hash"", ""{0}"".""Size"" FROM ""{0}"" WHERE ""{0}"".""Restored"" = ? ", m_tablename), 0)) yield return new KeyValuePair(rd.ConvertValueToString(0), rd.ConvertValueToInt64(1)); } @@ -208,14 +208,14 @@ namespace Duplicati.Library.Main.Database var cmdtxt = @"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""FilesetEntry"", ""Fileset"" WHERE ""RemoteVolume"".""ID"" = ""Fileset"".""VolumeID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""RemoteVolume"".""Type"" = ? AND ""FilesetEntry"".""FileID"" IN (SELECT DISTINCT ""ID"" FROM ( " + blocks + " UNION " + blocklists + " ))"; using (var cmd = m_connection.CreateCommand(m_transaction.Parent)) - foreach (var rd in cmd.ExecuteReaderEnumerable(string.Format(cmdtxt, m_tablename), RemoteVolumeType.Files.ToString())) + foreach (var rd in cmd.ExecuteReaderEnumerable(FormatInvariant(cmdtxt, m_tablename), RemoteVolumeType.Files.ToString())) yield return new RemoteVolume(rd.GetString(0), rd.ConvertValueToString(1), rd.ConvertValueToInt64(2)); } public IEnumerable GetMissingBlockSources() { using (var cmd = m_connection.CreateCommand(m_transaction.Parent)) - foreach (var rd in cmd.ExecuteReaderEnumerable(string.Format(@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""Block"", ""{0}"" WHERE ""Block"".""Hash"" = ""{0}"".""Hash"" AND ""Block"".""Size"" = ""{0}"".""Size"" AND ""Block"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""Remotevolume"".""Name"" != ? ", m_tablename), m_volumename)) + foreach (var rd in cmd.ExecuteReaderEnumerable(FormatInvariant(@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""Block"", ""{0}"" WHERE ""Block"".""Hash"" = ""{0}"".""Hash"" AND ""Block"".""Size"" = ""{0}"".""Size"" AND ""Block"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""Remotevolume"".""Name"" != ? ", m_tablename), m_volumename)) yield return new RemoteVolume(rd.GetString(0), rd.ConvertValueToString(1), rd.ConvertValueToInt64(2)); } @@ -226,7 +226,7 @@ namespace Duplicati.Library.Main.Database try { using (var cmd = m_connection.CreateCommand(m_transaction.Parent)) - cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", m_transaction)); + cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", m_transaction)); } catch { } finally { m_tablename = null; } @@ -265,36 +265,36 @@ namespace Duplicati.Library.Main.Database var tablename = "TmpFile-" + Guid.NewGuid().ToString("N"); - cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS SELECT * FROM ""File""", tablename)); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" AS SELECT * FROM ""File""", tablename)); var sql = @"SELECT ""A"".""ID"", ""B"".""BlocksetID"" FROM (SELECT MIN(""ID"") AS ""ID"", COUNT(""ID"") AS ""Duplicates"" FROM ""Metadataset"" GROUP BY ""BlocksetID"") ""A"", ""Metadataset"" ""B"" WHERE ""A"".""Duplicates"" > 1 AND ""A"".""ID"" = ""B"".""ID"""; using (var c2 = m_connection.CreateCommand(tr)) { - c2.CommandText = string.Format(@"UPDATE ""{0}"" SET ""MetadataID"" = ? WHERE ""MetadataID"" IN (SELECT ""ID"" FROM ""Metadataset"" WHERE ""BlocksetID"" = ?)", tablename); + c2.CommandText = FormatInvariant(@"UPDATE ""{0}"" SET ""MetadataID"" = ? WHERE ""MetadataID"" IN (SELECT ""ID"" FROM ""Metadataset"" WHERE ""BlocksetID"" = ?)", tablename); c2.CommandText += @"; DELETE FROM ""Metadataset"" WHERE ""BlocksetID"" = ? AND ""ID"" != ?"; using (var rd = cmd.ExecuteReader(sql)) while (rd.Read()) c2.ExecuteNonQuery(null, rd.GetValue(0), rd.GetValue(1), rd.GetValue(1), rd.GetValue(0)); } - sql = string.Format(@"SELECT ""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""Entries"" FROM ( + sql = FormatInvariant(@"SELECT ""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""Entries"" FROM ( SELECT MIN(""ID"") AS ""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Entries"" FROM ""{0}"" GROUP BY ""Path"", ""BlocksetID"", ""MetadataID"") WHERE ""Entries"" > 1 ORDER BY ""ID""", tablename); using (var c2 = m_connection.CreateCommand()) { c2.Transaction = tr; - c2.CommandText = string.Format(@"UPDATE ""FilesetEntry"" SET ""FileID"" = ? WHERE ""FileID"" IN (SELECT ""ID"" FROM ""{0}"" WHERE ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ?)", tablename); - c2.CommandText += string.Format(@"; DELETE FROM ""{0}"" WHERE ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ? AND ""ID"" != ?", tablename); + c2.CommandText = FormatInvariant(@"UPDATE ""FilesetEntry"" SET ""FileID"" = ? WHERE ""FileID"" IN (SELECT ""ID"" FROM ""{0}"" WHERE ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ?)", tablename); + c2.CommandText += FormatInvariant(@"; DELETE FROM ""{0}"" WHERE ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ? AND ""ID"" != ?", tablename); foreach (var rd in cmd.ExecuteReaderEnumerable(sql)) c2.ExecuteNonQuery(null, rd.GetValue(0), rd.GetValue(1), rd.GetValue(2), rd.GetValue(3), rd.GetValue(1), rd.GetValue(2), rd.GetValue(3), rd.GetValue(0)); } - cmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""FileLookup"" WHERE ""ID"" NOT IN (SELECT ""ID"" FROM ""{0}"") ", tablename)); - cmd.ExecuteNonQuery(string.Format(@"CREATE INDEX ""{0}-Ix"" ON ""{0}"" (""ID"", ""MetadataID"")", tablename)); - cmd.ExecuteNonQuery(string.Format(@"UPDATE ""FileLookup"" SET ""MetadataID"" = (SELECT ""MetadataID"" FROM ""{0}"" A WHERE ""A"".""ID"" = ""FileLookup"".""ID"") ", tablename)); - cmd.ExecuteNonQuery(string.Format(@"DROP TABLE ""{0}"" ", tablename)); + cmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""FileLookup"" WHERE ""ID"" NOT IN (SELECT ""ID"" FROM ""{0}"") ", tablename)); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE INDEX ""{0}-Ix"" ON ""{0}"" (""ID"", ""MetadataID"")", tablename)); + cmd.ExecuteNonQuery(FormatInvariant(@"UPDATE ""FileLookup"" SET ""MetadataID"" = (SELECT ""MetadataID"" FROM ""{0}"" A WHERE ""A"".""ID"" = ""FileLookup"".""ID"") ", tablename)); + cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE ""{0}"" ", tablename)); cmd.CommandText = sql_count; x = cmd.ExecuteScalarInt64(0); @@ -353,7 +353,7 @@ namespace Duplicati.Library.Main.Database { var hashsize = blockhasher.HashSize / 8; - var sql = string.Format(@"SELECT * FROM (SELECT ""N"".""BlocksetID"", ((""N"".""BlockCount"" + {0} - 1) / {0}) AS ""BlocklistHashCountExpected"", CASE WHEN ""G"".""BlocklistHashCount"" IS NULL THEN 0 ELSE ""G"".""BlocklistHashCount"" END AS ""BlocklistHashCountActual"" FROM (SELECT ""BlocksetID"", COUNT(*) AS ""BlockCount"" FROM ""BlocksetEntry"" GROUP BY ""BlocksetID"") ""N"" LEFT OUTER JOIN (SELECT ""BlocksetID"", COUNT(*) AS ""BlocklistHashCount"" FROM ""BlocklistHash"" GROUP BY ""BlocksetID"") ""G"" ON ""N"".""BlocksetID"" = ""G"".""BlocksetID"" WHERE ""N"".""BlockCount"" > 1) WHERE ""BlocklistHashCountExpected"" != ""BlocklistHashCountActual""", blocksize / hashsize); + var sql = FormatInvariant(@"SELECT * FROM (SELECT ""N"".""BlocksetID"", ((""N"".""BlockCount"" + {0} - 1) / {0}) AS ""BlocklistHashCountExpected"", CASE WHEN ""G"".""BlocklistHashCount"" IS NULL THEN 0 ELSE ""G"".""BlocklistHashCount"" END AS ""BlocklistHashCountActual"" FROM (SELECT ""BlocksetID"", COUNT(*) AS ""BlockCount"" FROM ""BlocksetEntry"" GROUP BY ""BlocksetID"") ""N"" LEFT OUTER JOIN (SELECT ""BlocksetID"", COUNT(*) AS ""BlocklistHashCount"" FROM ""BlocklistHash"" GROUP BY ""BlocksetID"") ""G"" ON ""N"".""BlocksetID"" = ""G"".""BlocksetID"" WHERE ""N"".""BlockCount"" > 1) WHERE ""BlocklistHashCountExpected"" != ""BlocklistHashCountActual""", blocksize / hashsize); var countsql = @"SELECT COUNT(*) FROM (" + sql + @")"; var itemswithnoblocklisthash = cmd.ExecuteScalarInt64(countsql, 0); @@ -391,12 +391,12 @@ namespace Duplicati.Library.Main.Database { var c = c5.ExecuteScalarInt64(null, -1, blkey, blocklistoffset, RemoteVolumeType.Blocks.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()); if (c <= 0) - throw new Exception(string.Format("Missing block for blocklisthash: {0}", blkey)); + throw new Exception($"Missing block for blocklisthash: {blkey}"); else { var rc = c6.ExecuteNonQuery(null, c, c); if (rc != 2) - throw new Exception(string.Format("Unexpected update count: {0}", rc)); + throw new Exception($"Unexpected update count: {rc}"); } } @@ -420,12 +420,12 @@ namespace Duplicati.Library.Main.Database { var c = c5.ExecuteScalarInt64(null, -1, blkeyfinal, blocklistoffset, RemoteVolumeType.Blocks.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()); if (c == 0) - throw new Exception(string.Format("Missing block for blocklisthash: {0}", blkeyfinal)); + throw new Exception($"Missing block for blocklisthash: {blkeyfinal}"); else { var rc = c6.ExecuteNonQuery(null, c, c); if (rc != 2) - throw new Exception(string.Format("Unexpected update count: {0}", rc)); + throw new Exception($"Unexpected update count: {rc}"); } } @@ -438,7 +438,7 @@ namespace Duplicati.Library.Main.Database itemswithnoblocklisthash = cmd.ExecuteScalarInt64(countsql, 0); if (itemswithnoblocklisthash != 0) - throw new Duplicati.Library.Interface.UserInformationException(string.Format("Failed to repair, after repair {0} blocklisthashes were missing", itemswithnoblocklisthash), "MissingBlocklistHashesRepairFailed"); + throw new Duplicati.Library.Interface.UserInformationException($"Failed to repair, after repair {itemswithnoblocklisthash} blocklisthashes were missing", "MissingBlocklistHashesRepairFailed"); Logging.Log.WriteInformationMessage(LOGTAG, "MissingBlocklisthashesRepaired", "Missing blocklisthashes repaired succesfully"); tr.Commit(); @@ -471,7 +471,7 @@ namespace Duplicati.Library.Main.Database var expected = rd.GetInt32(2) - 1; var actual = c2.ExecuteNonQuery(null, rd.GetValue(0), rd.GetValue(1), expected); if (actual != expected) - throw new Exception(string.Format("Unexpected number of results after fix, got: {0}, expected: {1}", actual, expected)); + throw new Exception($"Unexpected number of results after fix, got: {actual}, expected: {expected}"); } } @@ -483,7 +483,7 @@ namespace Duplicati.Library.Main.Database var real_count = cmd.ExecuteScalarInt64(@"SELECT Count(*) FROM ""BlocklistHash""", 0); if (real_count != unique_count) - throw new Duplicati.Library.Interface.UserInformationException(string.Format("Failed to repair, result should have been {0} blocklist hashes, but result was {1} blocklist hashes", unique_count, real_count), "DuplicateBlocklistHashesRepairFailed"); + throw new Duplicati.Library.Interface.UserInformationException($"Failed to repair, result should have been {unique_count} blocklist hashes, but result was {real_count} blocklist hashes", "DuplicateBlocklistHashesRepairFailed"); try { @@ -507,8 +507,8 @@ namespace Duplicati.Library.Main.Database { var tablename = "ProbeBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)", tablename)); - cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Hash"", ""Size"") VALUES (?, ?)", tablename); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)", tablename)); + cmd.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" (""Hash"", ""Size"") VALUES (?, ?)", tablename); cmd.AddParameters(2); foreach (var kp in blocks) @@ -519,12 +519,12 @@ namespace Duplicati.Library.Main.Database } var id = cmd.ExecuteScalarInt64(@"SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" = ?", -1, filename); - var aliens = cmd.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM (SELECT ""A"".""VolumeID"" FROM ""{0}"" B LEFT OUTER JOIN ""Block"" A ON ""A"".""Hash"" = ""B"".""Hash"" AND ""A"".""Size"" = ""B"".""Size"") WHERE ""VolumeID"" != ? ", tablename), 0, id); + var aliens = cmd.ExecuteScalarInt64(FormatInvariant(@"SELECT COUNT(*) FROM (SELECT ""A"".""VolumeID"" FROM ""{0}"" B LEFT OUTER JOIN ""Block"" A ON ""A"".""Hash"" = ""B"".""Hash"" AND ""A"".""Size"" = ""B"".""Size"") WHERE ""VolumeID"" != ? ", tablename), 0, id); - cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", tablename)); + cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", tablename)); if (aliens != 0) - throw new Exception(string.Format("Not all blocks were found in {0}", filename)); + throw new Exception($"Not all blocks were found in {filename}"); } } @@ -532,7 +532,7 @@ namespace Duplicati.Library.Main.Database { using (var cmd = m_connection.CreateCommand()) { - var query = string.Format(@" + var query = FormatInvariant(@" SELECT ""C"".""Hash"", ""C"".""Size"" @@ -572,13 +572,13 @@ ORDER BY foreach (var r in cmd.ExecuteReaderEnumerable(query, hash, length)) { if (!en.MoveNext()) - throw new Exception(string.Format("Too few entries in source blocklist with hash {0}", hash)); + throw new Exception($"Too few entries in source blocklist with hash {hash}"); if (en.Current != r.GetString(0)) - throw new Exception(string.Format("Mismatch in blocklist with hash {0}", hash)); + throw new Exception($"Mismatch in blocklist with hash {hash}"); } if (en.MoveNext()) - throw new Exception(string.Format("Too many source blocklist entries in {0}", hash)); + throw new Exception($"Too many source blocklist entries in {hash}"); } } } diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index 10aef6e8c..a1aca90e3 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -89,15 +89,15 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) { // How to handle METADATA? - cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", m_fileprogtable)); - cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (" + cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", m_fileprogtable)); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (" + @" ""FileId"" INTEGER PRIMARY KEY " + @", ""TotalBlocks"" INTEGER NOT NULL, ""TotalSize"" INTEGER NOT NULL " + @", ""BlocksRestored"" INTEGER NOT NULL, ""SizeRestored"" INTEGER NOT NULL " + @")", m_fileprogtable)); - cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", m_totalprogtable)); - cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (" + cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", m_totalprogtable)); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (" + @" ""TotalFiles"" INTEGER NOT NULL, ""TotalBlocks"" INTEGER NOT NULL, ""TotalSize"" INTEGER NOT NULL " + @", ""FilesFullyRestored"" INTEGER NOT NULL, ""FilesPartiallyRestored"" INTEGER NOT NULL " + @", ""BlocksRestored"" INTEGER NOT NULL, ""SizeRestored"" INTEGER NOT NULL " @@ -105,8 +105,8 @@ namespace Duplicati.Library.Main.Database if (createFilesNewlyDoneTracker) { - cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", m_filesnewlydonetable)); - cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (" + cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", m_filesnewlydonetable)); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (" + @" ""ID"" INTEGER PRIMARY KEY " + @")", m_filesnewlydonetable)); } @@ -118,7 +118,7 @@ namespace Duplicati.Library.Main.Database // Note: FilesNewlyDone is NOT initialized, as in initialization nothing is really new. string sql; - sql = string.Format( + sql = FormatInvariant( @" INSERT INTO ""{0}"" (""FileId"", ""TotalBlocks"", ""TotalSize"", ""BlocksRestored"", ""SizeRestored"") " + @" SELECT ""F"".""ID"", IFNULL(COUNT(""B"".""ID""), 0), IFNULL(SUM(""B"".""Size""), 0)" + @" , IFNULL(COUNT(CASE ""B"".""Restored"" WHEN 1 THEN ""B"".""ID"" ELSE NULL END), 0) " @@ -132,7 +132,7 @@ namespace Duplicati.Library.Main.Database // Will be one row per file. cmd.ExecuteNonQuery(sql); - sql = string.Format( + sql = FormatInvariant( @"INSERT INTO ""{0}"" (" + @" ""TotalFiles"", ""TotalBlocks"", ""TotalSize"" " + @", ""FilesFullyRestored"", ""FilesPartiallyRestored"", ""BlocksRestored"", ""SizeRestored""" @@ -154,7 +154,7 @@ namespace Duplicati.Library.Main.Database // with additional INSERT and DELETE triggers. // A trigger to update the file-stat entry each time a block changes restoration state. - sql = string.Format( + sql = FormatInvariant( @"CREATE TEMPORARY TRIGGER ""TrackRestoredBlocks_{1}"" AFTER UPDATE OF ""Restored"" ON ""{1}"" " + @" WHEN OLD.""Restored"" != NEW.""Restored"" AND NEW.""Metadata"" = 0 " + @" BEGIN UPDATE ""{0}"" " @@ -166,7 +166,7 @@ namespace Duplicati.Library.Main.Database cmd.ExecuteNonQuery(sql); // A trigger to update total stats each time a file stat changed (nested triggering by file-stats) - sql = string.Format( + sql = FormatInvariant( @"CREATE TEMPORARY TRIGGER ""UpdateTotalStats_{1}"" AFTER UPDATE ON ""{1}"" " + @" BEGIN UPDATE ""{0}"" " + @" SET ""FilesFullyRestored"" = ""{0}"".""FilesFullyRestored"" " @@ -185,7 +185,7 @@ namespace Duplicati.Library.Main.Database if (createFilesNewlyDoneTracker) { // A trigger checking if a file is done (all blocks restored in file-stat) (nested triggering by file-stats) - sql = string.Format( + sql = FormatInvariant( @"CREATE TEMPORARY TRIGGER ""UpdateFilesNewlyDone_{1}"" AFTER UPDATE OF ""BlocksRestored"", ""TotalBlocks"" ON ""{1}"" " + @" WHEN NEW.""BlocksRestored"" = NEW.""TotalBlocks"" " + @" BEGIN " @@ -232,17 +232,17 @@ namespace Duplicati.Library.Main.Database cmd.Parameters.Clear(); - cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", m_tempfiletable)); - cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", m_tempblocktable)); - cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""ID"" INTEGER PRIMARY KEY, ""Path"" TEXT NOT NULL, ""BlocksetID"" INTEGER NOT NULL, ""MetadataID"" INTEGER NOT NULL, ""TargetPath"" TEXT NULL, ""DataVerified"" BOOLEAN NOT NULL, ""LatestBlocksetId"" INTEGER, ""LocalSourceExists"" BOOLEAN) ", m_tempfiletable)); - cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""ID"" INTEGER PRIMARY KEY, ""FileID"" INTEGER NOT NULL, ""Index"" INTEGER NOT NULL, ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Restored"" BOOLEAN NOT NULL, ""Metadata"" BOOLEAN NOT NULL, ""VolumeID"" INTEGER NOT NULL, ""BlockID"" INTEGER NOT NULL)", m_tempblocktable)); + cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", m_tempfiletable)); + cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", m_tempblocktable)); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (""ID"" INTEGER PRIMARY KEY, ""Path"" TEXT NOT NULL, ""BlocksetID"" INTEGER NOT NULL, ""MetadataID"" INTEGER NOT NULL, ""TargetPath"" TEXT NULL, ""DataVerified"" BOOLEAN NOT NULL, ""LatestBlocksetId"" INTEGER, ""LocalSourceExists"" BOOLEAN) ", m_tempfiletable)); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (""ID"" INTEGER PRIMARY KEY, ""FileID"" INTEGER NOT NULL, ""Index"" INTEGER NOT NULL, ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Restored"" BOOLEAN NOT NULL, ""Metadata"" BOOLEAN NOT NULL, ""VolumeID"" INTEGER NOT NULL, ""BlockID"" INTEGER NOT NULL)", m_tempblocktable)); // TODO: Optimize to use the path prefix if (filter == null || filter.Empty) { // Simple case, restore everything - cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") SELECT ""File"".""ID"", ""File"".""Path"", ""File"".""BlocksetID"", ""File"".""MetadataID"", 0 FROM ""File"", ""FilesetEntry"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FilesetEntry"".""FilesetID"" = ? ", m_tempfiletable); + cmd.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" (""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") SELECT ""File"".""ID"", ""File"".""Path"", ""File"".""BlocksetID"", ""File"".""MetadataID"", 0 FROM ""File"", ""FilesetEntry"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FilesetEntry"".""FilesetID"" = ? ", m_tempfiletable); cmd.AddParameter(filesetId); cmd.ExecuteNonQuery(); } @@ -256,8 +256,8 @@ namespace Duplicati.Library.Main.Database var p = expression.GetSimpleList(); var m_filenamestable = "Filenames-" + m_temptabsetguid; cmd.Transaction = tr; - cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL) ", m_filenamestable)); - cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"") VALUES (?)", m_filenamestable); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL) ", m_filenamestable)); + cmd.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" (""Path"") VALUES (?)", m_filenamestable); cmd.AddParameter(); foreach (var s in p) @@ -266,7 +266,7 @@ namespace Duplicati.Library.Main.Database cmd.ExecuteNonQuery(); } - cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") SELECT ""File"".""ID"", ""File"".""Path"", ""File"".""BlocksetID"", ""File"".""MetadataID"", 0 FROM ""File"", ""FilesetEntry"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FilesetEntry"".""FilesetID"" = ? AND ""Path"" IN (SELECT DISTINCT ""Path"" FROM ""{1}"") ", m_tempfiletable, m_filenamestable); + cmd.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" (""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") SELECT ""File"".""ID"", ""File"".""Path"", ""File"".""BlocksetID"", ""File"".""MetadataID"", 0 FROM ""File"", ""FilesetEntry"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FilesetEntry"".""FilesetID"" = ? AND ""Path"" IN (SELECT DISTINCT ""Path"" FROM ""{1}"") ", m_tempfiletable, m_filenamestable); cmd.SetParameterValue(0, filesetId); var c = cmd.ExecuteNonQuery(); @@ -277,7 +277,7 @@ namespace Duplicati.Library.Main.Database var sb = new StringBuilder(); sb.AppendLine(); - using (var rd = cmd.ExecuteReader(string.Format(@"SELECT ""Path"" FROM ""{0}"" WHERE ""Path"" NOT IN (SELECT ""Path"" FROM ""{1}"")", m_filenamestable, m_tempfiletable))) + using (var rd = cmd.ExecuteReader(FormatInvariant(@"SELECT ""Path"" FROM ""{0}"" WHERE ""Path"" NOT IN (SELECT ""Path"" FROM ""{1}"")", m_filenamestable, m_tempfiletable))) while (rd.Read()) sb.AppendLine(rd.GetValue(0).ToString()); @@ -286,7 +286,7 @@ namespace Duplicati.Library.Main.Database cmd.Parameters.Clear(); } - cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", m_filenamestable)); + cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", m_filenamestable)); using (new Logging.Timer(LOGTAG, "CommitPrepareFileset", "CommitPrepareFileset")) tr.Commit(); @@ -302,7 +302,7 @@ namespace Duplicati.Library.Main.Database object[] values = new object[4]; using (var cmd2 = m_connection.CreateCommand()) { - cmd2.CommandText = string.Format(@"INSERT INTO ""{0}"" (""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") VALUES (?,?,?,?,0)", m_tempfiletable); + cmd2.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" (""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") VALUES (?,?,?,?,0)", m_tempfiletable); cmd2.AddParameter(); cmd2.AddParameter(); cmd2.AddParameter(); @@ -325,11 +325,11 @@ namespace Duplicati.Library.Main.Database } //creating indexes after insertion is much faster - cmd.ExecuteNonQuery(string.Format(@"CREATE INDEX ""{0}_ID"" ON ""{0}"" (""ID"")", m_tempfiletable)); - cmd.ExecuteNonQuery(string.Format(@"CREATE INDEX ""{0}_TargetPath"" ON ""{0}"" (""TargetPath"")", m_tempfiletable)); - cmd.ExecuteNonQuery(string.Format(@"CREATE INDEX ""{0}_Path"" ON ""{0}"" (""Path"")", m_tempfiletable)); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE INDEX ""{0}_ID"" ON ""{0}"" (""ID"")", m_tempfiletable)); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE INDEX ""{0}_TargetPath"" ON ""{0}"" (""TargetPath"")", m_tempfiletable)); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE INDEX ""{0}_Path"" ON ""{0}"" (""Path"")", m_tempfiletable)); - using (var rd = cmd.ExecuteReader(string.Format(@"SELECT COUNT(DISTINCT ""{0}"".""Path""), SUM(""Blockset"".""Length"") FROM ""{0}"", ""Blockset"" WHERE ""{0}"".""BlocksetID"" = ""Blockset"".""ID"" ", m_tempfiletable))) + using (var rd = cmd.ExecuteReader(FormatInvariant(@"SELECT COUNT(DISTINCT ""{0}"".""Path""), SUM(""Blockset"".""Length"") FROM ""{0}"", ""Blockset"" WHERE ""{0}"".""BlocksetID"" = ""Blockset"".""ID"" ", m_tempfiletable))) { var filecount = 0L; var filesize = 0L; @@ -356,7 +356,7 @@ namespace Duplicati.Library.Main.Database { using (var cmd = m_connection.CreateCommand()) { - cmd.CommandText = string.Format(@"SELECT ""Path"" FROM ""{0}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1", m_tempfiletable); + cmd.CommandText = FormatInvariant(@"SELECT ""Path"" FROM ""{0}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1", m_tempfiletable); var v0 = cmd.ExecuteScalar(); if (v0 == null || v0 == DBNull.Value) return null; @@ -369,7 +369,7 @@ namespace Duplicati.Library.Main.Database { using (var cmd = m_connection.CreateCommand()) { - cmd.CommandText = string.Format(@"SELECT ""Path"" FROM ""{0}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1", m_tempfiletable); + cmd.CommandText = FormatInvariant(@"SELECT ""Path"" FROM ""{0}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1", m_tempfiletable); var v0 = cmd.ExecuteScalar(); string maxpath = ""; if (v0 != null && v0 != DBNull.Value) @@ -377,12 +377,12 @@ namespace Duplicati.Library.Main.Database var dirsep = Util.GuessDirSeparator(maxpath); - cmd.CommandText = string.Format(@"SELECT COUNT(*) FROM ""{0}""", m_tempfiletable); + cmd.CommandText = FormatInvariant(@"SELECT COUNT(*) FROM ""{0}""", m_tempfiletable); var filecount = cmd.ExecuteScalarInt64(-1); long foundfiles = -1; //TODO: Handle FS case-sensitive? - cmd.CommandText = string.Format(@"SELECT COUNT(*) FROM ""{0}"" WHERE SUBSTR(""Path"", 1, ?) = ?", m_tempfiletable); + cmd.CommandText = FormatInvariant(@"SELECT COUNT(*) FROM ""{0}"" WHERE SUBSTR(""Path"", 1, ?) = ?", m_tempfiletable); cmd.AddParameter(); cmd.AddParameter(); @@ -423,19 +423,19 @@ namespace Duplicati.Library.Main.Database if ((OperatingSystem.IsMacOS() || OperatingSystem.IsLinux()) && dirsep == "\\") { // For Win -> Linux, we remove the colon from the drive letter, and use the drive letter as root folder - cmd.ExecuteNonQuery(string.Format(@"UPDATE ""{0}"" SET ""Targetpath"" = CASE WHEN SUBSTR(""Path"", 2, 1) == ':' THEN '\\' || SUBSTR(""Path"", 1, 1) || SUBSTR(""Path"", 3) ELSE ""Path"" END", m_tempfiletable)); - cmd.ExecuteNonQuery(string.Format(@"UPDATE ""{0}"" SET ""Targetpath"" = CASE WHEN SUBSTR(""Path"", 1, 2) == '\\' THEN '\\' || SUBSTR(""Path"", 2) ELSE ""Path"" END", m_tempfiletable)); + cmd.ExecuteNonQuery(FormatInvariant(@"UPDATE ""{0}"" SET ""Targetpath"" = CASE WHEN SUBSTR(""Path"", 2, 1) == ':' THEN '\\' || SUBSTR(""Path"", 1, 1) || SUBSTR(""Path"", 3) ELSE ""Path"" END", m_tempfiletable)); + cmd.ExecuteNonQuery(FormatInvariant(@"UPDATE ""{0}"" SET ""Targetpath"" = CASE WHEN SUBSTR(""Path"", 1, 2) == '\\' THEN '\\' || SUBSTR(""Path"", 2) ELSE ""Path"" END", m_tempfiletable)); } else if (OperatingSystem.IsWindows() && dirsep == "/") { // For Linux -> Win, we use the temporary folder's drive as the root path - cmd.ExecuteNonQuery(string.Format(@"UPDATE ""{0}"" SET ""Targetpath"" = CASE WHEN SUBSTR(""Path"", 1, 1) == '/' THEN ? || SUBSTR(""Path"", 2) ELSE ""Path"" END", m_tempfiletable), Util.AppendDirSeparator(System.IO.Path.GetPathRoot(Library.Utility.TempFolder.SystemTempPath)).Replace("\\", "/")); + cmd.ExecuteNonQuery(FormatInvariant(@"UPDATE ""{0}"" SET ""Targetpath"" = CASE WHEN SUBSTR(""Path"", 1, 1) == '/' THEN ? || SUBSTR(""Path"", 2) ELSE ""Path"" END", m_tempfiletable), Util.AppendDirSeparator(System.IO.Path.GetPathRoot(Library.Utility.TempFolder.SystemTempPath)).Replace("\\", "/")); } else { // Same OS, just use the path directly - cmd.ExecuteNonQuery(string.Format(@"UPDATE ""{0}"" SET ""Targetpath"" = ""Path"" ", m_tempfiletable)); + cmd.ExecuteNonQuery(FormatInvariant(@"UPDATE ""{0}"" SET ""Targetpath"" = ""Path"" ", m_tempfiletable)); } } else @@ -445,32 +445,32 @@ namespace Duplicati.Library.Main.Database //Special case, restoring to new folder, but files are from different drives (no shared root on Windows) // We use the format / / - cmd.ExecuteNonQuery(string.Format(@"UPDATE ""{0}"" SET ""TargetPath"" = CASE WHEN SUBSTR(""Path"", 2, 1) == ':' THEN SUBSTR(""Path"", 1, 1) || SUBSTR(""Path"", 3) ELSE ""Path"" END", m_tempfiletable)); + cmd.ExecuteNonQuery(FormatInvariant(@"UPDATE ""{0}"" SET ""TargetPath"" = CASE WHEN SUBSTR(""Path"", 2, 1) == ':' THEN SUBSTR(""Path"", 1, 1) || SUBSTR(""Path"", 3) ELSE ""Path"" END", m_tempfiletable)); // For UNC paths, we use \\server\folder -> / / - cmd.ExecuteNonQuery(string.Format(@"UPDATE ""{0}"" SET ""TargetPath"" = CASE WHEN SUBSTR(""Path"", 1, 2) == '\\' THEN SUBSTR(""Path"", 2) ELSE ""TargetPath"" END", m_tempfiletable)); + cmd.ExecuteNonQuery(FormatInvariant(@"UPDATE ""{0}"" SET ""TargetPath"" = CASE WHEN SUBSTR(""Path"", 1, 2) == '\\' THEN SUBSTR(""Path"", 2) ELSE ""TargetPath"" END", m_tempfiletable)); } else { largest_prefix = Util.AppendDirSeparator(largest_prefix, dirsep); - cmd.ExecuteNonQuery(string.Format(@"UPDATE ""{0}"" SET ""TargetPath"" = SUBSTR(""Path"", ?)", m_tempfiletable), largest_prefix.Length + 1); + cmd.ExecuteNonQuery(FormatInvariant(@"UPDATE ""{0}"" SET ""TargetPath"" = SUBSTR(""Path"", ?)", m_tempfiletable), largest_prefix.Length + 1); } } // Cross-os path remapping support if ((OperatingSystem.IsMacOS() || OperatingSystem.IsLinux()) && dirsep == "\\") // For Win paths on Linux - cmd.ExecuteNonQuery(string.Format(@"UPDATE ""{0}"" SET ""TargetPath"" = REPLACE(""TargetPath"", '\', '/')", m_tempfiletable)); + cmd.ExecuteNonQuery(FormatInvariant(@"UPDATE ""{0}"" SET ""TargetPath"" = REPLACE(""TargetPath"", '\', '/')", m_tempfiletable)); else if (OperatingSystem.IsWindows() && dirsep == "/") // For Linux paths on Windows - cmd.ExecuteNonQuery(string.Format(@"UPDATE ""{0}"" SET ""TargetPath"" = REPLACE(REPLACE(""TargetPath"", '\', '_'), '/', '\')", m_tempfiletable)); + cmd.ExecuteNonQuery(FormatInvariant(@"UPDATE ""{0}"" SET ""TargetPath"" = REPLACE(REPLACE(""TargetPath"", '\', '_'), '/', '\')", m_tempfiletable)); if (!string.IsNullOrEmpty(destination)) { // Paths are now relative with target-os naming system // so we prefix them with the target path - cmd.ExecuteNonQuery(string.Format(@"UPDATE ""{0}"" SET ""TargetPath"" = ? || ""TargetPath"" ", m_tempfiletable), Util.AppendDirSeparator(destination)); + cmd.ExecuteNonQuery(FormatInvariant(@"UPDATE ""{0}"" SET ""TargetPath"" = ? || ""TargetPath"" ", m_tempfiletable), Util.AppendDirSeparator(destination)); } @@ -481,23 +481,23 @@ namespace Duplicati.Library.Main.Database { using (var cmd = m_connection.CreateCommand()) { - cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""FileID"", ""Index"", ""Hash"", ""Size"", ""Restored"", ""Metadata"", ""VolumeId"", ""BlockId"") SELECT DISTINCT ""{1}"".""ID"", ""BlocksetEntry"".""Index"", ""Block"".""Hash"", ""Block"".""Size"", 0, 0, ""Block"".""VolumeID"", ""Block"".""ID"" FROM ""{1}"", ""BlocksetEntry"", ""Block"" WHERE ""{1}"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" ", m_tempblocktable, m_tempfiletable); + cmd.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" (""FileID"", ""Index"", ""Hash"", ""Size"", ""Restored"", ""Metadata"", ""VolumeId"", ""BlockId"") SELECT DISTINCT ""{1}"".""ID"", ""BlocksetEntry"".""Index"", ""Block"".""Hash"", ""Block"".""Size"", 0, 0, ""Block"".""VolumeID"", ""Block"".""ID"" FROM ""{1}"", ""BlocksetEntry"", ""Block"" WHERE ""{1}"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" ", m_tempblocktable, m_tempfiletable); var p1 = cmd.ExecuteNonQuery(); int p2 = 0; if (!skipMetadata) { - cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""FileID"", ""Index"", ""Hash"", ""Size"", ""Restored"", ""Metadata"", ""VolumeId"", ""BlockId"") SELECT DISTINCT ""{1}"".""ID"", ""BlocksetEntry"".""Index"", ""Block"".""Hash"", ""Block"".""Size"", 0, 1, ""Block"".""VolumeID"", ""Block"".""ID"" FROM ""{1}"", ""BlocksetEntry"", ""Block"", ""Metadataset"" WHERE ""{1}"".""MetadataID"" = ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" ", m_tempblocktable, m_tempfiletable); + cmd.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" (""FileID"", ""Index"", ""Hash"", ""Size"", ""Restored"", ""Metadata"", ""VolumeId"", ""BlockId"") SELECT DISTINCT ""{1}"".""ID"", ""BlocksetEntry"".""Index"", ""Block"".""Hash"", ""Block"".""Size"", 0, 1, ""Block"".""VolumeID"", ""Block"".""ID"" FROM ""{1}"", ""BlocksetEntry"", ""Block"", ""Metadataset"" WHERE ""{1}"".""MetadataID"" = ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" ", m_tempblocktable, m_tempfiletable); p2 = cmd.ExecuteNonQuery(); } //creating indexes after insertion is much faster - cmd.ExecuteNonQuery(string.Format(@"CREATE INDEX ""{0}_HashSizeIndex"" ON ""{0}"" (""Hash"", ""Size"")", m_tempblocktable)); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE INDEX ""{0}_HashSizeIndex"" ON ""{0}"" (""Hash"", ""Size"")", m_tempblocktable)); // better suited to speed up commit on UpdateBlocks - cmd.ExecuteNonQuery(string.Format(@"CREATE INDEX ""{0}_FileIdIndexIndex"" ON ""{0}"" (""FileId"", ""Index"")", m_tempblocktable)); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE INDEX ""{0}_FileIdIndexIndex"" ON ""{0}"" (""FileId"", ""Index"")", m_tempblocktable)); - var size = cmd.ExecuteScalarInt64(string.Format(@"SELECT SUM(""Size"") FROM ""{0}"" ", m_tempblocktable), 0); + var size = cmd.ExecuteScalarInt64(FormatInvariant(@"SELECT SUM(""Size"") FROM ""{0}"" ", m_tempblocktable), 0); Logging.Log.WriteVerboseMessage(LOGTAG, "RestoreSourceSize", "Restore list contains {0} blocks with a total size of {1}", p1 + p2, Library.Utility.Utility.FormatSizeString(size)); } } @@ -505,7 +505,7 @@ namespace Duplicati.Library.Main.Database public void UpdateTargetPath(long ID, string newname) { using (var cmd = m_connection.CreateCommand()) - cmd.ExecuteNonQuery(string.Format(@"UPDATE ""{0}"" SET ""TargetPath"" = ? WHERE ""ID"" = ?", m_tempfiletable), newname, ID); + cmd.ExecuteNonQuery(FormatInvariant(@"UPDATE ""{0}"" SET ""TargetPath"" = ? WHERE ""ID"" = ?", m_tempfiletable), newname, ID); } public interface IExistingFileBlock @@ -614,7 +614,7 @@ namespace Duplicati.Library.Main.Database { using (var cmd = connection.CreateCommand()) { - cmd.CommandText = string.Format(@"SELECT ""{0}"".""TargetPath"", ""Blockset"".""FullHash"", ""{0}"".""ID"", ""Blockset"".""Length"", ""Block"".""Hash"", ""BlocksetEntry"".""Index"", ""Block"".""Size"" FROM ""{0}"", ""Blockset"", ""BlocksetEntry"", ""Block"" WHERE ""{0}"".""BlocksetID"" = ""Blockset"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""{0}"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" ORDER BY ""{0}"".""TargetPath"", ""BlocksetEntry"".""Index""", tablename); + cmd.CommandText = FormatInvariant(@"SELECT ""{0}"".""TargetPath"", ""Blockset"".""FullHash"", ""{0}"".""ID"", ""Blockset"".""Length"", ""Block"".""Hash"", ""BlocksetEntry"".""Index"", ""Block"".""Size"" FROM ""{0}"", ""Blockset"", ""BlocksetEntry"", ""Block"" WHERE ""{0}"".""BlocksetID"" = ""Blockset"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""{0}"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" ORDER BY ""{0}"".""TargetPath"", ""BlocksetEntry"".""Index""", tablename); using (var rd = cmd.ExecuteReader()) if (rd.Read()) { @@ -716,7 +716,7 @@ namespace Duplicati.Library.Main.Database using (var cmd = connection.CreateCommand()) { // TODO: Skip metadata as required - cmd.CommandText = string.Format(@"SELECT DISTINCT ""A"".""TargetPath"", ""A"".""ID"", ""B"".""Hash"", (""B"".""Index"" * {2}), ""B"".""Index"", ""B"".""Size"", ""C"".""Path"", (""D"".""Index"" * {2}), ""E"".""Size"", ""B"".""Metadata"" FROM ""{0}"" ""A"", ""{1}"" ""B"", ""File"" ""C"", ""BlocksetEntry"" ""D"", ""Block"" E WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""C"".""BlocksetID"" = ""D"".""BlocksetID"" AND ""D"".""BlockID"" = ""E"".""ID"" AND ""B"".""Hash"" = ""E"".""Hash"" AND ""B"".""Size"" = ""E"".""Size"" AND ""B"".""Restored"" = 0", filetablename, blocktablename, blocksize); + cmd.CommandText = FormatInvariant(@"SELECT DISTINCT ""A"".""TargetPath"", ""A"".""ID"", ""B"".""Hash"", (""B"".""Index"" * {2}), ""B"".""Index"", ""B"".""Size"", ""C"".""Path"", (""D"".""Index"" * {2}), ""E"".""Size"", ""B"".""Metadata"" FROM ""{0}"" ""A"", ""{1}"" ""B"", ""File"" ""C"", ""BlocksetEntry"" ""D"", ""Block"" E WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""C"".""BlocksetID"" = ""D"".""BlocksetID"" AND ""D"".""BlockID"" = ""E"".""ID"" AND ""B"".""Hash"" = ""E"".""Hash"" AND ""B"".""Size"" = ""E"".""Size"" AND ""B"".""Restored"" = 0", filetablename, blocktablename, blocksize); using (var rd = cmd.ExecuteReader()) { if (rd.Read()) @@ -757,7 +757,7 @@ namespace Duplicati.Library.Main.Database // Now it is likely to restore all files from front to back. Large files will always be done last. // One could also use like the average block number in a volume, that needs to be measured. - cmd.CommandText = string.Format( + cmd.CommandText = FormatInvariant( @"SELECT ""RV"".""Name"", ""RV"".""Hash"", ""RV"".""Size"", ""BB"".""MaxIndex"" " + @" FROM ""RemoteVolume"" ""RV"" INNER JOIN " + @" (SELECT ""TB"".""VolumeID"", MAX(""TB"".""Index"") as ""MaxIndex"" " @@ -809,11 +809,11 @@ namespace Duplicati.Library.Main.Database using (var c = m_connection.CreateCommand()) { m_tmptable = "VolumeFiles-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - c.CommandText = string.Format(@"CREATE TEMPORARY TABLE ""{0}"" ( ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL )", m_tmptable); + c.CommandText = FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" ( ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL )", m_tmptable); c.ExecuteNonQuery(); - c.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Hash"", ""Size"") VALUES (?,?)", m_tmptable); + c.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" (""Hash"", ""Size"") VALUES (?,?)", m_tmptable); c.AddParameters(2); foreach (var s in curvolume.Blocks) { @@ -832,7 +832,7 @@ namespace Duplicati.Library.Main.Database if (m_tmptable != null) using (var c = m_connection.CreateCommand()) { - c.CommandText = string.Format(@"DROP TABLE IF EXISTS ""{0}""", m_tmptable); + c.CommandText = FormatInvariant(@"DROP TABLE IF EXISTS ""{0}""", m_tmptable); c.ExecuteNonQuery(); } } @@ -877,7 +877,7 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) { // The IN-clause with subquery enables SQLite to use indexes better. Three way join (A,B,C) is slow here! - cmd.CommandText = string.Format( + cmd.CommandText = FormatInvariant( @" SELECT DISTINCT ""A"".""TargetPath"", ""BB"".""FileID"", (""BB"".""Index"" * {3}), ""BB"".""Size"", ""BB"".""Hash"" " + @" FROM ""{0}"" ""A"", ""{1}"" ""BB"" " + @" WHERE ""A"".""ID"" = ""BB"".""FileID"" AND ""BB"".""Restored"" = 0 AND ""BB"".""Metadata"" = {4}" @@ -912,7 +912,7 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) { // The IN-clause with subquery enables SQLite to use indexes better. Three way join (A,B,C) is slow here! - cmd.CommandText = string.Format( + cmd.CommandText = FormatInvariant( @" SELECT DISTINCT ""A"".""TargetPath"", ""BB"".""FileID"", (""BB"".""Index"" * {3}), ""BB"".""Size"", ""BB"".""Hash"" " + @" FROM ""{0}"" ""A"", ""{1}"" ""BB"" " + @" WHERE ""A"".""ID"" = ""BB"".""FileID"" AND ""BB"".""Restored"" = 0 AND ""BB"".""Metadata"" = {4}" @@ -985,7 +985,7 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) { cmd.AddParameter(!onlyNonVerified); - using (var rd = cmd.ExecuteReader(string.Format(@"SELECT ""{0}"".""ID"", ""{0}"".""TargetPath"", ""Blockset"".""FullHash"", ""Blockset"".""Length"" FROM ""{0}"",""Blockset"" WHERE ""{0}"".""BlocksetID"" = ""Blockset"".""ID"" AND ""{0}"".""DataVerified"" <= ?", m_tempfiletable))) + using (var rd = cmd.ExecuteReader(FormatInvariant(@"SELECT ""{0}"".""ID"", ""{0}"".""TargetPath"", ""Blockset"".""FullHash"", ""Blockset"".""Length"" FROM ""{0}"",""Blockset"" WHERE ""{0}"".""BlocksetID"" = ""Blockset"".""ID"" AND ""{0}"".""DataVerified"" <= ?", m_tempfiletable))) while (rd.Read()) yield return new FileToRestore(rd.ConvertValueToInt64(0), rd.ConvertValueToString(1), rd.ConvertValueToString(2), rd.ConvertValueToInt64(3)); } @@ -1000,11 +1000,11 @@ namespace Duplicati.Library.Main.Database { using var cmd = m_connection.CreateCommand(); cmd.AddParameter(!onlyNonVerified); - using var rd = cmd.ExecuteReader($@" + using var rd = cmd.ExecuteReader(FormatInvariant($@" SELECT F.ID, F.Path, F.TargetPath, IFNULL(B.FullHash, ''), IFNULL(B.Length, 0), F.BlocksetID FROM ""{m_tempfiletable}"" F LEFT JOIN Blockset B ON F.BlocksetID = B.ID - WHERE F.BlocksetID != {FOLDER_BLOCKSET_ID}"); + WHERE F.BlocksetID != {FOLDER_BLOCKSET_ID}")); while (rd.Read()) yield return new FileRequest(rd.ConvertValueToInt64(0), rd.ConvertValueToString(1), rd.ConvertValueToString(2), rd.ConvertValueToString(3), rd.ConvertValueToInt64(4), rd.ConvertValueToInt64(5)); } @@ -1017,13 +1017,13 @@ namespace Duplicati.Library.Main.Database public IEnumerable<(long, long)> GetBlocksAndVolumeIDs(bool skipMetadata) { using var cmd = Connection.CreateCommand(); - using var reader = cmd.ExecuteReader($@" + using var reader = cmd.ExecuteReader(FormatInvariant($@" SELECT Block.ID, Block.VolumeID FROM BlocksetEntry INNER JOIN ""{m_tempfiletable}"" ON BlocksetEntry.BlocksetID = ""{m_tempfiletable}"".BlocksetID INNER JOIN Block ON BlocksetEntry.BlockID = Block.ID - " - + (skipMetadata ? "" : $@" + ") + + (skipMetadata ? "" : FormatInvariant($@" UNION ALL SELECT Block.ID, Block.VolumeID FROM ""{m_tempfiletable}"" @@ -1031,7 +1031,7 @@ namespace Duplicati.Library.Main.Database INNER JOIN BlocksetEntry ON Metadataset.BlocksetID = BlocksetEntry.BlocksetID INNER JOIN Block ON BlocksetEntry.BlockID = Block.ID WHERE ""{m_tempfiletable}"".BlocksetID IS NOT {FOLDER_BLOCKSET_ID} - ")); + "))); while (reader.Read()) yield return (reader.ConvertValueToInt64(0), reader.ConvertValueToInt64(1)); } @@ -1046,11 +1046,11 @@ namespace Duplicati.Library.Main.Database var connection = GetConnectionFromPool(); using var cmd = connection.CreateCommand(); - cmd.CommandText = @$" + cmd.CommandText = FormatInvariant(@$" SELECT Block.ID, Block.Hash, Block.Size, Block.VolumeID FROM BlocksetEntry INNER JOIN Block ON BlocksetEntry.BlockID = Block.ID - WHERE BlocksetEntry.BlocksetID = ?"; + WHERE BlocksetEntry.BlocksetID = ?"); cmd.AddParameter(); cmd.SetParameterValue(0, blocksetID); @@ -1074,14 +1074,14 @@ namespace Duplicati.Library.Main.Database var connection = GetConnectionFromPool(); using var cmd = connection.CreateCommand(); - cmd.CommandText = $@" + cmd.CommandText = FormatInvariant($@" SELECT Block.ID, Block.Hash, Block.Size, Block.VolumeID FROM File INNER JOIN Metadataset ON File.MetadataID = Metadataset.ID INNER JOIN BlocksetEntry ON Metadataset.BlocksetID = BlocksetEntry.BlocksetID INNER JOIN Block ON BlocksetEntry.BlockID = Block.ID WHERE File.ID = ? - "; + "); cmd.AddParameter(); cmd.SetParameterValue(0, fileID); @@ -1118,7 +1118,7 @@ namespace Duplicati.Library.Main.Database if (m_tempfiletable != null) try { - cmd.CommandText = string.Format(@"DROP TABLE IF EXISTS ""{0}""", m_tempfiletable); + cmd.CommandText = FormatInvariant(@"DROP TABLE IF EXISTS ""{0}""", m_tempfiletable); cmd.ExecuteNonQuery(); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } @@ -1127,7 +1127,7 @@ namespace Duplicati.Library.Main.Database if (m_tempblocktable != null) try { - cmd.CommandText = string.Format(@"DROP TABLE IF EXISTS ""{0}""", m_tempblocktable); + cmd.CommandText = FormatInvariant(@"DROP TABLE IF EXISTS ""{0}""", m_tempblocktable); cmd.ExecuteNonQuery(); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } @@ -1136,7 +1136,7 @@ namespace Duplicati.Library.Main.Database if (m_latestblocktable != null) try { - cmd.CommandText = string.Format(@"DROP TABLE IF EXISTS ""{0}""", m_latestblocktable); + cmd.CommandText = FormatInvariant(@"DROP TABLE IF EXISTS ""{0}""", m_latestblocktable); cmd.ExecuteNonQuery(); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } @@ -1145,7 +1145,7 @@ namespace Duplicati.Library.Main.Database if (m_fileprogtable != null) try { - cmd.CommandText = string.Format(@"DROP TABLE IF EXISTS ""{0}""", m_fileprogtable); + cmd.CommandText = FormatInvariant(@"DROP TABLE IF EXISTS ""{0}""", m_fileprogtable); cmd.ExecuteNonQuery(); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } @@ -1154,7 +1154,7 @@ namespace Duplicati.Library.Main.Database if (m_totalprogtable != null) try { - cmd.CommandText = string.Format(@"DROP TABLE IF EXISTS ""{0}""", m_totalprogtable); + cmd.CommandText = FormatInvariant(@"DROP TABLE IF EXISTS ""{0}""", m_totalprogtable); cmd.ExecuteNonQuery(); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } @@ -1163,7 +1163,7 @@ namespace Duplicati.Library.Main.Database if (m_filesnewlydonetable != null) try { - cmd.CommandText = string.Format(@"DROP TABLE IF EXISTS ""{0}""", m_filesnewlydonetable); + cmd.CommandText = FormatInvariant(@"DROP TABLE IF EXISTS ""{0}""", m_filesnewlydonetable); cmd.ExecuteNonQuery(); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } @@ -1216,23 +1216,23 @@ namespace Duplicati.Library.Main.Database m_blocktablename = blocktablename; m_filetablename = filetablename; - m_insertblockCommand.CommandText = string.Format( + m_insertblockCommand.CommandText = FormatInvariant( @"UPDATE ""{0}"" SET ""Restored"" = 1 " + @" WHERE ""FileID"" = ? AND ""Index"" = ? AND ""Hash"" = ? AND ""Size"" = ? AND ""Metadata"" = ? AND ""Restored"" = 0 " , m_blocktablename); m_insertblockCommand.AddParameters(5); - m_resetfileCommand.CommandText = string.Format( + m_resetfileCommand.CommandText = FormatInvariant( @"UPDATE ""{0}"" SET ""Restored"" = 0 WHERE ""FileID"" = ? " , m_blocktablename); m_resetfileCommand.AddParameters(1); - m_updateAsRestoredCommand.CommandText = string.Format( + m_updateAsRestoredCommand.CommandText = FormatInvariant( @"UPDATE ""{0}"" SET ""Restored"" = 1 WHERE ""FileID"" = ? AND ""Metadata"" <= ? " , m_blocktablename); m_updateAsRestoredCommand.AddParameters(2); - m_updateFileAsDataVerifiedCommand.CommandText = string.Format( + m_updateFileAsDataVerifiedCommand.CommandText = FormatInvariant( @"UPDATE ""{0}"" SET ""DataVerified"" = 1 WHERE ""ID"" = ?" , m_filetablename); m_updateFileAsDataVerifiedCommand.AddParameters(1); @@ -1241,10 +1241,10 @@ namespace Duplicati.Library.Main.Database { // Fields in Stats: TotalFiles, TotalBlocks, TotalSize // FilesFullyRestored, FilesPartiallyRestored, BlocksRestored, SizeRestored - m_statUpdateCommand.CommandText = string.Format(@"SELECT SUM(""FilesFullyRestored""), SUM(""SizeRestored"") FROM ""{0}"" ", statstablename); + m_statUpdateCommand.CommandText = FormatInvariant(@"SELECT SUM(""FilesFullyRestored""), SUM(""SizeRestored"") FROM ""{0}"" ", statstablename); } else // very slow fallback if stats tables were not created - m_statUpdateCommand.CommandText = string.Format(@"SELECT COUNT(DISTINCT ""FileID""), SUM(""Size"") FROM ""{0}"" WHERE ""Restored"" = 1 ", m_blocktablename); + m_statUpdateCommand.CommandText = FormatInvariant(@"SELECT COUNT(DISTINCT ""FileID""), SUM(""Size"") FROM ""{0}"" WHERE ""Restored"" = 1 ", m_blocktablename); } @@ -1370,7 +1370,7 @@ namespace Duplicati.Library.Main.Database public IEnumerable GetTargetFolders() { using (var cmd = m_connection.CreateCommand()) - using (var rd = cmd.ExecuteReader(string.Format(@"SELECT ""TargetPath"" FROM ""{0}"" WHERE ""BlocksetID"" == ?", m_tempfiletable), FOLDER_BLOCKSET_ID)) + using (var rd = cmd.ExecuteReader(FormatInvariant(@"SELECT ""TargetPath"" FROM ""{0}"" WHERE ""BlocksetID"" == ?", m_tempfiletable), FOLDER_BLOCKSET_ID)) while (rd.Read()) yield return rd.ConvertValueToString(0); } @@ -1435,11 +1435,11 @@ namespace Duplicati.Library.Main.Database { using (var cmd = m_connection.CreateCommand()) { - cmd.CommandText = string.Format(@"UPDATE ""{0}"" SET ""LocalSourceExists"" = 1 WHERE Path = ?", m_tempfiletable); + cmd.CommandText = FormatInvariant(@"UPDATE ""{0}"" SET ""LocalSourceExists"" = 1 WHERE Path = ?", m_tempfiletable); cmd.AddParameters(1); cmd.Transaction = transaction; - var fileset = string.Format(@"SELECT DISTINCT ""{0}"".""Path"" FROM ""{0}""", m_tempfiletable); + var fileset = FormatInvariant(@"SELECT DISTINCT ""{0}"".""Path"" FROM ""{0}""", m_tempfiletable); using (var rd = cmdReader.ExecuteReader(fileset)) { while (rd.Read()) @@ -1458,7 +1458,7 @@ namespace Duplicati.Library.Main.Database } //This localSourceExists index will make the query engine to start by searching FileSet table. As the result is ordered by FileSet.ID, we will get the cursor "instantly" - cmd.ExecuteNonQuery(string.Format(@"CREATE INDEX ""{0}_LocalSourceExists"" ON ""{0}"" (""LocalSourceExists"")", m_tempfiletable)); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE INDEX ""{0}_LocalSourceExists"" ON ""{0}"" (""LocalSourceExists"")", m_tempfiletable)); } } transaction.Commit(); @@ -1466,13 +1466,13 @@ namespace Duplicati.Library.Main.Database m_latestblocktable = "LatestBlocksetIds-" + m_temptabsetguid; - var whereclause = string.Format(@" + var whereclause = FormatInvariant(@" ""{0}"".""LocalSourceExists"" = 1 AND ""{1}"".""Restored"" = 0 AND ""{1}"".""Metadata"" = 0 AND ""{0}"".""TargetPath"" != ""{0}"".""Path""", m_tempfiletable, m_tempblocktable); - var latestBlocksetIds = string.Format(@" + var latestBlocksetIds = FormatInvariant(@" SELECT ""File"".""Path"" AS ""PATH"", ""File"".""BlocksetID"" AS ""BlocksetID"", @@ -1498,14 +1498,14 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) { - cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", m_latestblocktable)); - cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS {1}", m_latestblocktable, latestBlocksetIds)); - cmd.ExecuteNonQuery(string.Format(@"CREATE INDEX ""{0}_path"" ON ""{0}"" (""Path"")", m_latestblocktable)); + cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", m_latestblocktable)); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" AS {1}", m_latestblocktable, latestBlocksetIds)); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE INDEX ""{0}_path"" ON ""{0}"" (""Path"")", m_latestblocktable)); - cmd.ExecuteNonQuery(string.Format(@"UPDATE ""{0}"" SET LatestBlocksetId = (SELECT BlocksetId FROM ""{1}"" WHERE Path = ""{0}"".Path)", m_tempfiletable, m_latestblocktable)); + cmd.ExecuteNonQuery(FormatInvariant(@"UPDATE ""{0}"" SET LatestBlocksetId = (SELECT BlocksetId FROM ""{1}"" WHERE Path = ""{0}"".Path)", m_tempfiletable, m_latestblocktable)); } - var sources = string.Format( + var sources = FormatInvariant( @" SELECT DISTINCT " + @" ""{0}"".""TargetPath""," + @" ""{0}"".""Path""," + diff --git a/Duplicati/Library/Main/Database/LocalTestDatabase.cs b/Duplicati/Library/Main/Database/LocalTestDatabase.cs index 7d774c0ba..e6b8f8b1c 100644 --- a/Duplicati/Library/Main/Database/LocalTestDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalTestDatabase.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -187,13 +187,13 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) { cmd.Transaction = m_transaction; - cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" {1}", tablename, tableFormat)); + cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" {1}", tablename, tableFormat)); m_tablename = tablename; } m_insertCommand = m_connection.CreateCommand(); m_insertCommand.Transaction = m_transaction; - m_insertCommand.CommandText = string.Format(@"INSERT INTO ""{0}"" {1}", m_tablename, insertCommand); + m_insertCommand.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" {1}", m_tablename, insertCommand); m_insertCommand.AddParameters(insertArguments); } @@ -205,7 +205,7 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) { cmd.Transaction = m_transaction; - cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}""", m_tablename)); + cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}""", m_tablename)); } } catch { } @@ -267,15 +267,15 @@ namespace Duplicati.Library.Main.Database try { - cmd.ExecuteNonQuery(string.Format(create, m_tablename, cmpName), m_volumename); - using (var rd = cmd.ExecuteReader(string.Format(extra + " UNION " + missing + " UNION " + modified, m_tablename, cmpName), (int)Library.Interface.TestEntryStatus.Extra, (int)Library.Interface.TestEntryStatus.Missing, (int)Library.Interface.TestEntryStatus.Modified)) + cmd.ExecuteNonQuery(FormatInvariant(create, m_tablename, cmpName), m_volumename); + using (var rd = cmd.ExecuteReader(FormatInvariant(extra + " UNION " + missing + " UNION " + modified, m_tablename, cmpName), (int)Library.Interface.TestEntryStatus.Extra, (int)Library.Interface.TestEntryStatus.Missing, (int)Library.Interface.TestEntryStatus.Modified)) while (rd.Read()) yield return new KeyValuePair((Duplicati.Library.Interface.TestEntryStatus)rd.GetInt64(0), rd.GetString(1)); } finally { - try { cmd.ExecuteNonQuery(string.Format(drop, m_tablename, cmpName)); } + try { cmd.ExecuteNonQuery(FormatInvariant(drop, m_tablename, cmpName)); } catch { } } } @@ -323,15 +323,15 @@ namespace Duplicati.Library.Main.Database try { - cmd.ExecuteNonQuery(string.Format(create, m_tablename, cmpName), m_volumename); - using (var rd = cmd.ExecuteReader(string.Format(extra + " UNION " + missing + " UNION " + modified, m_tablename, cmpName), (int)Library.Interface.TestEntryStatus.Extra, (int)Library.Interface.TestEntryStatus.Missing, (int)Library.Interface.TestEntryStatus.Modified)) + cmd.ExecuteNonQuery(FormatInvariant(create, m_tablename, cmpName), m_volumename); + using (var rd = cmd.ExecuteReader(FormatInvariant(extra + " UNION " + missing + " UNION " + modified, m_tablename, cmpName), (int)Library.Interface.TestEntryStatus.Extra, (int)Library.Interface.TestEntryStatus.Missing, (int)Library.Interface.TestEntryStatus.Modified)) while (rd.Read()) yield return new KeyValuePair((Duplicati.Library.Interface.TestEntryStatus)rd.GetInt64(0), rd.GetString(1)); } finally { - try { cmd.ExecuteNonQuery(string.Format(drop, m_tablename, cmpName)); } + try { cmd.ExecuteNonQuery(FormatInvariant(drop, m_tablename, cmpName)); } catch { } } } @@ -380,15 +380,15 @@ namespace Duplicati.Library.Main.Database try { - cmd.ExecuteNonQuery(string.Format(create, cmpName, curBlocks, delBlocks, duplBlocks), m_volumename, m_volumename, m_volumename); - using (var rd = cmd.ExecuteReader(string.Format(extra + " UNION " + missing + " UNION " + modified, m_tablename, cmpName), (int)Library.Interface.TestEntryStatus.Extra, (int)Library.Interface.TestEntryStatus.Missing, (int)Library.Interface.TestEntryStatus.Modified)) + cmd.ExecuteNonQuery(FormatInvariant(create, cmpName, curBlocks, delBlocks, duplBlocks), m_volumename, m_volumename, m_volumename); + using (var rd = cmd.ExecuteReader(FormatInvariant(extra + " UNION " + missing + " UNION " + modified, m_tablename, cmpName), (int)Library.Interface.TestEntryStatus.Extra, (int)Library.Interface.TestEntryStatus.Missing, (int)Library.Interface.TestEntryStatus.Modified)) while (rd.Read()) yield return new KeyValuePair((Duplicati.Library.Interface.TestEntryStatus)rd.GetInt64(0), rd.GetString(1)); } finally { - try { cmd.ExecuteNonQuery(string.Format(drop, m_tablename, cmpName)); } + try { cmd.ExecuteNonQuery(FormatInvariant(drop, m_tablename, cmpName)); } catch { } } } diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index 071a65ec5..9410e6bd6 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -1635,5 +1635,22 @@ namespace Duplicati.Library.Utility return sanitizedUrl; } + + /// + /// Formats the string using the invariant culture + /// + /// The format string + /// The arguments to format + /// The formatted string + public static string FormatInvariant(this string format, params object?[] args) + => string.Format(CultureInfo.InvariantCulture, format, args); + + /// + /// Formats the string using the invariant culture + /// + /// The formattable string + /// The formatted string + public static string FormatInvariant(FormattableString formattable) + => formattable.ToString(CultureInfo.InvariantCulture); } } From c1bdaa069e7148f2472ad4ece707dacb33a421f6 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 14 Mar 2025 10:41:14 +0100 Subject: [PATCH 352/362] Async operation handlers Rewrote all operation handlers to be async. This moves the async/sync threshold out into the controller. Fixed all the uses of cancellation token in the handlers to use the provided cancellation token. Introduced a new ReusableTransaction that can be passed around, avoiding a ref parameter passing of the transaction instance. --- Duplicati/Library/Main/Controller.cs | 151 +++++++++--------- .../Main/Database/ReusableTransaction.cs | 70 ++++++++ .../Library/Main/Operation/BackupHandler.cs | 87 ++++------ .../Library/Main/Operation/CompactHandler.cs | 142 +++++++--------- .../Main/Operation/CreateBugReportHandler.cs | 64 ++++---- .../Library/Main/Operation/DeleteHandler.cs | 74 ++++----- .../Library/Main/Operation/ListAffected.cs | 50 +++--- .../Main/Operation/ListBrokenFilesHandler.cs | 25 +-- .../Main/Operation/ListChangesHandler.cs | 28 ++-- .../Main/Operation/ListControlFilesHandler.cs | 11 +- .../Main/Operation/ListFilesHandler.cs | 2 +- .../Main/Operation/PurgeBrokenFilesHandler.cs | 90 +++++------ .../Main/Operation/PurgeFilesHandler.cs | 49 +++--- .../Main/Operation/RecreateDatabaseHandler.cs | 25 ++- .../Library/Main/Operation/RepairHandler.cs | 37 ++--- .../Operation/RestoreControlFilesHandler.cs | 35 ++-- .../Library/Main/Operation/RestoreHandler.cs | 17 +- .../Main/Operation/SystemInfoHandler.cs | 7 +- .../Library/Main/Operation/TestHandler.cs | 20 +-- .../Library/Main/Operation/VacuumHandler.cs | 59 +++---- 20 files changed, 519 insertions(+), 524 deletions(-) create mode 100644 Duplicati/Library/Main/Database/ReusableTransaction.cs diff --git a/Duplicati/Library/Main/Controller.cs b/Duplicati/Library/Main/Controller.cs index 5fc7d8554..ef4abc5db 100644 --- a/Duplicati/Library/Main/Controller.cs +++ b/Duplicati/Library/Main/Controller.cs @@ -123,13 +123,12 @@ namespace Duplicati.Library.Main CheckAutoCompactInterval(); CheckAutoVacuumInterval(); - return RunAction(new BackupResults(), ref inputsources, ref filter, (result, backendManager) => + return RunAction(new BackupResults(), ref inputsources, ref filter, async (result, backendManager) => { using (var h = new Operation.BackupHandler(m_options, result)) - { - h.RunAsync(ExpandInputSources(inputsources, filter), backendManager, filter).Await(); - } + await h.RunAsync(ExpandInputSources(inputsources, filter), backendManager, filter) + .ConfigureAwait(false); UsageReporter.Reporter.Report("BACKUP_FILECOUNT", result.ExaminedFiles); UsageReporter.Reporter.Report("BACKUP_FILESIZE", result.SizeOfExaminedFiles); @@ -139,9 +138,11 @@ namespace Duplicati.Library.Main public IRestoreResults Restore(string[] paths, IFilter filter = null) { - return RunAction(new RestoreResults(), ref paths, ref filter, (result, backendManager) => + return RunAction(new RestoreResults(), ref paths, ref filter, async (result, backendManager) => { - new Operation.RestoreHandler(m_options, result).Run(paths, backendManager, filter); + await new Operation.RestoreHandler(m_options, result) + .RunAsync(paths, backendManager, filter) + .ConfigureAwait(false); UsageReporter.Reporter.Report("RESTORE_FILECOUNT", result.RestoredFiles); UsageReporter.Reporter.Report("RESTORE_FILESIZE", result.SizeOfRestoredFiles); @@ -152,25 +153,25 @@ namespace Duplicati.Library.Main public IRestoreControlFilesResults RestoreControlFiles(IEnumerable files = null, IFilter filter = null) { return RunAction(new RestoreControlFilesResults(), ref filter, (result, backendManager) => - { - new Operation.RestoreControlFilesHandler(m_options, result).Run(files, backendManager, filter); - }); + new Operation.RestoreControlFilesHandler(m_options, result) + .RunAsync(files, backendManager, filter) + ); } public IDeleteResults Delete() { return RunAction(new DeleteResults(), (result, backendManager) => - { - new Operation.DeleteHandler(m_options, result).Run(backendManager); - }); + new Operation.DeleteHandler(m_options, result) + .RunAsync(backendManager) + ); } public IRepairResults Repair(IFilter filter = null) { return RunAction(new RepairResults(), ref filter, (result, backendManager) => - { - new Operation.RepairHandler(m_options, result).Run(backendManager, filter); - }); + new Operation.RepairHandler(m_options, result) + .RunAsync(backendManager, filter) + ); } public IListResults List() @@ -186,38 +187,37 @@ namespace Duplicati.Library.Main public IListResults List(IEnumerable filterstrings, IFilter filter) { return RunAction(new ListResults(), ref filter, (result, backendManager) => - { - new Operation.ListFilesHandler(m_options, result).Run(backendManager, filterstrings, filter).Await(); - }); + new Operation.ListFilesHandler(m_options, result) + .RunAsync(backendManager, filterstrings, filter) + ); } public IListResults ListControlFiles(IEnumerable filterstrings, IFilter filter) { return RunAction(new ListResults(), ref filter, (result, backendManager) => - { - new Operation.ListControlFilesHandler(m_options, result).Run(backendManager, filterstrings, filter); - }); + new Operation.ListControlFilesHandler(m_options, result) + .RunAsync(backendManager, filterstrings, filter) + ); } public IListRemoteResults ListRemote() { - return RunAction(new ListRemoteResults(), (result, backendManager) => + return RunAction(new ListRemoteResults(), async (result, backendManager) => { - using (var tf = File.Exists(m_options.Dbpath) ? null : new Library.Utility.TempFile()) - using (var db = new Database.LocalDatabase(((string)tf) ?? m_options.Dbpath, "list-remote", true)) - result.SetResult(backendManager.ListAsync(CancellationToken.None).Await()); + using (var tf = File.Exists(m_options.Dbpath) ? null : new TempFile()) + using (var db = new LocalDatabase(((string)tf) ?? m_options.Dbpath, "list-remote", true)) + result.SetResult(await backendManager.ListAsync(CancellationToken.None).ConfigureAwait(false)); }); } public IListRemoteResults DeleteAllRemoteFiles() { - return RunAction(new ListRemoteResults(), (result, backendManager) => + return RunAction(new ListRemoteResults(), async (result, backendManager) => { - var cancelToken = CancellationToken.None; result.OperationProgressUpdater.UpdatePhase(OperationPhase.Delete_Listing); { // Only delete files that match the expected pattern and prefix - var list = backendManager.ListAsync(cancelToken).Await() + var list = (await backendManager.ListAsync(result.TaskControl.ProgressToken).ConfigureAwait(false)) .Select(x => Volumes.VolumeBase.ParseFilename(x)) .Where(x => x != null) .Where(x => x.Prefix == m_options.Prefix) @@ -231,8 +231,8 @@ namespace Duplicati.Library.Main { using (LocalDatabase db = new LocalDatabase(m_options.Dbpath, "list-remote", true)) { - IEnumerable dbRemoteVolumes = db.GetRemoteVolumes(); - HashSet dbRemoteFiles = new HashSet(dbRemoteVolumes.Select(x => x.Name)); + var dbRemoteVolumes = db.GetRemoteVolumes(); + var dbRemoteFiles = dbRemoteVolumes.Select(x => x.Name).ToHashSet(); list = list.Where(x => dbRemoteFiles.Contains(x.File.Name)).ToList(); } } @@ -243,7 +243,7 @@ namespace Duplicati.Library.Main { try { - backendManager.DeleteAsync(list[i].File.Name, list[i].File.Size, true, cancelToken).Await(); + await backendManager.DeleteAsync(list[i].File.Name, list[i].File.Size, true, result.TaskControl.ProgressToken).ConfigureAwait(false); } catch (Exception ex) { @@ -261,19 +261,20 @@ namespace Duplicati.Library.Main CheckAutoVacuumInterval(); return RunAction(new CompactResults(), (result, backendManager) => - { - new Operation.CompactHandler(m_options, result).Run(backendManager).Await(); - }); + new Operation.CompactHandler(m_options, result) + .RunAsync(backendManager) + ); } public IRecreateDatabaseResults UpdateDatabaseWithVersions(IFilter filter = null) { var filelistfilter = Operation.RestoreHandler.FilterNumberedFilelist(m_options.Time, m_options.Version, singleTimeMatch: true); - return RunAction(new RecreateDatabaseResults(), ref filter, (result, backendManager) => + return RunAction(new RecreateDatabaseResults(), ref filter, async (result, backendManager) => { using (var h = new Operation.RecreateDatabaseHandler(m_options, result)) - h.RunUpdate(backendManager, filter, filelistfilter, null); + await h.RunUpdateAsync(backendManager, filter, filelistfilter, null) + .ConfigureAwait(false); }); } @@ -282,9 +283,8 @@ namespace Duplicati.Library.Main var t = new string[] { targetpath }; return RunAction(new CreateLogDatabaseResults(), ref t, (result, backendManager) => - { - new Operation.CreateBugReportHandler(t[0], m_options, result).Run(); - }); + new Operation.CreateBugReportHandler(t[0], m_options, result).RunAsync() + ); } public IListChangesResults ListChanges(string baseVersion, string targetVersion, IEnumerable filterstrings = null, IFilter filter = null, Action>> callback = null) @@ -292,17 +292,17 @@ namespace Duplicati.Library.Main var t = new string[] { baseVersion, targetVersion }; return RunAction(new ListChangesResults(), ref t, ref filter, (result, backendManager) => - { - new Operation.ListChangesHandler(m_options, result).Run(t[0], t[1], backendManager, filterstrings, filter, callback); - }); + new Operation.ListChangesHandler(m_options, result) + .RunAsync(t[0], t[1], backendManager, filterstrings, filter, callback) + ); } public IListAffectedResults ListAffected(List args, Action callback = null) { return RunAction(new ListAffectedResults(), (result, backendManager) => - { - new Operation.ListAffected(m_options, result).Run(args, callback); - }); + new Operation.ListAffected(m_options, result) + .RunAsync(args, callback) + ); } public ITestResults Test(long samples = 1) @@ -311,9 +311,9 @@ namespace Duplicati.Library.Main m_options.RawOptions["full-remote-verification"] = "true"; return RunAction(new TestResults(), (result, backendManager) => - { - new Operation.TestHandler(m_options, result).Run(samples, backendManager); - }); + new Operation.TestHandler(m_options, result) + .RunAsync(samples, backendManager) + ); } public ITestFilterResults TestFilter(string[] paths, IFilter filter = null) @@ -326,42 +326,41 @@ namespace Duplicati.Library.Main using (Logging.Log.StartScope(m_messageSink.WriteMessage, x => x.FilterTag.Contains(filtertag))) { return RunAction(new TestFilterResults(), ref paths, ref filter, (result, backendManager) => - { - new Operation.TestFilterHandler(m_options, result).RunAsync(ExpandInputSources(paths, filter), filter).Await(); - }); + new Operation.TestFilterHandler(m_options, result) + .RunAsync(ExpandInputSources(paths, filter), filter) + ); } } public ISystemInfoResults SystemInfo() { return RunAction(new SystemInfoResults(), (result, backendManager) => - { - Operation.SystemInfoHandler.Run(result); - }); + Operation.SystemInfoHandler.RunAsync(result) + ); } public IPurgeFilesResults PurgeFiles(IFilter filter) { return RunAction(new PurgeFilesResults(), (result, backendManager) => - { - new Operation.PurgeFilesHandler(m_options, result).Run(backendManager, filter); - }); + new Operation.PurgeFilesHandler(m_options, result) + .RunAsync(backendManager, filter) + ); } public IListBrokenFilesResults ListBrokenFiles(IFilter filter, Func callbackhandler = null) { return RunAction(new ListBrokenFilesResults(), (result, backendManager) => - { - new Operation.ListBrokenFilesHandler(m_options, result).Run(backendManager, filter, callbackhandler); - }); + new Operation.ListBrokenFilesHandler(m_options, result) + .RunAsync(backendManager, filter, callbackhandler) + ); } public IPurgeBrokenFilesResults PurgeBrokenFiles(IFilter filter) { return RunAction(new PurgeBrokenFilesResults(), (result, backendManager) => - { - new Operation.PurgeBrokenFilesHandler(m_options, result).Run(backendManager, filter); - }); + new Operation.PurgeBrokenFilesHandler(m_options, result) + .RunAsync(backendManager, filter) + ); } public ISendMailResults SendMail() @@ -387,22 +386,24 @@ namespace Duplicati.Library.Main using (Logging.Log.StartScope(m_messageSink.WriteMessage, x => x.FilterTag.Contains(filtertag))) { return RunAction(new SendMailResults(), (result, backendManager) => - { - result.Lines = new string[0]; - System.Threading.Thread.Sleep(5); - }); + Task.Run(() => + { + result.Lines = new string[0]; + System.Threading.Thread.Sleep(5); + }) + ); } } public IVacuumResults Vacuum() { return RunAction(new VacuumResults(), (result, backendManager) => - { - new Operation.VacuumHandler(m_options, result).Run(); - }); + new Operation.VacuumHandler(m_options, result) + .RunAsync() + ); } - private T RunAction(T result, Action method) + private T RunAction(T result, Func method) where T : ISetCommonOptions, ITaskControlProvider, Logging.ILogDestination, IBasicResults, IBackendWriterProvider { var tmp = new string[0]; @@ -410,21 +411,21 @@ namespace Duplicati.Library.Main return RunAction(result, ref tmp, ref tempfilter, method); } - private T RunAction(T result, ref string[] paths, Action method) + private T RunAction(T result, ref string[] paths, Func method) where T : ISetCommonOptions, ITaskControlProvider, Logging.ILogDestination, IBasicResults, IBackendWriterProvider { IFilter tempfilter = null; return RunAction(result, ref paths, ref tempfilter, method); } - private T RunAction(T result, ref IFilter filter, Action method) + private T RunAction(T result, ref IFilter filter, Func method) where T : ISetCommonOptions, ITaskControlProvider, Logging.ILogDestination, IBasicResults, IBackendWriterProvider { var tmp = new string[0]; return RunAction(result, ref tmp, ref filter, method); } - private T RunAction(T result, ref string[] paths, ref IFilter filter, Action method) + private T RunAction(T result, ref string[] paths, ref IFilter filter, Func method) where T : ISetCommonOptions, ITaskControlProvider, Logging.ILogDestination, IBasicResults, IBackendWriterProvider { OnOperationStarted?.Invoke(result); @@ -449,7 +450,7 @@ namespace Duplicati.Library.Main using (m_options.ConcurrencyMaxThreads <= 0 ? null : new CoCoL.CappedThreadedThreadPool(m_options.ConcurrencyMaxThreads)) using (var backend = new Backend.BackendManager(m_backendUrl, m_options, result.BackendWriter, result.TaskControl)) { - method(result, backend); + method(result, backend).Await(); // TODO: Should also have a single shared database connection for all operations // The transactions should be managed inside the connection, and not passed around diff --git a/Duplicati/Library/Main/Database/ReusableTransaction.cs b/Duplicati/Library/Main/Database/ReusableTransaction.cs new file mode 100644 index 000000000..d83e6c9b2 --- /dev/null +++ b/Duplicati/Library/Main/Database/ReusableTransaction.cs @@ -0,0 +1,70 @@ +using System; +using System.Data; + +#nullable enable + +namespace Duplicati.Library.Main.Database; + +/// +/// Wraps a transaction so it can be comitted and restarted +/// +internal class ReusableTransaction : IDisposable +{ + /// + /// The tag used for logging + /// + private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(LocalDatabase)); + + /// + /// The database to use + /// + private readonly LocalDatabase m_db; + /// + /// The current transaction + /// + private IDbTransaction? m_transaction; + + /// + /// Creates a new reusable transaction + /// + /// The database to use + public ReusableTransaction(LocalDatabase db, IDbTransaction? transaction = null) + { + m_db = db; + m_transaction = transaction ?? db.BeginTransaction(); + } + + /// + /// The current transaction + /// + public IDbTransaction Transaction => m_transaction ?? throw new InvalidOperationException("Transaction is disposed"); + + /// + /// Commits the current transaction and optionally restarts it + /// + /// The log message to use + /// True if the transaction should be restarted + public void Commit(string? message = null, bool restart = true) + { + if (m_transaction == null) + throw new InvalidOperationException("Transaction is already disposed"); + + if (m_transaction != null) + { + using (var timer = string.IsNullOrWhiteSpace(message) ? null : new Logging.Timer(LOGTAG, message, "CommitTransaction")) + m_transaction.Commit(); + m_transaction.Dispose(); + m_transaction = null; + } + + if (restart) + m_transaction = m_db.BeginTransaction(); + } + + /// + public void Dispose() + { + m_transaction?.Dispose(); + m_transaction = null; + } +} diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 967d8c3c4..01af2633c 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -57,8 +57,6 @@ namespace Duplicati.Library.Main.Operation private readonly Options m_options; private LocalBackupDatabase m_database; - private IDbTransaction m_transaction; - private IFilter m_filter; private readonly BackupResults m_result; @@ -346,7 +344,9 @@ namespace Duplicati.Library.Main.Operation database = null; result.SetDatabase(null); - new RepairHandler(options, (RepairResults)result.RepairResults).Run(backendManager, null); + await new RepairHandler(options, (RepairResults)result.RepairResults) + .RunAsync(backendManager, null) + .ConfigureAwait(false); // Re-open the database and backend manager database = new LocalBackupDatabase(options.Dbpath, options); @@ -469,7 +469,7 @@ namespace Duplicati.Library.Main.Operation } } - private async Task CompactIfRequired(IBackendManager backendManager, long lastVolumeSize) + private async Task CompactIfRequired(IBackendManager backendManager, ReusableTransaction transaction, long lastVolumeSize) { var currentIsSmall = lastVolumeSize != -1 && lastVolumeSize <= m_options.SmallFileSize; @@ -478,7 +478,9 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Delete); m_result.DeleteResults = new DeleteResults(m_result); using (var db = new LocalDeleteDatabase(m_database)) - new DeleteHandler(m_options, (DeleteResults)m_result.DeleteResults).DoRun(db, ref m_transaction, true, currentIsSmall, backendManager); + await new DeleteHandler(m_options, (DeleteResults)m_result.DeleteResults) + .DoRunAsync(db, transaction, true, currentIsSmall, backendManager) + .ConfigureAwait(false); } else if (currentIsSmall && !m_options.NoAutoCompact) @@ -486,19 +488,18 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Compact); m_result.CompactResults = new CompactResults(m_result); using (var db = new LocalDeleteDatabase(m_database)) - { - (var _, var tr) = await new CompactHandler(m_options, (CompactResults)m_result.CompactResults).DoCompact(db, true, m_transaction, backendManager); - m_transaction = tr; - } + await new CompactHandler(m_options, (CompactResults)m_result.CompactResults) + .DoCompactAsync(db, true, transaction, backendManager) + .ConfigureAwait(false); } } - private async Task PostBackupVerification(string currentFilelistVolume, string previousTemporaryFilelist, IBackendManager backendManager) + private async Task PostBackupVerification(string currentFilelistVolume, string previousTemporaryFilelist, IBackendManager backendManager, ReusableTransaction rtr) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PostBackupVerify); using (new Logging.Timer(LOGTAG, "AfterBackupVerify", "AfterBackupVerify")) await FilelistProcessor.VerifyRemoteList(backendManager, m_options, m_database, m_result.BackendWriter, [currentFilelistVolume], [previousTemporaryFilelist], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); - await backendManager.WaitForEmptyAsync(m_database, m_transaction, m_taskReader.ProgressToken); + await backendManager.WaitForEmptyAsync(m_database, rtr.Transaction, m_taskReader.ProgressToken); // Calculate the number of samples to test, using the largest number of file of a given type long remoteVolumeCount = m_database.GetRemoteVolumes() @@ -515,7 +516,8 @@ namespace Duplicati.Library.Main.Operation using (var testdb = new LocalTestDatabase(m_database)) await new TestHandler(m_options, (TestResults)m_result.TestResults) - .DoRun(samplesToTest, testdb, backendManager).ConfigureAwait(false); + .DoRunAsync(samplesToTest, testdb, backendManager) + .ConfigureAwait(false); } } @@ -696,66 +698,40 @@ namespace Duplicati.Library.Main.Operation // Wait for upload completion m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_WaitForUpload); - var lastVolumeSize = await FlushBackend(m_database, m_transaction, m_result, backendManager).ConfigureAwait(false); + var lastVolumeSize = await FlushBackend(m_database, null, m_result, backendManager).ConfigureAwait(false); if (!m_options.Dryrun) database.TerminatedWithActiveUploads = false; // Make sure we have the database up-to-date await db.CommitTransactionAsync("CommitAfterUpload", false); - // TODO: Remove this later - m_transaction = m_database.BeginTransaction(); - - try + using (var rtr = new ReusableTransaction(m_database)) { // If this throws, we should roll back the transaction if (await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - await CompactIfRequired(backendManager, lastVolumeSize); + await CompactIfRequired(backendManager, rtr, lastVolumeSize); if (m_options.UploadVerificationFile && await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_VerificationUpload); - await FilelistProcessor.UploadVerificationFile(backendManager, m_options, m_database, m_transaction); + await FilelistProcessor.UploadVerificationFile(backendManager, m_options, m_database, rtr.Transaction); } - } - catch - { - try + + if (!m_options.Dryrun) { - m_transaction.Rollback(); - m_transaction.Dispose(); - m_transaction = null; - } - catch (Exception ex) - { - Logging.Log.WriteErrorMessage(LOGTAG, "RollbackError", ex, "Rollback error: {0}", ex.Message); - } - - // Re-throw the original exception - throw; - } - - if (m_options.Dryrun) - { - m_transaction.Rollback(); - m_transaction = null; - } - else - { - using (new Logging.Timer(LOGTAG, "CommitFinalizingBackup", "CommitFinalizingBackup")) - m_transaction.Commit(); - - m_transaction = null; - - if (await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - { - if (m_options.NoBackendverification) - await UpdateStorageStatsFromDatabase(m_result, m_database, m_options, backendManager, m_taskReader.ProgressToken).ConfigureAwait(false); - else - await PostBackupVerification(filesetvolume.RemoteFilename, lastTempFilelist.Name, backendManager).ConfigureAwait(false); + rtr.Commit("CommitFinalizingBackup"); + if (await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + { + if (m_options.NoBackendverification) + await UpdateStorageStatsFromDatabase(m_result, m_database, m_options, backendManager, m_taskReader.ProgressToken).ConfigureAwait(false); + else + await PostBackupVerification(filesetvolume.RemoteFilename, lastTempFilelist.Name, backendManager, rtr).ConfigureAwait(false); + } + rtr.Commit("CommitAfterVerification", restart: false); } } + m_database.WriteResults(); m_database.PurgeLogData(m_options.LogRetention); m_database.PurgeDeletedVolumes(DateTime.UtcNow); @@ -763,7 +739,8 @@ namespace Duplicati.Library.Main.Operation if (m_options.AutoVacuum) { m_result.VacuumResults = new VacuumResults(m_result); - new VacuumHandler(m_options, (VacuumResults)m_result.VacuumResults).Run(); + await new VacuumHandler(m_options, (VacuumResults)m_result.VacuumResults) + .RunAsync(); } m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Complete); return; diff --git a/Duplicati/Library/Main/Operation/CompactHandler.cs b/Duplicati/Library/Main/Operation/CompactHandler.cs index da38fa8a1..2f4b3f02a 100644 --- a/Duplicati/Library/Main/Operation/CompactHandler.cs +++ b/Duplicati/Library/Main/Operation/CompactHandler.cs @@ -24,7 +24,9 @@ using Duplicati.Library.Main.Volumes; using Duplicati.Library.Utility; using System; using System.Collections.Generic; +using System.Data; using System.Linq; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; @@ -45,76 +47,59 @@ namespace Duplicati.Library.Main.Operation m_result = result; } - public virtual async Task Run(IBackendManager backendManager) + public async Task RunAsync(IBackendManager backendManager) { if (!System.IO.File.Exists(m_options.Dbpath)) throw new Exception(string.Format("Database file does not exist: {0}", m_options.Dbpath)); using (var db = new LocalDeleteDatabase(m_options.Dbpath, "Compact")) + using (var tr = new ReusableTransaction(db)) { - var tr = db.BeginTransaction(); - try + m_result.SetDatabase(db); + Utility.UpdateOptionsFromDb(db, m_options); + Utility.VerifyOptionsAndUpdateDatabase(db, m_options); + + var changed = await DoCompactAsync(db, false, tr, backendManager).ConfigureAwait(false); + + if (changed && m_options.UploadVerificationFile) + await FilelistProcessor.UploadVerificationFile(backendManager, m_options, db, null); + + if (!m_options.Dryrun) { - m_result.SetDatabase(db); - Utility.UpdateOptionsFromDb(db, m_options); - Utility.VerifyOptionsAndUpdateDatabase(db, m_options); - - // TODO: Handle transaction scopes better - var (changed, tr2) = DoCompact(db, false, tr, backendManager).Await(); - tr = tr2; - - if (changed && m_options.UploadVerificationFile) - await FilelistProcessor.UploadVerificationFile(backendManager, m_options, db, null); - - if (!m_options.Dryrun) + tr.Commit("CommitCompact", restart: false); + if (changed) { - using (new Logging.Timer(LOGTAG, "CommitCompact", "CommitCompact")) - tr.Commit(); - if (changed) + db.WriteResults(); + if (m_options.AutoVacuum) { - db.WriteResults(); - if (m_options.AutoVacuum) - { - m_result.VacuumResults = new VacuumResults(m_result); - new VacuumHandler(m_options, (VacuumResults)m_result.VacuumResults).Run(); - } + m_result.VacuumResults = new VacuumResults(m_result); + await new VacuumHandler(m_options, (VacuumResults)m_result.VacuumResults).RunAsync().ConfigureAwait(false); } } - else - tr.Rollback(); - - tr = null; - } - finally - { - if (tr != null) - try { tr.Rollback(); } - catch { } } } } - internal async Task<(bool, System.Data.IDbTransaction)> DoCompact(LocalDeleteDatabase db, bool hasVerifiedBackend, System.Data.IDbTransaction transaction, IBackendManager backendManager) + internal async Task DoCompactAsync(LocalDeleteDatabase db, bool hasVerifiedBackend, ReusableTransaction rtr, IBackendManager backendManager) { - var cancellationToken = CancellationToken.None; - var report = db.GetCompactReport(m_options.VolumeSize, m_options.Threshold, m_options.SmallFileSize, m_options.SmallFileMaxCount, transaction); + var report = db.GetCompactReport(m_options.VolumeSize, m_options.Threshold, m_options.SmallFileSize, m_options.SmallFileMaxCount, rtr.Transaction); report.ReportCompactData(); if (report.ShouldReclaim || report.ShouldCompact) { // Workaround where we allow a running backendmanager to be used if (!hasVerifiedBackend) - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, true, FilelistProcessor.VerifyMode.VerifyStrict, transaction).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, true, FilelistProcessor.VerifyMode.VerifyStrict, rtr.Transaction).ConfigureAwait(false); var newvol = new BlockVolumeWriter(m_options); - newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, transaction); + newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, rtr.Transaction); IndexVolumeWriter newvolindex = null; if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None) { newvolindex = new IndexVolumeWriter(m_options); - newvolindex.VolumeID = db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, transaction); - db.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, transaction); + newvolindex.VolumeID = db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, rtr.Transaction); + db.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, rtr.Transaction); } var blocksInVolume = 0L; @@ -137,7 +122,8 @@ namespace Duplicati.Library.Main.Operation .Cast() .ToList(); } - deletedVolumes.AddRange(DoDelete(db, backendManager, fullyDeleteable, ref transaction, cancellationToken)); + await foreach (var d in DoDelete(db, backendManager, fullyDeleteable, rtr, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + deletedVolumes.Add(d); // This list is used to pick up unused volumes, // so they can be deleted once the upload of the @@ -162,17 +148,17 @@ namespace Duplicati.Library.Main.Operation .ToList(); } - using (var q = db.CreateBlockQueryHelper(transaction)) + using (var q = db.CreateBlockQueryHelper(rtr.Transaction)) { await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(volumesToDownload, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { using (tmpfile) { var entry = new RemoteVolume(name, hash, size); - if (!m_result.TaskControl.ProgressRendevouz().Await()) + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - backendManager.WaitForEmptyAsync(db, transaction, cancellationToken).Await(); - return (false, transaction); + await backendManager.WaitForEmptyAsync(db, rtr.Transaction, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + return false; } downloadedVolumes.Add(new KeyValuePair(entry.Name, entry.Size)); @@ -181,46 +167,43 @@ namespace Duplicati.Library.Main.Operation { foreach (var e in f.Blocks) { - if (q.UseBlock(e.Key, e.Value, transaction)) + if (q.UseBlock(e.Key, e.Value, rtr.Transaction)) { //TODO: How do we get the compression hint? Reverse query for filename in db? var s = f.ReadBlock(e.Key, buffer); if (s != e.Value) throw new Exception(string.Format("Size mismatch problem for block {0}, {1} vs {2}", e.Key, s, e.Value)); - newvol.AddBlock(e.Key, buffer, 0, s, Duplicati.Library.Interface.CompressionHint.Compressible); + newvol.AddBlock(e.Key, buffer, 0, s, Interface.CompressionHint.Compressible); if (newvolindex != null) newvolindex.AddBlock(e.Key, e.Value); - db.RegisterDuplicatedBlock(e.Key, e.Value, newvol.VolumeID, transaction); + db.RegisterDuplicatedBlock(e.Key, e.Value, newvol.VolumeID, rtr.Transaction); blocksInVolume++; if (newvol.Filesize > (m_options.VolumeSize - m_options.Blocksize)) { - FinishVolumeAndUpload(db, backendManager, newvol, newvolindex, uploadedVolumes); + await FinishVolumeAndUpload(db, backendManager, newvol, newvolindex, uploadedVolumes); newvol = new BlockVolumeWriter(m_options); - newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, transaction); + newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, rtr.Transaction); if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None) { newvolindex = new IndexVolumeWriter(m_options); - newvolindex.VolumeID = db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, transaction); - db.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, transaction); + newvolindex.VolumeID = db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, rtr.Transaction); + db.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, rtr.Transaction); newvolindex.StartVolume(newvol.RemoteFilename); } blocksInVolume = 0; // Wait for the backend to catch up - backendManager.WaitForEmptyAsync(db, transaction, cancellationToken).Await(); + await backendManager.WaitForEmptyAsync(db, rtr.Transaction, m_result.TaskControl.ProgressToken).ConfigureAwait(false); // Commit as we have uploaded a volume if (!m_options.Dryrun) - { - transaction.Commit(); - transaction = db.BeginTransaction(); - } + rtr.Commit(); } } } @@ -232,14 +215,14 @@ namespace Duplicati.Library.Main.Operation if (blocksInVolume > 0) { - FinishVolumeAndUpload(db, backendManager, newvol, newvolindex, uploadedVolumes); + await FinishVolumeAndUpload(db, backendManager, newvol, newvolindex, uploadedVolumes).ConfigureAwait(false); } else { - db.RemoveRemoteVolume(newvol.RemoteFilename, transaction); + db.RemoveRemoteVolume(newvol.RemoteFilename, rtr.Transaction); if (newvolindex != null) { - db.RemoveRemoteVolume(newvolindex.RemoteFilename, transaction); + db.RemoveRemoteVolume(newvolindex.RemoteFilename, rtr.Transaction); newvolindex.FinishVolume(null, 0); } } @@ -255,7 +238,8 @@ namespace Duplicati.Library.Main.Operation newvol.Dispose(); } - deletedVolumes.AddRange(DoDelete(db, backendManager, deleteableVolumes, ref transaction, cancellationToken)); + await foreach (var d in DoDelete(db, backendManager, deleteableVolumes, rtr, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + deletedVolumes.Add(d); var downloadSize = downloadedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a, x) => a + x.Value); var deletedSize = deletedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a, x) => a + x.Value); @@ -299,25 +283,25 @@ namespace Duplicati.Library.Main.Operation Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize - m_result.UploadedFileSize)); } - backendManager.WaitForEmptyAsync(db, transaction, cancellationToken).Await(); + await backendManager.WaitForEmptyAsync(db, rtr.Transaction, m_result.TaskControl.ProgressToken).ConfigureAwait(false); m_result.EndTime = DateTime.UtcNow; - return ((m_result.DeletedFileCount + m_result.UploadedFileCount) > 0, transaction); + return (m_result.DeletedFileCount + m_result.UploadedFileCount) > 0; } else { m_result.EndTime = DateTime.UtcNow; - return (false, transaction); + return false; } } - private IEnumerable> DoDelete(LocalDeleteDatabase db, IBackendManager backend, IEnumerable deleteableVolumes, ref System.Data.IDbTransaction transaction, CancellationToken cancellationToken) + private async IAsyncEnumerable> DoDelete(LocalDeleteDatabase db, IBackendManager backend, IEnumerable deleteableVolumes, ReusableTransaction rtr, [EnumeratorCancellation] CancellationToken cancellationToken) { // Find volumes that can be deleted - var remoteFilesToRemove = db.ReOrderDeleteableVolumes(deleteableVolumes, transaction).ToList(); + var remoteFilesToRemove = db.ReOrderDeleteableVolumes(deleteableVolumes, rtr.Transaction).ToList(); // Make sure we do not re-assign blocks to any of the volumes we are about to delete - var toRemoveVolumeIds = db.GetRemoteVolumeIDs(remoteFilesToRemove.Select(x => x.Name), transaction) + var toRemoveVolumeIds = db.GetRemoteVolumeIDs(remoteFilesToRemove.Select(x => x.Name), rtr.Transaction) .Select(x => x.Value) .Distinct() .ToList(); @@ -325,23 +309,21 @@ namespace Duplicati.Library.Main.Operation // Mark all volumes and relevant index files as disposable foreach (var f in remoteFilesToRemove) { - db.PrepareForDelete(f.Name, toRemoveVolumeIds, transaction); - db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, transaction); + db.PrepareForDelete(f.Name, toRemoveVolumeIds, rtr.Transaction); + db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, rtr.Transaction); } // Before we commit the current state, make sure the backend has caught up - backend.WaitForEmptyAsync(db, transaction, cancellationToken).Await(); + await backend.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - { - transaction.Commit(); - transaction = db.BeginTransaction(); - } + rtr.Commit(); - return PerformDelete(backend, remoteFilesToRemove, cancellationToken); + await foreach (var d in PerformDelete(backend, remoteFilesToRemove, cancellationToken).ConfigureAwait(false)) + yield return d; } - private void FinishVolumeAndUpload(LocalDeleteDatabase db, IBackendManager backendManager, BlockVolumeWriter newvol, IndexVolumeWriter newvolindex, List> uploadedVolumes) + private async Task FinishVolumeAndUpload(LocalDeleteDatabase db, IBackendManager backendManager, BlockVolumeWriter newvol, IndexVolumeWriter newvolindex, List> uploadedVolumes) { Action indexVolumeFinished = () => { @@ -358,17 +340,17 @@ namespace Duplicati.Library.Main.Operation if (newvolindex != null) uploadedVolumes.Add(new KeyValuePair(newvolindex.RemoteFilename, newvolindex.Filesize)); if (!m_options.Dryrun) - backendManager.PutAsync(newvol, newvolindex, indexVolumeFinished, false, CancellationToken.None).Await(); + await backendManager.PutAsync(newvol, newvolindex, indexVolumeFinished, false, m_result.TaskControl.ProgressToken).ConfigureAwait(false); else Logging.Log.WriteDryrunMessage(LOGTAG, "WouldUploadGeneratedBlockset", "Would upload generated blockset of size {0}", Library.Utility.Utility.FormatSizeString(newvol.Filesize)); } - private IEnumerable> PerformDelete(IBackendManager backendManager, IEnumerable list, CancellationToken cancelToken) + private async IAsyncEnumerable> PerformDelete(IBackendManager backendManager, IEnumerable list, [EnumeratorCancellation] CancellationToken cancelToken) { foreach (var f in list) { if (!m_options.Dryrun) - backendManager.DeleteAsync(f.Name, f.Size, false, cancelToken).Await(); + await backendManager.DeleteAsync(f.Name, f.Size, false, cancelToken).ConfigureAwait(false); else Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteRemoteFile", "Would delete remote file: {0}, size: {1}", f.Name, Library.Utility.Utility.FormatSizeString(f.Size)); diff --git a/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs b/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs index cc0896d4f..740f76734 100644 --- a/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs +++ b/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs @@ -1,28 +1,30 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.IO; +using System.Threading.Tasks; using Duplicati.Library.Interface; using Duplicati.Library.Main.Database; +using Duplicati.Library.Utility; namespace Duplicati.Library.Main.Operation { @@ -43,18 +45,18 @@ namespace Duplicati.Library.Main.Operation m_result = result; } - public void Run() + public async Task RunAsync() { - var ext = System.IO.Path.GetExtension(m_targetpath); + var ext = Path.GetExtension(m_targetpath); var module = m_options.CompressionModule; if (ext == "" || string.Compare(ext, 1, module, 0, module.Length, StringComparison.OrdinalIgnoreCase) != 0) m_targetpath = m_targetpath + "." + module; - if (System.IO.File.Exists(m_targetpath)) + if (File.Exists(m_targetpath)) throw new UserInformationException(string.Format("Output file already exists, not overwriting: {0}", m_targetpath), "BugReportTargetAlreadyExists"); - if (!System.IO.File.Exists(m_options.Dbpath)) + if (!File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "BugReportSourceDatabaseNotFound"); m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.BugReport_Running); @@ -62,27 +64,25 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteInformationMessage(LOGTAG, "ScrubbingFilenames", "Scrubbing filenames from database, this may take a while, please wait"); - using (var tmp = new Library.Utility.TempFile()) + using (var tmp = new TempFile()) { - System.IO.File.Copy(m_options.Dbpath, tmp, true); + File.Copy(m_options.Dbpath, tmp, true); using (var db = new LocalBugReportDatabase(tmp)) { m_result.SetDatabase(db); db.Fix(); if (m_options.AutoVacuum) - { db.Vacuum(); - } } - using (var stream = new System.IO.FileStream(m_targetpath, FileMode.Create, FileAccess.Write, FileShare.Read)) + using (var stream = new FileStream(m_targetpath, FileMode.Create, FileAccess.Write, FileShare.Read)) using (ICompression cm = DynamicLoader.CompressionLoader.GetModule(module, stream, Interface.ArchiveMode.Write, m_options.RawOptions)) { - using (var cs = cm.CreateFile("log-database.sqlite", Duplicati.Library.Interface.CompressionHint.Compressible, DateTime.UtcNow)) - using (var fs = System.IO.File.Open(tmp, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.ReadWrite)) - Library.Utility.Utility.CopyStream(fs, cs); + using (var cs = cm.CreateFile("log-database.sqlite", CompressionHint.Compressible, DateTime.UtcNow)) + using (var fs = File.Open(tmp, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) + await Library.Utility.Utility.CopyStreamAsync(fs, cs, m_result.TaskControl.ProgressToken).ConfigureAwait(false); - using (var cs = new System.IO.StreamWriter(cm.CreateFile("system-info.txt", Duplicati.Library.Interface.CompressionHint.Compressible, DateTime.UtcNow))) + using (var cs = new StreamWriter(cm.CreateFile("system-info.txt", CompressionHint.Compressible, DateTime.UtcNow))) foreach (var line in SystemInfoHandler.GetSystemInfo()) cs.WriteLine(line); } diff --git a/Duplicati/Library/Main/Operation/DeleteHandler.cs b/Duplicati/Library/Main/Operation/DeleteHandler.cs index cc19cf748..1d89e43ed 100644 --- a/Duplicati/Library/Main/Operation/DeleteHandler.cs +++ b/Duplicati/Library/Main/Operation/DeleteHandler.cs @@ -26,6 +26,7 @@ using Duplicati.Library.Interface; using Duplicati.Library.Main.Database; using Duplicati.Library.Utility; using System.Threading; +using System.Threading.Tasks; namespace Duplicati.Library.Main.Operation { @@ -45,59 +46,43 @@ namespace Duplicati.Library.Main.Operation m_result = result; } - public void Run(IBackendManager backendManager) + public async Task RunAsync(IBackendManager backendManager) { if (!System.IO.File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseFileMissing"); - using (var db = new Database.LocalDeleteDatabase(m_options.Dbpath, "Delete")) + using (var db = new LocalDeleteDatabase(m_options.Dbpath, "Delete")) + using (var tr = new ReusableTransaction(db)) { - var tr = db.BeginTransaction(); - try + m_result.SetDatabase(db); + Utility.UpdateOptionsFromDb(db, m_options); + Utility.VerifyOptionsAndUpdateDatabase(db, m_options); + + await DoRunAsync(db, tr, false, false, backendManager).ConfigureAwait(false); + + if (!m_options.Dryrun) { - m_result.SetDatabase(db); - Utility.UpdateOptionsFromDb(db, m_options); - Utility.VerifyOptionsAndUpdateDatabase(db, m_options); - - DoRun(db, ref tr, false, false, backendManager); - - if (!m_options.Dryrun) - { - using (new Logging.Timer(LOGTAG, "CommitDelete", "CommitDelete")) - tr.Commit(); - - db.WriteResults(); - } - else - tr.Rollback(); - - tr = null; - } - finally - { - if (tr != null) - try { tr.Rollback(); } - catch { } + tr.Commit("ComitDelete", restart: false); + db.WriteResults(); } } } - public void DoRun(Database.LocalDeleteDatabase db, ref System.Data.IDbTransaction transaction, bool hasVerifiedBackend, bool forceCompact, IBackendManager backendManager) + public async Task DoRunAsync(LocalDeleteDatabase db, ReusableTransaction rtr, bool hasVerifiedBackend, bool forceCompact, IBackendManager backendManager) { - CancellationToken cancellationToken = CancellationToken.None; if (!hasVerifiedBackend) - FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict, transaction).Await(); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict, rtr.Transaction).ConfigureAwait(false); IListResultFileset[] filesets = db.FilesetsWithBackupVersion.ToArray(); List versionsToDelete = [ - .. new SpecificVersionsRemover(this.m_options).GetFilesetsToDelete(filesets), - .. new KeepTimeRemover(this.m_options).GetFilesetsToDelete(filesets), - .. new RetentionPolicyRemover(this.m_options).GetFilesetsToDelete(filesets), + .. new SpecificVersionsRemover(m_options).GetFilesetsToDelete(filesets), + .. new KeepTimeRemover(m_options).GetFilesetsToDelete(filesets), + .. new RetentionPolicyRemover(m_options).GetFilesetsToDelete(filesets), ]; // When determining the number of full versions to keep, we need to ignore the versions already marked for removal. - versionsToDelete.AddRange(new KeepVersionsRemover(this.m_options).GetFilesetsToDelete(filesets.Except(versionsToDelete))); + versionsToDelete.AddRange(new KeepVersionsRemover(m_options).GetFilesetsToDelete(filesets.Except(versionsToDelete))); if (!m_options.AllowFullRemoval && filesets.Length == versionsToDelete.Count) { @@ -108,31 +93,28 @@ namespace Duplicati.Library.Main.Operation if (versionsToDelete.Count > 0) Logging.Log.WriteInformationMessage(LOGTAG, "DeleteRemoteFileset", "Deleting {0} remote fileset(s) ...", versionsToDelete.Count); - var lst = db.DropFilesetsFromTable(versionsToDelete.Select(x => x.Time).ToArray(), transaction).ToArray(); + var lst = db.DropFilesetsFromTable(versionsToDelete.Select(x => x.Time).ToArray(), rtr.Transaction).ToArray(); foreach (var f in lst) - db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, transaction); + db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, rtr.Transaction); if (!m_options.Dryrun) - { - transaction.Commit(); - transaction = db.BeginTransaction(); - } + rtr.Commit(); foreach (var f in lst) { - if (!m_result.TaskControl.ProgressRendevouz().Await()) + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - backendManager.WaitForEmptyAsync(db, transaction, cancellationToken).Await(); + await backendManager.WaitForEmptyAsync(db, rtr.Transaction, m_result.TaskControl.ProgressToken).ConfigureAwait(false); return; } if (!m_options.Dryrun) - backendManager.DeleteAsync(f.Key, f.Value, false, cancellationToken).Await(); + await backendManager.DeleteAsync(f.Key, f.Value, false, m_result.TaskControl.ProgressToken).ConfigureAwait(false); else Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteRemoteFileset", "Would delete remote fileset: {0}", f.Key); } - backendManager.WaitForEmptyAsync(db, transaction, cancellationToken).Await(); + await backendManager.WaitForEmptyAsync(db, rtr.Transaction, m_result.TaskControl.ProgressToken).ConfigureAwait(false); var count = lst.Length; if (!m_options.Dryrun) @@ -157,8 +139,8 @@ namespace Duplicati.Library.Main.Operation if (!m_options.NoAutoCompact && (forceCompact || versionsToDelete.Count > 0)) { m_result.CompactResults = new CompactResults(m_result); - var (_, tr) = new CompactHandler(m_options, (CompactResults)m_result.CompactResults).DoCompact(db, true, transaction, backendManager).Await(); - transaction = tr; + await new CompactHandler(m_options, (CompactResults)m_result.CompactResults) + .DoCompactAsync(db, true, rtr, backendManager).ConfigureAwait(false); } m_result.SetResults(versionsToDelete.Select(v => new Tuple(v.Version, v.Time)), m_options.Dryrun); diff --git a/Duplicati/Library/Main/Operation/ListAffected.cs b/Duplicati/Library/Main/Operation/ListAffected.cs index 1608fc5b1..50c1d3694 100644 --- a/Duplicati/Library/Main/Operation/ListAffected.cs +++ b/Duplicati/Library/Main/Operation/ListAffected.cs @@ -1,28 +1,30 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Linq; using System.Collections.Generic; using Duplicati.Library.Interface; +using System.Threading.Tasks; +using System.IO; namespace Duplicati.Library.Main.Operation { @@ -36,13 +38,13 @@ namespace Duplicati.Library.Main.Operation m_options = options; m_result = result; } - - public void Run(List args, Action callback = null) + + public Task RunAsync(List args, Action callback = null) { - if (!System.IO.File.Exists(m_options.Dbpath)) + if (!File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - using(var db = new Database.LocalListAffectedDatabase(m_options.Dbpath)) + using (var db = new Database.LocalListAffectedDatabase(m_options.Dbpath)) { m_result.SetDatabase(db); if (callback == null) @@ -66,6 +68,8 @@ namespace Duplicati.Library.Main.Operation callback(m_result); } } + + return Task.CompletedTask; } } } diff --git a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs index 0df3125b6..7e82c7754 100644 --- a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs @@ -22,7 +22,9 @@ using System; using System.Collections; using System.Collections.Generic; +using System.IO; using System.Linq; +using System.Threading.Tasks; using Duplicati.Library.Interface; using Duplicati.Library.Utility; @@ -43,19 +45,19 @@ namespace Duplicati.Library.Main.Operation m_result = result; } - public void Run(IBackendManager backendManager, IFilter filter, Func callbackhandler = null) + public async Task RunAsync(IBackendManager backendManager, IFilter filter, Func callbackhandler = null) { - if (!System.IO.File.Exists(m_options.Dbpath)) + if (!File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); using (var db = new Database.LocalListBrokenFilesDatabase(m_options.Dbpath)) using (var tr = db.BeginTransaction()) - DoRun(backendManager, db, tr, filter, callbackhandler); + await DoRunAsync(backendManager, db, tr, filter, callbackhandler).ConfigureAwait(false); } - public static Tuple[] GetBrokenFilesetsFromRemote(IBackendManager backendManager, BasicResults result, Database.LocalListBrokenFilesDatabase db, System.Data.IDbTransaction transaction, Options options, out List missing) + public static async Task<(Tuple[], List Missing)> GetBrokenFilesetsFromRemote(IBackendManager backendManager, BasicResults result, Database.LocalListBrokenFilesDatabase db, System.Data.IDbTransaction transaction, Options options) { - missing = null; + List missing = null; var brokensets = db.GetBrokenFilesets(options.Time, options.Version, transaction).ToArray(); if (brokensets.Length == 0) @@ -65,7 +67,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesetsInDatabase", "No broken filesets found in database, checking for missing remote files"); - var remotestate = FilelistProcessor.RemoteListAnalysis(backendManager, options, db, result.BackendWriter, null, null, FilelistProcessor.VerifyMode.VerifyOnly).Await(); + var remotestate = await FilelistProcessor.RemoteListAnalysis(backendManager, options, db, result.BackendWriter, null, null, FilelistProcessor.VerifyMode.VerifyOnly).ConfigureAwait(false); if (!remotestate.ParsedVolumes.Any()) throw new UserInformationException("No remote volumes were found, refusing purge", "CannotPurgeWithNoRemoteVolumes"); @@ -73,7 +75,7 @@ namespace Duplicati.Library.Main.Operation if (missing.Count == 0) { Logging.Log.WriteInformationMessage(LOGTAG, "NoMissingFilesFound", "Skipping operation because no files were found to be missing, and no filesets were recorded as broken."); - return null; + return (null, missing); } // Mark all volumes as disposable @@ -87,10 +89,10 @@ namespace Duplicati.Library.Main.Operation brokensets = db.GetBrokenFilesets(options.Time, options.Version, transaction).ToArray(); } - return brokensets; + return (brokensets, missing); } - private void DoRun(IBackendManager backendManager, Database.LocalListBrokenFilesDatabase db, System.Data.IDbTransaction transaction, Library.Utility.IFilter filter, Func callbackhandler) + private async Task DoRunAsync(IBackendManager backendManager, Database.LocalListBrokenFilesDatabase db, System.Data.IDbTransaction transaction, IFilter filter, Func callbackhandler) { if (filter != null && !filter.Empty) throw new UserInformationException("Filters are not supported for this operation", "FiltersAreNotSupportedForListBrokenFiles"); @@ -98,14 +100,13 @@ namespace Duplicati.Library.Main.Operation if (db.PartiallyRecreated) throw new UserInformationException("The command does not work on partially recreated databases", "ListBrokenFilesDoesNotWorkOnPartialDatabase"); - List missing; - var brokensets = GetBrokenFilesetsFromRemote(backendManager, m_result, db, transaction, m_options, out missing); + (var brokensets, var missing) = await GetBrokenFilesetsFromRemote(backendManager, m_result, db, transaction, m_options).ConfigureAwait(false); if (brokensets == null) return; if (brokensets.Length == 0) { - m_result.BrokenFiles = new Tuple>>[0]; + m_result.BrokenFiles = []; if (missing == null) Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesets", "Found no broken filesets"); diff --git a/Duplicati/Library/Main/Operation/ListChangesHandler.cs b/Duplicati/Library/Main/Operation/ListChangesHandler.cs index 037532e38..44ac5bca8 100644 --- a/Duplicati/Library/Main/Operation/ListChangesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListChangesHandler.cs @@ -23,6 +23,7 @@ using System; using System.Collections.Generic; using System.Linq; using System.Threading; +using System.Threading.Tasks; using Duplicati.Library.Interface; using Duplicati.Library.Utility; @@ -50,7 +51,7 @@ namespace Duplicati.Library.Main.Operation Tuple res; if (!long.TryParse(value, out indexValue)) { - var t = Library.Utility.Timeparser.ParseTimeInterval(value, DateTime.Now, true); + var t = Timeparser.ParseTimeInterval(value, DateTime.Now, true); res = list.OrderBy(x => Math.Abs((x.Item2 - t).Ticks)).First(); } else @@ -64,10 +65,9 @@ namespace Duplicati.Library.Main.Operation return res; } - public void Run(string baseVersion, string compareVersion, IBackendManager backendManager, IEnumerable filterstrings, IFilter compositefilter, Action>> callback) + public async Task RunAsync(string baseVersion, string compareVersion, IBackendManager backendManager, IEnumerable filterstrings, IFilter compositefilter, Action>> callback) { - var cancellationToken = CancellationToken.None; - var filter = Library.Utility.JoinedFilterExpression.Join(new Library.Utility.FilterExpression(filterstrings), compositefilter); + var filter = JoinedFilterExpression.Join(new FilterExpression(filterstrings), compositefilter); var useLocalDb = !m_options.NoLocalDb && System.IO.File.Exists(m_options.Dbpath); baseVersion = string.IsNullOrEmpty(baseVersion) ? "1" : baseVersion; @@ -79,7 +79,7 @@ namespace Duplicati.Library.Main.Operation DateTime baseVersionTime; DateTime compareVersionTime; - using (var tmpdb = useLocalDb ? null : new Library.Utility.TempFile()) + using (var tmpdb = useLocalDb ? null : new TempFile()) using (var db = new Database.LocalListChangesDatabase(useLocalDb ? m_options.Dbpath : (string)tmpdb)) using (var storageKeeper = db.CreateStorageHelper()) { @@ -106,11 +106,11 @@ namespace Duplicati.Library.Main.Operation { Logging.Log.WriteInformationMessage(LOGTAG, "NoLocalDatabase", "No local database, accessing remote store"); - var parsedlist = (from n in backendManager.ListAsync(cancellationToken).Await() - let p = Volumes.VolumeBase.ParseFilename(n) - where p != null && p.FileType == RemoteVolumeType.Files - orderby p.Time descending - select p).ToArray(); + var parsedlist = (await backendManager.ListAsync(m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + .Select(n => Volumes.VolumeBase.ParseFilename(n)) + .Where(p => p != null && p.FileType == RemoteVolumeType.Files) + .OrderByDescending(p => p.Time) + .ToArray(); var numberedList = parsedlist.Zip(Enumerable.Range(0, parsedlist.Length), (a, b) => new Tuple(b, a.Time, a)).ToList(); if (numberedList.Count < 2) @@ -138,19 +138,19 @@ namespace Duplicati.Library.Main.Operation } }; - if (!m_result.TaskControl.ProgressRendevouz().Await()) + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) return; - using (var tmpfile = backendManager.GetAsync(baseFile.File.Name, null, baseFile.File.Size, cancellationToken).Await()) + using (var tmpfile = await backendManager.GetAsync(baseFile.File.Name, null, baseFile.File.Size, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) using (var rd = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(baseFile.File.Name), tmpfile, m_options)) foreach (var f in rd.Files) if (FilterExpression.Matches(filter, f.Path)) storageKeeper.AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), false); - if (!m_result.TaskControl.ProgressRendevouz().Await()) + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) return; - using (var tmpfile = backendManager.GetAsync(compareFile.File.Name, null, compareFile.File.Size, cancellationToken).Await()) + using (var tmpfile = await backendManager.GetAsync(compareFile.File.Name, null, compareFile.File.Size, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) using (var rd = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(compareFile.File.Name), tmpfile, m_options)) foreach (var f in rd.Files) if (FilterExpression.Matches(filter, f.Path)) diff --git a/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs b/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs index 27aa80653..a9f32c9b2 100644 --- a/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs @@ -24,6 +24,7 @@ using System.Collections.Generic; using Duplicati.Library.Main.Database; using Duplicati.Library.Utility; using System.Threading; +using System.Threading.Tasks; namespace Duplicati.Library.Main.Operation { @@ -38,7 +39,7 @@ namespace Duplicati.Library.Main.Operation m_result = result; } - public void Run(IBackendManager backendManager, IEnumerable filterstrings, Library.Utility.IFilter compositefilter) + public async Task RunAsync(IBackendManager backendManager, IEnumerable filterstrings, IFilter compositefilter) { var cancellationToken = CancellationToken.None; using (var tmpdb = new TempFile()) @@ -50,7 +51,7 @@ namespace Duplicati.Library.Main.Operation try { - var filteredList = ListFilesHandler.ParseAndFilterFilesets(backendManager.ListAsync(cancellationToken).Await(), m_options); + var filteredList = ListFilesHandler.ParseAndFilterFilesets(await backendManager.ListAsync(cancellationToken).ConfigureAwait(false), m_options); if (filteredList.Count == 0) throw new Exception("No filesets found on remote target"); @@ -59,14 +60,14 @@ namespace Duplicati.Library.Main.Operation foreach (var fileversion in filteredList) try { - if (!m_result.TaskControl.ProgressRendevouz().Await()) + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) return; var file = fileversion.Value.File; var entry = db.GetRemoteVolume(file.Name); var files = new List(); - using (var tmpfile = backendManager.GetAsync(file.Name, entry.Hash, entry.Size < 0 ? file.Size : entry.Size, cancellationToken).Await()) + using (var tmpfile = await backendManager.GetAsync(file.Name, entry.Hash, entry.Size < 0 ? file.Size : entry.Size, cancellationToken).ConfigureAwait(false)) using (var tmp = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(file.Name), tmpfile, m_options)) foreach (var cf in tmp.ControlFiles) if (Library.Utility.FilterExpression.Matches(filter, cf.Key)) @@ -88,7 +89,7 @@ namespace Duplicati.Library.Main.Operation } finally { - backendManager.WaitForEmptyAsync(db, null, cancellationToken).Await(); + await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false); } } } diff --git a/Duplicati/Library/Main/Operation/ListFilesHandler.cs b/Duplicati/Library/Main/Operation/ListFilesHandler.cs index 263009748..cb59afec8 100644 --- a/Duplicati/Library/Main/Operation/ListFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFilesHandler.cs @@ -45,7 +45,7 @@ namespace Duplicati.Library.Main.Operation m_result = result; } - public async Task Run(IBackendManager backendManager, IEnumerable filterstrings, IFilter compositefilter) + public async Task RunAsync(IBackendManager backendManager, IEnumerable filterstrings, IFilter compositefilter) { var cancellationToken = m_result.TaskControl.ProgressToken; var parsedfilter = new FilterExpression(filterstrings); diff --git a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs index f4487c6d8..e2c281990 100644 --- a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs @@ -1,28 +1,30 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; using Duplicati.Library.Interface; +using Duplicati.Library.Main.Database; namespace Duplicati.Library.Main.Operation { @@ -41,7 +43,7 @@ namespace Duplicati.Library.Main.Operation m_result = result; } - public void Run(IBackendManager backendManager, Library.Utility.IFilter filter) + public async Task RunAsync(IBackendManager backendManager, Library.Utility.IFilter filter) { if (!System.IO.File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); @@ -49,15 +51,13 @@ namespace Duplicati.Library.Main.Operation if (filter != null && !filter.Empty) throw new UserInformationException("Filters are not supported for this operation", "FiltersNotAllowedOnPurgeBrokenFiles"); - List missing = null; - - using (var db = new Database.LocalListBrokenFilesDatabase(m_options.Dbpath)) + using (var db = new LocalListBrokenFilesDatabase(m_options.Dbpath)) using (var tr = db.BeginTransaction()) { if (db.PartiallyRecreated) throw new UserInformationException("The command does not work on partially recreated databases", "CannotPurgeOnPartialDatabase"); - var sets = ListBrokenFilesHandler.GetBrokenFilesetsFromRemote(backendManager, m_result, db, tr, m_options, out missing); + (var sets, var missing) = await ListBrokenFilesHandler.GetBrokenFilesetsFromRemote(backendManager, m_result, db, tr, m_options).ConfigureAwait(false); if (sets == null) return; @@ -105,35 +105,21 @@ namespace Duplicati.Library.Main.Operation m_result.DeleteResults = new DeleteResults(m_result); using (var rmdb = new Database.LocalDeleteDatabase(db)) + using (var deltr = new ReusableTransaction(rmdb)) { - var deltr = rmdb.BeginTransaction(); - try + var opts = new Options(new Dictionary(m_options.RawOptions)); + opts.RawOptions["version"] = string.Join(",", fully_emptied.Select(x => x.Version.ToString())); + opts.RawOptions.Remove("time"); + opts.RawOptions["no-auto-compact"] = "true"; + + await new DeleteHandler(opts, (DeleteResults)m_result.DeleteResults) + .DoRunAsync(rmdb, deltr, true, false, backendManager).ConfigureAwait(false); + + if (!m_options.Dryrun) { - var opts = new Options(new Dictionary(m_options.RawOptions)); - opts.RawOptions["version"] = string.Join(",", fully_emptied.Select(x => x.Version.ToString())); - opts.RawOptions.Remove("time"); - opts.RawOptions["no-auto-compact"] = "true"; - - new DeleteHandler(opts, (DeleteResults)m_result.DeleteResults) - .DoRun(rmdb, ref deltr, true, false, backendManager); - - if (!m_options.Dryrun) - { - using (new Logging.Timer(LOGTAG, "CommitDelete", "CommitDelete")) - deltr.Commit(); - - rmdb.WriteResults(); - } - else - deltr.Rollback(); + deltr.Commit("CommitDelete", restart: false); + rmdb.WriteResults(); } - finally - { - if (deltr != null) - try { deltr.Rollback(); } - catch { } - } - } pgoffset += (pgspan * fully_emptied.Length); @@ -161,12 +147,12 @@ namespace Duplicati.Library.Main.Operation opts.RawOptions.Remove("time"); opts.RawOptions["no-auto-compact"] = "true"; - new PurgeFilesHandler(opts, (PurgeFilesResults)m_result.PurgeResults).Run(backendManager, pgdb, pgoffset, pgspan, (cmd, filesetid, tablename) => + await new PurgeFilesHandler(opts, (PurgeFilesResults)m_result.PurgeResults).RunAsync(backendManager, pgdb, pgoffset, pgspan, (cmd, filesetid, tablename) => { if (filesetid != bs.FilesetID) throw new Exception(string.Format("Unexpected filesetid: {0}, expected {1}", filesetid, bs.FilesetID)); db.InsertBrokenFileIDsIntoTable(filesetid, tablename, "FileID", cmd.Transaction); - }); + }).ConfigureAwait(false); } pgoffset += pgspan; diff --git a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs index 718793687..51ae1296c 100644 --- a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs @@ -22,6 +22,7 @@ using System; using System.Linq; using System.Threading; +using System.Threading.Tasks; using Duplicati.Library.Interface; using Duplicati.Library.Utility; @@ -42,7 +43,7 @@ namespace Duplicati.Library.Main.Operation m_result = result; } - public void Run(IBackendManager backendManager, Library.Utility.IFilter filter) + public async Task RunAsync(IBackendManager backendManager, IFilter filter) { if (filter == null || filter.Empty) throw new UserInformationException("Cannot purge with an empty filter, as that would cause all files to be removed.\nTo remove an entire backup set, use the \"delete\" command.", "EmptyFilterPurgeNotAllowed"); @@ -51,17 +52,14 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); using (var db = new Database.LocalPurgeDatabase(m_options.Dbpath)) - DoRun(backendManager, db, filter, null, 0, 1); + await DoRunAsync(backendManager, db, filter, null, 0, 1).ConfigureAwait(false); } - public void Run(IBackendManager backendManager, Database.LocalPurgeDatabase db, float pgoffset, float pgspan, Action filtercommand) - { - DoRun(backendManager, db, null, filtercommand, pgoffset, pgspan); - } + public Task RunAsync(IBackendManager backendManager, Database.LocalPurgeDatabase db, float pgoffset, float pgspan, Action filtercommand) + => DoRunAsync(backendManager, db, null, filtercommand, pgoffset, pgspan); - private void DoRun(IBackendManager backendManager, Database.LocalPurgeDatabase db, IFilter filter, Action filtercommand, float pgoffset, float pgspan) + private async Task DoRunAsync(IBackendManager backendManager, Database.LocalPurgeDatabase db, IFilter filter, Action filtercommand, float pgoffset, float pgspan) { - var cancellationToken = CancellationToken.None; m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Begin); Logging.Log.WriteInformationMessage(LOGTAG, "StartingPurge", "Starting purge operation"); @@ -89,9 +87,9 @@ namespace Duplicati.Library.Main.Operation db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, false, null); if (m_options.NoBackendverification) - FilelistProcessor.VerifyLocalList(backendManager, db, cancellationToken).Await(); + await FilelistProcessor.VerifyLocalList(backendManager, db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); else - FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, null, null, logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).Await(); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, null, null, logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); } var filesets = db.FilesetTimes.OrderByDescending(x => x.Value).ToArray(); @@ -212,9 +210,9 @@ namespace Duplicati.Library.Main.Operation db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, tr); tr.Commit(); - backendManager.PutAsync(vol, null, null, true, cancellationToken).Await(); - backendManager.DeleteAsync(prevfilename, -1, true, cancellationToken).Await(); - backendManager.WaitForEmptyAsync(db, null, cancellationToken).Await(); + await backendManager.PutAsync(vol, null, null, true, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await backendManager.DeleteAsync(prevfilename, -1, true, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); } } } @@ -240,30 +238,21 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Compact); m_result.CompactResults = new CompactResults(m_result); using (var cdb = new Database.LocalDeleteDatabase(db)) + using (var ctr = new Database.ReusableTransaction(cdb)) { - var tr = cdb.BeginTransaction(); - try - { - var (_, tr2) = new CompactHandler(m_options, (CompactResults)m_result.CompactResults).DoCompact(cdb, true, tr, backendManager).Await(); - tr2 = tr; - } - catch - { - try { tr.Rollback(); } - catch { } - } - finally - { - try { tr.Commit(); } - catch { } - } + await new CompactHandler(m_options, (CompactResults)m_result.CompactResults) + .DoCompactAsync(cdb, true, ctr, backendManager) + .ConfigureAwait(false); + + ctr.Commit(restart: false); + cdb.WriteResults(); } } m_result.OperationProgressUpdater.UpdateProgress(pgoffset + pgspan); m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Complete); - backendManager.WaitForEmptyAsync(db, null, cancellationToken).Await(); + await backendManager.WaitForEmptyAsync(db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); } } } diff --git a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs index 7849e343d..f449adfe1 100644 --- a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs +++ b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs @@ -59,15 +59,15 @@ namespace Duplicati.Library.Main.Operation /// A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist /// Filters the files in a filelist to prevent downloading unwanted data /// A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists - public void Run(string path, IBackendManager backendManager, IFilter filter, NumberedFilterFilelistDelegate filelistfilter, BlockVolumePostProcessor blockprocessor) + public async Task RunAsync(string path, IBackendManager backendManager, IFilter filter, NumberedFilterFilelistDelegate filelistfilter, BlockVolumePostProcessor blockprocessor) { - if (System.IO.File.Exists(path)) + if (File.Exists(path)) throw new UserInformationException(string.Format("Cannot recreate database because file already exists: {0}", path), "RecreateTargetDatabaseExists"); using (var db = new LocalDatabase(path, "Recreate", true)) { m_result.SetDatabase(db); - DoRun(backendManager, db, false, filter, filelistfilter, blockprocessor).Await(); + await DoRunAsync(backendManager, db, false, filter, filelistfilter, blockprocessor).ConfigureAwait(false); db.WriteResults(); } } @@ -78,7 +78,7 @@ namespace Duplicati.Library.Main.Operation /// A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist /// Filters the files in a filelist to prevent downloading unwanted data /// A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists - public void RunUpdate(IBackendManager backendManager, Library.Utility.IFilter filter, NumberedFilterFilelistDelegate filelistfilter, BlockVolumePostProcessor blockprocessor) + public async Task RunUpdateAsync(IBackendManager backendManager, Library.Utility.IFilter filter, NumberedFilterFilelistDelegate filelistfilter, BlockVolumePostProcessor blockprocessor) { if (!m_options.RepairOnlyPaths) throw new UserInformationException(string.Format("Can only update with paths, try setting --{0}", "repair-only-paths"), "RepairUpdateRequiresPathsOnly"); @@ -100,7 +100,7 @@ namespace Duplicati.Library.Main.Operation if (preexistingOptionsInDatabase) Utility.VerifyOptionsAndUpdateDatabase(db, m_options, null); - DoRun(backendManager, db, true, filter, filelistfilter, blockprocessor).Await(); + await DoRunAsync(backendManager, db, true, filter, filelistfilter, blockprocessor).ConfigureAwait(false); db.WriteResults(); } } @@ -113,9 +113,8 @@ namespace Duplicati.Library.Main.Operation /// A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist /// Filters the files in a filelist to prevent downloading unwanted data /// A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists - internal async Task DoRun(IBackendManager backendManager, LocalDatabase dbparent, bool updating, IFilter filter = null, NumberedFilterFilelistDelegate filelistfilter = null, BlockVolumePostProcessor blockprocessor = null) + internal async Task DoRunAsync(IBackendManager backendManager, LocalDatabase dbparent, bool updating, IFilter filter = null, NumberedFilterFilelistDelegate filelistfilter = null, BlockVolumePostProcessor blockprocessor = null) { - var cancellationToken = CancellationToken.None; m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Recreate_Running); //We build a local database in steps. @@ -130,7 +129,7 @@ namespace Duplicati.Library.Main.Operation expRecreateDb = true; } - var rawlist = await backendManager.ListAsync(cancellationToken); + var rawlist = await backendManager.ListAsync(m_result.TaskControl.ProgressToken).ConfigureAwait(false); //First step is to examine the remote storage to see what // kind of data we can find @@ -213,7 +212,7 @@ namespace Duplicati.Library.Main.Operation { if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(restoredb, tr, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(restoredb, tr, m_result.TaskControl.ProgressToken).ConfigureAwait(false); m_result.EndTime = DateTime.UtcNow; return; } @@ -311,7 +310,7 @@ namespace Duplicati.Library.Main.Operation { if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(restoredb, tr, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(restoredb, tr, m_result.TaskControl.ProgressToken).ConfigureAwait(false); m_result.EndTime = DateTime.UtcNow; return; } @@ -451,9 +450,9 @@ namespace Duplicati.Library.Main.Operation using (var rd = new BlockVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options)) using (var tr = restoredb.BeginTransaction()) { - if (!m_result.TaskControl.ProgressRendevouz().Await()) + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - backendManager.WaitForEmptyAsync(restoredb, tr, cancellationToken).Await(); + await backendManager.WaitForEmptyAsync(restoredb, tr, m_result.TaskControl.ProgressToken).ConfigureAwait(false); m_result.EndTime = DateTime.UtcNow; return; } @@ -517,7 +516,7 @@ namespace Duplicati.Library.Main.Operation } } - backendManager.WaitForEmptyAsync(restoredb, null, cancellationToken).Await(); + await backendManager.WaitForEmptyAsync(restoredb, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); if (!m_options.RepairOnlyPaths) { diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 929b318fa..0b72185c8 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -49,11 +49,11 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException(Strings.Common.PassphraseChangeUnsupported, "PassphraseChangeUnsupported"); } - public void Run(IBackendManager backendManager, IFilter filter) + public async Task RunAsync(IBackendManager backendManager, IFilter filter) { - if (!System.IO.File.Exists(m_options.Dbpath)) + if (!File.Exists(m_options.Dbpath)) { - RunRepairLocal(backendManager, filter); + await RunRepairLocalAsync(backendManager, filter).ConfigureAwait(false); RunRepairCommon(); m_result.EndTime = DateTime.UtcNow; return; @@ -78,47 +78,48 @@ namespace Duplicati.Library.Main.Operation } else { - var baseName = System.IO.Path.ChangeExtension(m_options.Dbpath, "backup"); + var baseName = Path.ChangeExtension(m_options.Dbpath, "backup"); var i = 0; - while (System.IO.File.Exists(baseName) && i++ < 1000) - baseName = System.IO.Path.ChangeExtension(m_options.Dbpath, "backup-" + i.ToString()); + while (File.Exists(baseName) && i++ < 1000) + baseName = Path.ChangeExtension(m_options.Dbpath, "backup-" + i.ToString()); Logging.Log.WriteInformationMessage(LOGTAG, "RenamingDatabase", "Renaming existing db from {0} to {1}", m_options.Dbpath, baseName); - System.IO.File.Move(m_options.Dbpath, baseName); + File.Move(m_options.Dbpath, baseName); } - RunRepairLocal(backendManager, filter); + await RunRepairLocalAsync(backendManager, filter).ConfigureAwait(false); RunRepairCommon(); } else { RunRepairCommon(); - RunRepairRemote(backendManager, m_result.TaskControl.ProgressToken).Await(); + await RunRepairRemoteAsync(backendManager, m_result.TaskControl.ProgressToken).ConfigureAwait(false); } m_result.EndTime = DateTime.UtcNow; } - public void RunRepairLocal(IBackendManager backendManager, Library.Utility.IFilter filter) + public async Task RunRepairLocalAsync(IBackendManager backendManager, IFilter filter) { m_result.RecreateDatabaseResults = new RecreateDatabaseResults(m_result); using (new Logging.Timer(LOGTAG, "RecreateDbForRepair", "Recreate database for repair")) - using (var f = m_options.Dryrun ? new Library.Utility.TempFile() : null) + using (var f = m_options.Dryrun ? new TempFile() : null) { - if (f != null && System.IO.File.Exists(f)) - System.IO.File.Delete(f); + if (f != null && File.Exists(f)) + File.Delete(f); var filelistfilter = RestoreHandler.FilterNumberedFilelist(m_options.Time, m_options.Version); - new RecreateDatabaseHandler(m_options, (RecreateDatabaseResults)m_result.RecreateDatabaseResults) - .Run(m_options.Dryrun ? (string)f : m_options.Dbpath, backendManager, filter, filelistfilter, null); + await new RecreateDatabaseHandler(m_options, (RecreateDatabaseResults)m_result.RecreateDatabaseResults) + .RunAsync(m_options.Dryrun ? (string)f : m_options.Dbpath, backendManager, filter, filelistfilter, null) + .ConfigureAwait(false); } } - public async Task RunRepairRemote(IBackendManager backendManager, CancellationToken cancellationToken) + public async Task RunRepairRemoteAsync(IBackendManager backendManager, CancellationToken cancellationToken) { - if (!System.IO.File.Exists(m_options.Dbpath)) + if (!File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "RepairDatabaseFileDoesNotExist"); m_result.OperationProgressUpdater.UpdateProgress(0); @@ -579,7 +580,7 @@ namespace Duplicati.Library.Main.Operation public void RunRepairCommon() { - if (!System.IO.File.Exists(m_options.Dbpath)) + if (!File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); m_result.OperationProgressUpdater.UpdateProgress(0); diff --git a/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs b/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs index e3739f4dc..8bff6e456 100644 --- a/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs @@ -20,7 +20,9 @@ // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; +using System.IO; using System.Threading; +using System.Threading.Tasks; using Duplicati.Library.Utility; namespace Duplicati.Library.Main.Operation @@ -36,35 +38,34 @@ namespace Duplicati.Library.Main.Operation m_result = result; } - public void Run(IEnumerable filterstrings, IBackendManager backendManager, Library.Utility.IFilter compositefilter) + public async Task RunAsync(IEnumerable filterstrings, IBackendManager backendManager, Library.Utility.IFilter compositefilter) { - var cancellationToken = CancellationToken.None; if (string.IsNullOrEmpty(m_options.Restorepath)) throw new Exception("Cannot restore control files without --restore-path"); - if (!System.IO.Directory.Exists(m_options.Restorepath)) - System.IO.Directory.CreateDirectory(m_options.Restorepath); + if (!Directory.Exists(m_options.Restorepath)) + Directory.CreateDirectory(m_options.Restorepath); - using (var tmpdb = new Library.Utility.TempFile()) - using (var db = new Database.LocalDatabase(System.IO.File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "RestoreControlFiles", true)) + using (var tmpdb = new TempFile()) + using (var db = new Database.LocalDatabase(File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "RestoreControlFiles", true)) { m_result.SetDatabase(db); - var filter = Library.Utility.JoinedFilterExpression.Join(new Library.Utility.FilterExpression(filterstrings), compositefilter); + var filter = JoinedFilterExpression.Join(new FilterExpression(filterstrings), compositefilter); try { - var filteredList = ListFilesHandler.ParseAndFilterFilesets(backendManager.ListAsync(cancellationToken).Await(), m_options); + var filteredList = ListFilesHandler.ParseAndFilterFilesets(await backendManager.ListAsync(m_result.TaskControl.ProgressToken).ConfigureAwait(false), m_options); if (filteredList.Count == 0) throw new Exception("No filesets found on remote target"); - Exception lastEx = new Exception("No suitable files found on remote target"); + var lastEx = new Exception("No suitable files found on remote target"); foreach (var fileversion in filteredList) try { - if (!m_result.TaskControl.ProgressRendevouz().Await()) + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - backendManager.WaitForEmptyAsync(db, null, cancellationToken).Await(); + await backendManager.WaitForEmptyAsync(db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); return; } @@ -72,14 +73,14 @@ namespace Duplicati.Library.Main.Operation var entry = db.GetRemoteVolume(file.Name); var res = new List(); - using (var tmpfile = backendManager.GetAsync(file.Name, entry.Hash, entry.Size < 0 ? file.Size : entry.Size, cancellationToken).Await()) + using (var tmpfile = await backendManager.GetAsync(file.Name, entry.Hash, entry.Size < 0 ? file.Size : entry.Size, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) using (var tmp = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(file.Name), tmpfile, m_options)) foreach (var cf in tmp.ControlFiles) - if (Library.Utility.FilterExpression.Matches(filter, cf.Key)) + if (FilterExpression.Matches(filter, cf.Key)) { - var targetpath = System.IO.Path.Combine(m_options.Restorepath, cf.Key); - using (var ts = System.IO.File.Create(targetpath)) - Library.Utility.Utility.CopyStream(cf.Value, ts); + var targetpath = Path.Combine(m_options.Restorepath, cf.Key); + using (var ts = File.Create(targetpath)) + await Library.Utility.Utility.CopyStreamAsync(cf.Value, ts, m_result.TaskControl.ProgressToken).ConfigureAwait(false); res.Add(targetpath); } @@ -100,7 +101,7 @@ namespace Duplicati.Library.Main.Operation } finally { - backendManager.WaitForEmptyAsync(db, null, cancellationToken).Await(); + await backendManager.WaitForEmptyAsync(db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); } db.WriteResults(); diff --git a/Duplicati/Library/Main/Operation/RestoreHandler.cs b/Duplicati/Library/Main/Operation/RestoreHandler.cs index ef69763b0..f6cc80f48 100644 --- a/Duplicati/Library/Main/Operation/RestoreHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreHandler.cs @@ -102,7 +102,7 @@ namespace Duplicati.Library.Main.Operation }; } - public void Run(string[] paths, IBackendManager backendManager, Library.Utility.IFilter filter) + public async Task RunAsync(string[] paths, IBackendManager backendManager, Library.Utility.IFilter filter) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_Begin); @@ -126,8 +126,9 @@ namespace Duplicati.Library.Main.Operation db = new LocalRestoreDatabase(tmpdb); m_result.RecreateDatabaseResults = new RecreateDatabaseResults(m_result); using (new Logging.Timer(LOGTAG, "RecreateTempDbForRestore", "Recreate temporary database for restore")) - new RecreateDatabaseHandler(m_options, (RecreateDatabaseResults)m_result.RecreateDatabaseResults) - .DoRun(backendManager, db, false, filter, filelistfilter, null).Await(); + await new RecreateDatabaseHandler(m_options, (RecreateDatabaseResults)m_result.RecreateDatabaseResults) + .DoRunAsync(backendManager, db, false, filter, filelistfilter, null) + .ConfigureAwait(false); if (!m_options.SkipMetadata) ApplyStoredMetadata(m_options, new RestoreHandlerMetadataStorage()); @@ -139,9 +140,9 @@ namespace Duplicati.Library.Main.Operation } if (m_options.RestoreLegacy) - DoRun(backendManager, db, filter, m_result.TaskControl.ProgressToken).Await(); + await DoRunAsync(backendManager, db, filter, m_result.TaskControl.ProgressToken).ConfigureAwait(false); else - DoRunNew(backendManager, db, filter, m_result.TaskControl.ProgressToken).Await(); + await DoRunNewAsync(backendManager, db, filter, m_result.TaskControl.ProgressToken).ConfigureAwait(false); db.WriteResults(); } @@ -307,7 +308,7 @@ namespace Duplicati.Library.Main.Operation /// /// The database containing information about the restore. /// The filter of which files to restore. - private async Task DoRunNew(IBackendManager backendManager, LocalRestoreDatabase database, Library.Utility.IFilter filter, CancellationToken cancellationToken) + private async Task DoRunNewAsync(IBackendManager backendManager, LocalRestoreDatabase database, Library.Utility.IFilter filter, CancellationToken cancellationToken) { // Perform initial setup Utility.UpdateOptionsFromDb(database, m_options); @@ -403,7 +404,7 @@ namespace Duplicati.Library.Main.Operation // Drop the temp tables database.DropRestoreTable(); - await backendManager.WaitForEmptyAsync(database, null, CancellationToken.None).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(database, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); // Report that the restore is complete m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_Complete); @@ -416,7 +417,7 @@ namespace Duplicati.Library.Main.Operation /// /// The database containing information about the restore. /// The filter of which files to restore. - private async Task DoRun(IBackendManager backendManager, LocalRestoreDatabase database, Library.Utility.IFilter filter, CancellationToken cancellationToken) + private async Task DoRunAsync(IBackendManager backendManager, LocalRestoreDatabase database, Library.Utility.IFilter filter, CancellationToken cancellationToken) { //In this case, we check that the remote storage fits with the database. //We can then query the database and find the blocks that we need to do the restore diff --git a/Duplicati/Library/Main/Operation/SystemInfoHandler.cs b/Duplicati/Library/Main/Operation/SystemInfoHandler.cs index 04d9f0d94..07071de20 100644 --- a/Duplicati/Library/Main/Operation/SystemInfoHandler.cs +++ b/Duplicati/Library/Main/Operation/SystemInfoHandler.cs @@ -23,15 +23,14 @@ using System; using System.Collections.Generic; using System.Linq; using System.Globalization; +using System.Threading.Tasks; namespace Duplicati.Library.Main.Operation { internal static class SystemInfoHandler { - public static void Run(SystemInfoResults results) - { - results.Lines = GetSystemInfo().ToArray(); - } + public static Task RunAsync(SystemInfoResults results) + => Task.Run(() => results.Lines = GetSystemInfo().ToArray()); public static IEnumerable GetSystemInfo() { diff --git a/Duplicati/Library/Main/Operation/TestHandler.cs b/Duplicati/Library/Main/Operation/TestHandler.cs index 5c2608d69..747e1c8ef 100644 --- a/Duplicati/Library/Main/Operation/TestHandler.cs +++ b/Duplicati/Library/Main/Operation/TestHandler.cs @@ -46,7 +46,7 @@ namespace Duplicati.Library.Main.Operation m_results = results; } - public void Run(long samples, IBackendManager backendManager) + public async Task RunAsync(long samples, IBackendManager backendManager) { if (!System.IO.File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); @@ -57,14 +57,14 @@ namespace Duplicati.Library.Main.Operation Utility.UpdateOptionsFromDb(db, m_options); Utility.VerifyOptionsAndUpdateDatabase(db, m_options); db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, !m_options.DisableFilelistConsistencyChecks, null); - FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_results.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly, null).Await(); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_results.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly, null).ConfigureAwait(false); - DoRun(samples, db, backendManager).Await(); + await DoRunAsync(samples, db, backendManager).ConfigureAwait(false); db.WriteResults(); } } - public async Task DoRun(long samples, LocalTestDatabase db, IBackendManager backend) + public async Task DoRunAsync(long samples, LocalTestDatabase db, IBackendManager backend) { var files = db.SelectTestTargets(samples, m_options).ToList(); @@ -74,14 +74,14 @@ namespace Duplicati.Library.Main.Operation if (m_options.FullRemoteVerification != Options.RemoteTestStrategy.False) { - await foreach (var (tf, hash, size, name) in backend.GetFilesOverlappedAsync(files, CancellationToken.None).ConfigureAwait(false)) + await foreach (var (tf, hash, size, name) in backend.GetFilesOverlappedAsync(files, m_results.TaskControl.ProgressToken).ConfigureAwait(false)) { var vol = new RemoteVolume(name, hash, size); try { - if (!m_results.TaskControl.ProgressRendevouz().Await()) + if (!await m_results.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - backend.WaitForEmptyAsync(db, null, CancellationToken.None).Await(); + await backend.WaitForEmptyAsync(db, null, m_results.TaskControl.ProgressToken).ConfigureAwait(false); m_results.EndTime = DateTime.UtcNow; return; } @@ -138,7 +138,7 @@ namespace Duplicati.Library.Main.Operation { try { - if (!m_results.TaskControl.ProgressRendevouz().Await()) + if (!await m_results.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { m_results.EndTime = DateTime.UtcNow; return; @@ -152,7 +152,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteInformationMessage(LOGTAG, "MissingRemoteHash", "No hash or size recorded for {0}, performing full verification", f.Name); KeyValuePair>> res; - (var tf, var hash, var size) = backend.GetWithInfoAsync(f.Name, f.Hash, f.Size, CancellationToken.None).Await(); + (var tf, var hash, var size) = await backend.GetWithInfoAsync(f.Name, f.Hash, f.Size, m_results.TaskControl.ProgressToken).ConfigureAwait(false); using (tf) res = TestVolumeInternals(db, f, tf, m_options, 1); @@ -176,7 +176,7 @@ namespace Duplicati.Library.Main.Operation } else { - using (var tf = backend.GetAsync(f.Name, f.Hash, f.Size, CancellationToken.None).Await()) + using (var tf = await backend.GetAsync(f.Name, f.Hash, f.Size, m_results.TaskControl.ProgressToken).ConfigureAwait(false)) { } } diff --git a/Duplicati/Library/Main/Operation/VacuumHandler.cs b/Duplicati/Library/Main/Operation/VacuumHandler.cs index c2e873d47..cb324129a 100644 --- a/Duplicati/Library/Main/Operation/VacuumHandler.cs +++ b/Duplicati/Library/Main/Operation/VacuumHandler.cs @@ -1,23 +1,23 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; using System.Linq; @@ -37,15 +37,16 @@ namespace Duplicati.Library.Main.Operation m_result = result; } - public virtual void Run() - { - using (var db = new Database.LocalDatabase(m_options.Dbpath, "Vacuum", false)) + public virtual Task RunAsync() + => Task.Run(() => { - m_result.SetDatabase(db); - m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Vacuum_Running); - db.Vacuum(); - m_result.EndTime = DateTime.UtcNow; - } - } + using (var db = new Database.LocalDatabase(m_options.Dbpath, "Vacuum", false)) + { + m_result.SetDatabase(db); + m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Vacuum_Running); + db.Vacuum(); + m_result.EndTime = DateTime.UtcNow; + } + }); } } From 7234d4396144f83a2aad03bee5de149a4db2da8e Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 14 Mar 2025 12:13:17 +0100 Subject: [PATCH 353/362] Fixed all SQL strings to use interpolation and forced them into the FormattableString handler using the invariant formatting. --- .../Main/Database/LocalBackupDatabase.cs | 30 +- .../Main/Database/LocalBugReportDatabase.cs | 24 +- .../Library/Main/Database/LocalDatabase.cs | 92 ++--- .../Main/Database/LocalDeleteDatabase.cs | 18 +- .../Database/LocalListAffectedDatabase.cs | 89 ++--- .../Database/LocalListBrokenFilesDatabase.cs | 48 ++- .../Main/Database/LocalListChangesDatabase.cs | 188 ++++----- .../Main/Database/LocalListDatabase.cs | 170 ++++----- .../Main/Database/LocalPurgeDatabase.cs | 30 +- .../Main/Database/LocalRecreateDatabase.cs | 207 ++++------ .../Main/Database/LocalRepairDatabase.cs | 68 ++-- .../Main/Database/LocalRestoreDatabase.cs | 359 ++++++++---------- .../Main/Database/LocalTestDatabase.cs | 54 +-- 13 files changed, 636 insertions(+), 741 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index c5db46e6e..6d121a8ff 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -185,14 +185,10 @@ namespace Duplicati.Library.Main.Database @" WHERE ""A"".""ID"" = ""B"".""FileID"" " + @" AND ""B"".""FilesetID"" = ? "; - m_findfileCommand.CommandText = FormatInvariant( - @"SELECT ""C"".""ID"" AS ""FileID"", ""C"".""LastModified"", ""D"".""Length"", ""E"".""FullHash"" as ""Metahash"", ""E"".""Length"" AS ""Metasize"" " + - @" FROM " + - @" ({0}) AS ""C"", ""Blockset"" AS ""D"", ""Blockset"" AS ""E"", ""Metadataset"" ""F"" " + - @" WHERE ""C"".""BlocksetID"" == ""D"".""ID"" AND ""C"".""MetadataID"" == ""F"".""ID"" AND ""F"".""BlocksetID"" = ""E"".""ID"" " + - @" LIMIT 1", - getLastFileEntryForPath - ); + m_findfileCommand.CommandText = FormatInvariant($@"SELECT ""C"".""ID"" AS ""FileID"", ""C"".""LastModified"", ""D"".""Length"", ""E"".""FullHash"" as ""Metahash"", ""E"".""Length"" AS ""Metasize"" +FROM ({getLastFileEntryForPath}) AS ""C"", ""Blockset"" AS ""D"", ""Blockset"" AS ""E"", ""Metadataset"" ""F"" +WHERE ""C"".""BlocksetID"" == ""D"".""ID"" AND ""C"".""MetadataID"" == ""F"".""ID"" AND ""F"".""BlocksetID"" = ""E"".""ID"" +LIMIT 1"); break; // Potentially faster query: https://forum.duplicati.com/t/release-2-0-3-10-canary-2018-08-30/4497/25 @@ -616,22 +612,22 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist { var subqueryFiles = @"SELECT ""File"".""Path"" AS ""Path"", ""A"".""Fullhash"" AS ""Filehash"", ""B"".""Fullhash"" AS ""Metahash"" FROM ""File"", ""FilesetEntry"", ""Blockset"" A, ""Blockset"" B, ""Metadataset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""A"".""ID"" = ""File"".""BlocksetID"" AND ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""MetadataID"" = ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""B"".""ID"" "; - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" AS " + subqueryFiles, tmpName1), lastFilesetId); - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" AS " + subqueryFiles, tmpName2), m_filesetId); - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE INDEX ""nn_tmpName1"" ON ""{0}"" (""Path"")", tmpName1)); - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE INDEX ""nn_tmpName2"" ON ""{0}"" (""Path"")", tmpName2)); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tmpName1}"" AS {subqueryFiles}"), lastFilesetId); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tmpName2}"" AS {subqueryFiles}"), m_filesetId); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""nn_tmpName1"" ON ""{tmpName1}"" (""Path"")")); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""nn_tmpName2"" ON ""{tmpName2}"" (""Path"")")); - results.AddedFiles = cmd.ExecuteScalarInt64(FormatInvariant(@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""BlocksetID"" != ? AND ""File"".""BlocksetID"" != ? AND NOT ""File"".""Path"" IN (SELECT ""Path"" FROM ""{0}"")", tmpName1), 0, m_filesetId, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID); - results.DeletedFiles = cmd.ExecuteScalarInt64(FormatInvariant(@"SELECT COUNT(*) FROM ""{0}"" WHERE ""{0}"".""Path"" NOT IN (SELECT ""Path"" FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ?)", tmpName1), 0, m_filesetId); - results.ModifiedFiles = cmd.ExecuteScalarInt64(FormatInvariant(@"SELECT COUNT(*) FROM ""{0}"" A, ""{1}"" B WHERE ""A"".""Path"" = ""B"".""Path"" AND (""A"".""Filehash"" != ""B"".""Filehash"" OR ""A"".""Metahash"" != ""B"".""Metahash"")", tmpName1, tmpName2), 0); + results.AddedFiles = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""BlocksetID"" != ? AND ""File"".""BlocksetID"" != ? AND NOT ""File"".""Path"" IN (SELECT ""Path"" FROM ""{tmpName1}"")"), 0, m_filesetId, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID); + results.DeletedFiles = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM ""{tmpName1}"" WHERE ""{tmpName1}"".""Path"" NOT IN (SELECT ""Path"" FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ?)"), 0, m_filesetId); + results.ModifiedFiles = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM ""{tmpName1}"" A, ""{tmpName2}"" B WHERE ""A"".""Path"" = ""B"".""Path"" AND (""A"".""Filehash"" != ""B"".""Filehash"" OR ""A"".""Metahash"" != ""B"".""Metahash"")"), 0); } finally { - try { cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"";", tmpName1)); } + try { cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{tmpName1}"";")); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "DisposeError", ex, "Dispose temp table error"); } - try { cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"";", tmpName2)); } + try { cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{tmpName2}"";")); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "DisposeError", ex, "Dispose temp table error"); } } } diff --git a/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs b/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs index f155b0784..bcf56106b 100644 --- a/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs @@ -40,21 +40,21 @@ namespace Duplicati.Library.Main.Database public void Fix() { - using(var tr = m_connection.BeginTransaction()) - using(var cmd = m_connection.CreateCommand()) + using (var tr = m_connection.BeginTransaction()) + using (var cmd = m_connection.CreateCommand()) { cmd.Transaction = tr; var tablename = "PathMap-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); // TODO: Rewrite this to use PathPrefix // TODO: Needs to be much faster - using(var upcmd = m_connection.CreateCommand()) + using (var upcmd = m_connection.CreateCommand()) { - + upcmd.Transaction = tr; - upcmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (""ID"" INTEGER PRIMARY KEY, ""RealPath"" TEXT NOT NULL, ""Obfuscated"" TEXT NULL)", tablename)); - upcmd.ExecuteNonQuery(FormatInvariant(@"INSERT INTO ""{0}"" (""RealPath"") SELECT DISTINCT ""Path"" FROM ""File"" ORDER BY ""Path"" ", tablename)); - upcmd.ExecuteNonQuery(FormatInvariant(@"UPDATE ""{0}"" SET ""Obfuscated"" = ? || length(""RealPath"") || ? || ""ID"" || (CASE WHEN substr(""RealPath"", length(""RealPath"")) = ? THEN ? ELSE ? END) ", tablename), !OperatingSystem.IsWindows() ? "/" : "X:\\", Util.DirectorySeparatorString, Util.DirectorySeparatorString, Util.DirectorySeparatorString, ".bin"); + upcmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" (""ID"" INTEGER PRIMARY KEY, ""RealPath"" TEXT NOT NULL, ""Obfuscated"" TEXT NULL)")); + upcmd.ExecuteNonQuery(FormatInvariant($@"INSERT INTO ""{tablename}"" (""RealPath"") SELECT DISTINCT ""Path"" FROM ""File"" ORDER BY ""Path"" ")); + upcmd.ExecuteNonQuery(FormatInvariant($@"UPDATE ""{tablename}"" SET ""Obfuscated"" = ? || length(""RealPath"") || ? || ""ID"" || (CASE WHEN substr(""RealPath"", length(""RealPath"")) = ? THEN ? ELSE ? END) "), !OperatingSystem.IsWindows() ? "/" : "X:\\", Util.DirectorySeparatorString, Util.DirectorySeparatorString, Util.DirectorySeparatorString, ".bin"); /*long id = 1; using(var rd = cmd.ExecuteReader(string.Format(@"SELECT ""RealPath"", ""Obfuscated"" FROM ""{0}"" ", tablename))) @@ -71,16 +71,16 @@ namespace Duplicati.Library.Main.Database cmd.ExecuteNonQuery(@"UPDATE ""Configuration"" SET ""Value"" = 'ERASED!' WHERE ""Key"" = 'passphrase' "); - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TABLE ""FixedFile"" AS SELECT ""B"".""ID"" AS ""ID"", ""A"".""Obfuscated"" AS ""Path"", ""B"".""BlocksetID"" AS ""BlocksetID"", ""B"".""MetadataID"" AS ""MetadataID"" FROM ""{0}"" ""A"", ""File"" ""B"" WHERE ""A"".""RealPath"" = ""B"".""Path"" ", tablename)); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TABLE ""FixedFile"" AS SELECT ""B"".""ID"" AS ""ID"", ""A"".""Obfuscated"" AS ""Path"", ""B"".""BlocksetID"" AS ""BlocksetID"", ""B"".""MetadataID"" AS ""MetadataID"" FROM ""{tablename}"" ""A"", ""File"" ""B"" WHERE ""A"".""RealPath"" = ""B"".""Path"" ")); cmd.ExecuteNonQuery(@"DROP VIEW ""File"" "); cmd.ExecuteNonQuery(@"DROP TABLE ""FileLookup"" "); cmd.ExecuteNonQuery(@"DROP TABLE ""PathPrefix"" "); - cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", tablename)); - - using(new Logging.Timer(LOGTAG, "CommitUpdateBugReport", "CommitUpdateBugReport")) + cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{tablename}"" ")); + + using (new Logging.Timer(LOGTAG, "CommitUpdateBugReport", "CommitUpdateBugReport")) tr.Commit(); - + cmd.Transaction = null; cmd.ExecuteNonQuery("VACUUM"); diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 332eb2b44..15310f2fb 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -97,15 +97,6 @@ namespace Duplicati.Library.Main.Database return c; } - /// - /// Formats a string using the invariant culture - /// - /// The format string - /// The arguments to format - /// The formatted string - public static string FormatInvariant(string format, params object[] args) - => Library.Utility.Utility.FormatInvariant(format, args); - /// /// Formats the string using the invariant culture /// @@ -204,7 +195,7 @@ namespace Duplicati.Library.Main.Database m_selectremotevolumesCommand.CommandText = @"SELECT ""ID"", ""Name"", ""Type"", ""Size"", ""Hash"", ""State"", ""DeleteGraceTime"" FROM ""Remotevolume"""; m_selectremotevolumeCommand.CommandText = m_selectremotevolumesCommand.CommandText + @" WHERE ""Name"" = ?"; - m_selectduplicateRemoteVolumesCommand.CommandText = FormatInvariant(@"SELECT DISTINCT ""Name"", ""State"" FROM ""Remotevolume"" WHERE ""Name"" IN (SELECT ""Name"" FROM ""Remotevolume"" WHERE ""State"" IN ('{0}', '{1}')) AND NOT ""State"" IN ('{0}', '{1}')", RemoteVolumeState.Deleted.ToString(), RemoteVolumeState.Deleting.ToString()); + m_selectduplicateRemoteVolumesCommand.CommandText = FormatInvariant($@"SELECT DISTINCT ""Name"", ""State"" FROM ""Remotevolume"" WHERE ""Name"" IN (SELECT ""Name"" FROM ""Remotevolume"" WHERE ""State"" IN ('{RemoteVolumeState.Deleted.ToString()}', '{RemoteVolumeState.Deleting.ToString()}')) AND NOT ""State"" IN ('{RemoteVolumeState.Deleted.ToString()}', '{RemoteVolumeState.Deleting.ToString()}')"); m_selectremotevolumeCommand.AddParameter(); @@ -488,8 +479,8 @@ namespace Duplicati.Library.Main.Database var blocksetidstable = "DelBlockSetIds-" + temptransguid; // Create and fill a temp table with the volids to delete. We avoid using too many parameters that way. - deletecmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMP TABLE ""{0}"" (""ID"" INTEGER PRIMARY KEY)", volidstable)); - deletecmd.CommandText = FormatInvariant(@"INSERT OR IGNORE INTO ""{0}"" (""ID"") VALUES (?)", volidstable); + deletecmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMP TABLE ""{volidstable}"" (""ID"" INTEGER PRIMARY KEY)")); + deletecmd.CommandText = FormatInvariant($@"INSERT OR IGNORE INTO ""{volidstable}"" (""ID"") VALUES (?)"); deletecmd.Parameters.Clear(); deletecmd.AddParameters(1); foreach (var name in names) @@ -498,22 +489,21 @@ namespace Duplicati.Library.Main.Database deletecmd.SetParameterValue(0, volumeid); deletecmd.ExecuteNonQuery(); } - var volIdsSubQuery = FormatInvariant(@"SELECT ""ID"" FROM ""{0}"" ", volidstable); + var volIdsSubQuery = FormatInvariant($@"SELECT ""ID"" FROM ""{volidstable}"" "); deletecmd.Parameters.Clear(); - var bsIdsSubQuery = FormatInvariant( - @"SELECT DISTINCT ""BlocksetEntry"".""BlocksetID"" FROM ""BlocksetEntry"", ""Block""" - + @" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" IN ({0}) " - + @"UNION ALL " - + @"SELECT DISTINCT ""BlocksetID"" FROM ""BlocklistHash""" - + @" WHERE ""Hash"" IN (SELECT ""Hash"" FROM ""Block"" WHERE ""VolumeID"" IN ({0}))" - , volIdsSubQuery); + var bsIdsSubQuery = FormatInvariant(@$" +SELECT DISTINCT ""BlocksetEntry"".""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" +WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" IN ({volIdsSubQuery}) +UNION ALL +SELECT DISTINCT ""BlocksetID"" FROM ""BlocklistHash"" +WHERE ""Hash"" IN (SELECT ""Hash"" FROM ""Block"" WHERE ""VolumeID"" IN ({volIdsSubQuery}))"); // Create a temporary table to cache subquery result, as it might take long (SQLite does not cache at all). - deletecmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMP TABLE ""{0}"" (""ID"" INTEGER PRIMARY KEY)", blocksetidstable)); - deletecmd.ExecuteNonQuery(FormatInvariant(@"INSERT OR IGNORE INTO ""{0}"" (""ID"") {1}", blocksetidstable, bsIdsSubQuery)); - bsIdsSubQuery = FormatInvariant(@"SELECT DISTINCT ""ID"" FROM ""{0}"" ", blocksetidstable); + deletecmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMP TABLE ""{blocksetidstable}"" (""ID"" INTEGER PRIMARY KEY)")); + deletecmd.ExecuteNonQuery(FormatInvariant($@"INSERT OR IGNORE INTO ""{blocksetidstable}"" (""ID"") {bsIdsSubQuery}")); + bsIdsSubQuery = FormatInvariant($@"SELECT DISTINCT ""ID"" FROM ""{blocksetidstable}"" "); deletecmd.Parameters.Clear(); // Create a temp table to associate metadata that is being deleted to a fileset @@ -543,17 +533,17 @@ SELECT ID FROM FileLookup WHERE FileLookup.BlocksetID IN ({bsIdsSubQuery}))")); deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM FileLookup WHERE FileLookup.MetadataID IN (SELECT MetadataID FROM ""{metadataFilesetTable}"")")); - deletecmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""Metadataset"" WHERE ""BlocksetID"" IN ({0})", bsIdsSubQuery)); - deletecmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""FileLookup"" WHERE ""BlocksetID"" IN ({0})", bsIdsSubQuery)); - deletecmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""Blockset"" WHERE ""ID"" IN ({0})", bsIdsSubQuery)); - deletecmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ({0})", bsIdsSubQuery)); - deletecmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""BlocklistHash"" WHERE ""BlocklistHash"".""BlocksetID"" IN ({0})", bsIdsSubQuery)); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""Metadataset"" WHERE ""BlocksetID"" IN ({bsIdsSubQuery})")); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""FileLookup"" WHERE ""BlocksetID"" IN ({bsIdsSubQuery})")); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""Blockset"" WHERE ""ID"" IN ({bsIdsSubQuery})")); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ({bsIdsSubQuery})")); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""BlocklistHash"" WHERE ""BlocklistHash"".""BlocksetID"" IN ({bsIdsSubQuery})")); // If the volume is a block or index volume, this will update the crosslink table, otherwise nothing will happen - deletecmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN ({0}) OR ""IndexVolumeID"" IN ({0})", volIdsSubQuery)); - deletecmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""Block"" WHERE ""VolumeID"" IN ({0})", volIdsSubQuery)); - deletecmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""DeletedBlock"" WHERE ""VolumeID"" IN ({0})", volIdsSubQuery)); - deletecmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""ChangeJournalData"" WHERE ""FilesetID"" IN (SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ({0}))", volIdsSubQuery)); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN ({volIdsSubQuery}) OR ""IndexVolumeID"" IN ({volIdsSubQuery})")); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""Block"" WHERE ""VolumeID"" IN ({volIdsSubQuery})")); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""DeletedBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery})")); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""ChangeJournalData"" WHERE ""FilesetID"" IN (SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ({volIdsSubQuery}))")); deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM FilesetEntry WHERE FilesetID IN (SELECT ID FROM Fileset WHERE VolumeID IN ({volIdsSubQuery}))")); deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM Fileset WHERE VolumeID IN ({volIdsSubQuery})")); @@ -566,14 +556,14 @@ AND Fileset.ID NOT IN // Clean up temp tables for subqueries. We truncate content and then try to delete. // Drop in try-block, as it fails in nested transactions (SQLite problem) // System.Data.SQLite.SQLiteException (0x80004005): database table is locked - deletecmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""{0}"" ", blocksetidstable)); - deletecmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""{0}"" ", volidstable)); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{blocksetidstable}"" ")); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{volidstable}"" ")); try { deletecmd.CommandTimeout = 2; - deletecmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", blocksetidstable)); - deletecmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", volidstable)); - deletecmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", metadataFilesetTable)); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{blocksetidstable}"" ")); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{volidstable}"" ")); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{metadataFilesetTable}"" ")); } catch { /* Ignore, will be deleted on close anyway. */ } @@ -883,7 +873,7 @@ ON if (real_count != unique_count) throw new DatabaseInconsistencyException($"Found {real_count} blocklist hashes, but there should be {unique_count}. Run repair to fix it."); - var itemswithnoblocklisthash = cmd.ExecuteScalarInt64(FormatInvariant(@"SELECT COUNT(*) FROM (SELECT * FROM (SELECT ""N"".""BlocksetID"", ((""N"".""BlockCount"" + {0} - 1) / {0}) AS ""BlocklistHashCountExpected"", CASE WHEN ""G"".""BlocklistHashCount"" IS NULL THEN 0 ELSE ""G"".""BlocklistHashCount"" END AS ""BlocklistHashCountActual"" FROM (SELECT ""BlocksetID"", COUNT(*) AS ""BlockCount"" FROM ""BlocksetEntry"" GROUP BY ""BlocksetID"") ""N"" LEFT OUTER JOIN (SELECT ""BlocksetID"", COUNT(*) AS ""BlocklistHashCount"" FROM ""BlocklistHash"" GROUP BY ""BlocksetID"") ""G"" ON ""N"".""BlocksetID"" = ""G"".""BlocksetID"" WHERE ""N"".""BlockCount"" > 1) WHERE ""BlocklistHashCountExpected"" != ""BlocklistHashCountActual"")", blocksize / hashsize), 0); + var itemswithnoblocklisthash = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM (SELECT * FROM (SELECT ""N"".""BlocksetID"", ((""N"".""BlockCount"" + {blocksize / hashsize} - 1) / {blocksize / hashsize}) AS ""BlocklistHashCountExpected"", CASE WHEN ""G"".""BlocklistHashCount"" IS NULL THEN 0 ELSE ""G"".""BlocklistHashCount"" END AS ""BlocklistHashCountActual"" FROM (SELECT ""BlocksetID"", COUNT(*) AS ""BlockCount"" FROM ""BlocksetEntry"" GROUP BY ""BlocksetID"") ""N"" LEFT OUTER JOIN (SELECT ""BlocksetID"", COUNT(*) AS ""BlocklistHashCount"" FROM ""BlocklistHash"" GROUP BY ""BlocksetID"") ""G"" ON ""N"".""BlocksetID"" = ""G"".""BlocksetID"" WHERE ""N"".""BlockCount"" > 1) WHERE ""BlocklistHashCountExpected"" != ""BlocklistHashCountActual"")"), 0); if (itemswithnoblocklisthash != 0) throw new DatabaseInconsistencyException($"Found {itemswithnoblocklisthash} file(s) with missing blocklist hashes"); @@ -926,7 +916,7 @@ ON { foreach (var filesetid in cmd.ExecuteReaderEnumerable(@"SELECT ""ID"" FROM ""Fileset"" ").Select(x => x.ConvertValueToInt64(0, -1))) { - var expandedCmd = FormatInvariant(@"SELECT COUNT(*) FROM (SELECT DISTINCT ""Path"" FROM ({0}) UNION SELECT DISTINCT ""Path"" FROM ({1}))", LocalDatabase.LIST_FILESETS, LocalDatabase.LIST_FOLDERS_AND_SYMLINKS); + var expandedCmd = FormatInvariant($@"SELECT COUNT(*) FROM (SELECT DISTINCT ""Path"" FROM ({LocalDatabase.LIST_FILESETS}) UNION SELECT DISTINCT ""Path"" FROM ({LocalDatabase.LIST_FOLDERS_AND_SYMLINKS}))"); var expandedlist = cmd2.ExecuteScalarInt64(expandedCmd, 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, filesetid); //var storedfilelist = cmd2.ExecuteScalarInt64(FormatInvariant(@"SELECT COUNT(*) FROM ""FilesetEntry"", ""FileLookup"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FileLookup"".""BlocksetID"" != ? AND ""FileLookup"".""BlocksetID"" != ?"), 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID); var storedlist = cmd2.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetEntry"".""FilesetID"" = ?", 0, filesetid); @@ -1319,7 +1309,7 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != ? ORDER BY Timest using (var cmd = m_connection.CreateCommand()) { cmd.Transaction = transaction; - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" AS SELECT DISTINCT ""Path"" FROM ""File"" ", Tablename)); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{Tablename}"" AS SELECT DISTINCT ""Path"" FROM ""File"" ")); return; } } @@ -1331,10 +1321,10 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != ? ORDER BY Timest // TODO: Optimize this to not rely on the "File" view, and not instantiate the paths in full cmd.Transaction = transaction; - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL)", Tablename)); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{Tablename}"" (""Path"" TEXT NOT NULL)")); using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) { - cmd.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" (""Path"") VALUES (?)", Tablename); + cmd.CommandText = FormatInvariant($@"INSERT INTO ""{Tablename}"" (""Path"") VALUES (?)"); cmd.AddParameter(); cmd.Transaction = tr.Parent; using (var c2 = m_connection.CreateCommand()) @@ -1378,8 +1368,8 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != ? ORDER BY Timest using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) { cmd.Transaction = tr.Parent; - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL)", Tablename)); - cmd.ExecuteNonQuery(FormatInvariant(@"INSERT INTO ""{0}"" SELECT DISTINCT ""Path"" FROM ""File"" WHERE " + sb, Tablename), args.ToArray()); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{Tablename}"" (""Path"" TEXT NOT NULL)")); + cmd.ExecuteNonQuery(FormatInvariant($@"INSERT INTO ""{Tablename}"" SELECT DISTINCT ""Path"" FROM ""File"" WHERE {sb}"), args.ToArray()); tr.Commit(); } } @@ -1391,7 +1381,7 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != ? ORDER BY Timest try { using (var cmd = m_connection.CreateCommand()) - cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", Tablename)); + cmd.ExecuteNonQuery(FormatInvariant(@$"DROP TABLE IF EXISTS ""{Tablename}"" ")); } catch { } finally { Tablename = null; } @@ -1460,13 +1450,11 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != ? ORDER BY Timest using (var cmd = m_connection.CreateCommand(transaction)) { // Group subquery by hash to ensure that each blocklist hash appears only once in the result - var sql = FormatInvariant(@"SELECT ""A"".""Hash"", ""C"".""Hash"" FROM " + - @"(SELECT ""BlocklistHash"".""BlocksetID"", ""Block"".""Hash"", ""BlocklistHash"".""Index"" FROM ""BlocklistHash"",""Block"" WHERE ""BlocklistHash"".""Hash"" = ""Block"".""Hash"" AND ""Block"".""VolumeID"" = ? GROUP BY ""Block"".""Hash"", ""Block"".""Size"") A, " + - @" ""BlocksetEntry"" B, ""Block"" C WHERE ""B"".""BlocksetID"" = ""A"".""BlocksetID"" AND " + - @" ""B"".""Index"" >= (""A"".""Index"" * {0}) AND ""B"".""Index"" < ((""A"".""Index"" + 1) * {0}) AND ""C"".""ID"" = ""B"".""BlockID"" " + - @" ORDER BY ""A"".""BlocksetID"", ""B"".""Index""", - blocksize / hashsize - ); + var sql = FormatInvariant($@"SELECT ""A"".""Hash"", ""C"".""Hash"" FROM +(SELECT ""BlocklistHash"".""BlocksetID"", ""Block"".""Hash"", ""BlocklistHash"".""Index"" FROM ""BlocklistHash"",""Block"" WHERE ""BlocklistHash"".""Hash"" = ""Block"".""Hash"" AND ""Block"".""VolumeID"" = ? GROUP BY ""Block"".""Hash"", ""Block"".""Size"") A, + ""BlocksetEntry"" B, ""Block"" C WHERE ""B"".""BlocksetID"" = ""A"".""BlocksetID"" AND + ""B"".""Index"" >= (""A"".""Index"" * {blocksize / hashsize}) AND ""B"".""Index"" < ((""A"".""Index"" + 1) * {blocksize / hashsize}) AND ""C"".""ID"" = ""B"".""BlockID"" + ORDER BY ""A"".""BlocksetID"", ""B"".""Index"""); string curHash = null; int count = 0; diff --git a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs index fb5f5ae6e..0e8910e55 100644 --- a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs @@ -185,7 +185,7 @@ namespace Duplicati.Library.Main.Database var combined = active + " UNION " + inactive + " UNION " + empty; var collected = @"SELECT ""VolumeID"" AS ""VolumeID"", SUM(""ActiveSize"") AS ""ActiveSize"", SUM(""InactiveSize"") AS ""InactiveSize"", MAX(""Sorttime"") AS ""Sorttime"" FROM (" + combined + @") GROUP BY ""VolumeID"" "; - var createtable = FormatInvariant(@$"CREATE {TEMPORARY} TABLE ""{tmptablename}"" AS " + collected); + var createtable = FormatInvariant($"{@$"CREATE {TEMPORARY} TABLE ""{tmptablename}"" AS "}{collected}"); using (var cmd = m_connection.CreateCommand()) { @@ -193,13 +193,13 @@ namespace Duplicati.Library.Main.Database try { cmd.ExecuteNonQuery(createtable, RemoteVolumeType.Blocks.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()); - using (var rd = cmd.ExecuteReader(FormatInvariant(@"SELECT ""A"".""Name"", ""B"".""ActiveSize"", ""B"".""InactiveSize"", ""A"".""Size"" FROM ""Remotevolume"" A, ""{0}"" B WHERE ""A"".""ID"" = ""B"".""VolumeID"" ORDER BY ""B"".""Sorttime"" ASC ", tmptablename))) + using (var rd = cmd.ExecuteReader(FormatInvariant($@"SELECT ""A"".""Name"", ""B"".""ActiveSize"", ""B"".""InactiveSize"", ""A"".""Size"" FROM ""Remotevolume"" A, ""{tmptablename}"" B WHERE ""A"".""ID"" = ""B"".""VolumeID"" ORDER BY ""B"".""Sorttime"" ASC "))) while (rd.Read()) yield return new VolumeUsage(rd.GetValue(0).ToString(), rd.ConvertValueToInt64(1, 0) + rd.ConvertValueToInt64(2, 0), rd.ConvertValueToInt64(2, 0), rd.ConvertValueToInt64(3, 0)); } finally { - try { cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", tmptablename)); } + try { cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{tmptablename}"" ")); } catch { } } } @@ -388,9 +388,9 @@ namespace Duplicati.Library.Main.Database var replacementBlocks = "ReplacementBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); try { - cmd.ExecuteNonQuery(FormatInvariant(@$"CREATE {TEMPORARY} TABLE """ + updatedBlocks + @""" AS SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" = ? "), deletedVolume.ID); - cmd.ExecuteNonQuery(FormatInvariant(@$"CREATE {TEMPORARY} TABLE """ + replacementBlocks + @""" AS SELECT ""BlockID"", MAX(""VolumeID"") AS ""VolumeID"" FROM ""DuplicateBlock"" WHERE ""VolumeID"" NOT IN (?) AND ""BlockID"" IN (SELECT ""ID"" FROM """ + updatedBlocks + @""") GROUP BY ""BlockID"" "), volumeIdsToBeRemoved.ToArray()); - var targetCount = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM """ + updatedBlocks + @""" "); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE {TEMPORARY} TABLE ""{updatedBlocks}"" AS SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" = ? "), deletedVolume.ID); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE {TEMPORARY} TABLE ""{replacementBlocks}"" AS SELECT ""BlockID"", MAX(""VolumeID"") AS ""VolumeID"" FROM ""DuplicateBlock"" WHERE ""VolumeID"" NOT IN (?) AND ""BlockID"" IN (SELECT ""ID"" FROM ""{updatedBlocks}"") GROUP BY ""BlockID"" "), volumeIdsToBeRemoved.ToArray()); + var targetCount = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM ""{updatedBlocks}"" ")); if (targetCount == 0) return; @@ -401,13 +401,13 @@ namespace Duplicati.Library.Main.Database throw new Exception($"Unexpected number of rows updated. Expected {targetCount} but got updated {updateCount}, deleted {deleteCount}, and replaced {replacementCount}"); // Remove knowledge of any old blocks - cmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""DuplicateBlock"" WHERE ""VolumeID"" = ?"), deletedVolume.ID); + cmd.ExecuteNonQuery(FormatInvariant(@$"DELETE FROM ""DuplicateBlock"" WHERE ""VolumeID"" = ?"), deletedVolume.ID); } finally { - try { cmd.ExecuteNonQuery(@"DROP TABLE IF EXISTS """ + updatedBlocks + @""" "); } + try { cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{updatedBlocks}"" ")); } catch { } - try { cmd.ExecuteNonQuery(@"DROP TABLE IF EXISTS """ + replacementBlocks + @""" "); } + try { cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{replacementBlocks}"" ")); } catch { } } diff --git a/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs b/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs index 4317fa11b..c9ed24200 100644 --- a/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs @@ -63,27 +63,25 @@ namespace Duplicati.Library.Main.Database { var filesets = FilesetTimes.ToArray(); var dict = new Dictionary(); - for(var i = 0; i < filesets.Length; i++) + for (var i = 0; i < filesets.Length; i++) dict[filesets[i].Key] = i; - var sql = FormatInvariant( - @"SELECT DISTINCT ""FilesetID"" FROM (" + - @"SELECT ""FilesetID"" FROM ""FilesetEntry"" WHERE ""FileID"" IN ( SELECT ""ID"" FROM ""FileLookup"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN ( SELECT ""ID"" From ""Block"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN ({0})))))" + - " UNION " + - @"SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN ({0}))" + - ")", - string.Join(",", items.Select(x => "?")) - ); + var sql = FormatInvariant($@"SELECT DISTINCT ""FilesetID"" FROM ( +SELECT ""FilesetID"" FROM ""FilesetEntry"" WHERE ""FileID"" IN ( SELECT ""ID"" FROM ""FileLookup"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN ( SELECT ""ID"" From ""Block"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN ({string.Join(",", items.Select(x => "?"))}))))) +UNION +SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN ({string.Join(",", items.Select(x => "?"))})) +)"); var it = new List(items); it.AddRange(items); - using(var cmd = m_connection.CreateCommand()) - using(var rd = cmd.ExecuteReader(sql, it.ToArray())) + using (var cmd = m_connection.CreateCommand()) + using (var rd = cmd.ExecuteReader(sql, it.ToArray())) while (rd.Read()) { var v = dict[rd.GetInt64(0)]; - yield return new ListResultFileset() { + yield return new ListResultFileset() + { Version = v, Time = filesets[v].Value }; @@ -92,47 +90,43 @@ namespace Duplicati.Library.Main.Database public IEnumerable GetFiles(IEnumerable items) { - var sql = FormatInvariant( - @"SELECT DISTINCT ""Path"" FROM (" + - @"SELECT ""Path"" FROM ""File"" WHERE ""BlocksetID"" IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN (SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" IN (SELECT ""ID"" from ""RemoteVolume"" WHERE ""Name"" IN ({0}))))" + - @" UNION " + - @"SELECT ""Path"" FROM ""File"" WHERE ""MetadataID"" IN (SELECT ""ID"" FROM ""Metadataset"" WHERE ""BlocksetID"" IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN (SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" IN (SELECT ""ID"" from ""RemoteVolume"" WHERE ""Name"" IN ({0})))))" + - @" UNION " + - @"SELECT ""Path"" FROM ""File"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" IN ( SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN ({0}))))" + - @") ORDER BY ""Path"" ", - string.Join(",", items.Select(x => "?")) - ); - + var sql = FormatInvariant($@"SELECT DISTINCT ""Path"" FROM ( +SELECT ""Path"" FROM ""File"" WHERE ""BlocksetID"" IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN (SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" IN (SELECT ""ID"" from ""RemoteVolume"" WHERE ""Name"" IN ({string.Join(",", items.Select(x => "?"))})))) + UNION +SELECT ""Path"" FROM ""File"" WHERE ""MetadataID"" IN (SELECT ""ID"" FROM ""Metadataset"" WHERE ""BlocksetID"" IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN (SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" IN (SELECT ""ID"" from ""RemoteVolume"" WHERE ""Name"" IN ({string.Join(",", items.Select(x => "?"))}))))) + UNION +SELECT ""Path"" FROM ""File"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" IN ( SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN ({string.Join(",", items.Select(x => "?"))})))) +) ORDER BY ""Path"""); + var it = new List(items); it.AddRange(items); it.AddRange(items); - using(var cmd = m_connection.CreateCommand()) - using(var rd = cmd.ExecuteReader(sql, it.ToArray())) + using (var cmd = m_connection.CreateCommand()) + using (var rd = cmd.ExecuteReader(sql, it.ToArray())) while (rd.Read()) - yield return new ListResultFile() { + yield return new ListResultFile() + { Path = Convert.ToString(rd.GetValue(0)) }; - + } public IEnumerable GetLogLines(IEnumerable items) { - var sql = FormatInvariant( - @"SELECT ""TimeStamp"", ""Message"" || ' ' || CASE WHEN ""Exception"" IS NULL THEN '' ELSE ""Exception"" END FROM ""LogData"" WHERE {0}" + - @" UNION " + - @"SELECT ""Timestamp"", ""Data"" FROM ""RemoteOperation"" WHERE ""Path"" IN ({1})", - string.Join(" OR ", items.Select(x => @"""Message"" LIKE ?")), - string.Join(",", items.Select(x => "?")) - ); + var sql = FormatInvariant($@" +SELECT ""TimeStamp"", ""Message"" || ' ' || CASE WHEN ""Exception"" IS NULL THEN '' ELSE ""Exception"" END FROM ""LogData"" WHERE {string.Join(" OR ", items.Select(x => @"""Message"" LIKE ?"))} +UNION +SELECT ""Timestamp"", ""Data"" FROM ""RemoteOperation"" WHERE ""Path"" IN ({string.Join(",", items.Select(x => "?"))})"); var it = new List(from n in items select "%" + n + "%"); it.AddRange(items); - using(var cmd = m_connection.CreateCommand()) - using(var rd = cmd.ExecuteReader(sql, it.ToArray())) + using (var cmd = m_connection.CreateCommand()) + using (var rd = cmd.ExecuteReader(sql, it.ToArray())) while (rd.Read()) - yield return new ListResultRemoteLog() { + yield return new ListResultRemoteLog() + { Timestamp = ParseFromEpochSeconds(rd.GetInt64(0)), Message = rd.GetString(1) }; @@ -140,22 +134,19 @@ namespace Duplicati.Library.Main.Database public IEnumerable GetVolumes(IEnumerable items) { - var sql = FormatInvariant( - @"SELECT DISTINCT ""Name"" FROM ( " + - @" SELECT ""Name"" FROM ""Remotevolume"" WHERE ""ID"" IN ( SELECT ""VolumeID"" FROM ""Block"" WHERE ""ID"" IN ( SELECT ""BlockID"" FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""FileLookup"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" IN ( SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN ({0}))))))) " + - @" UNION " + - @" SELECT ""Name"" FROM ""Remotevolume"" WHERE ""ID"" IN ( SELECT ""VolumeID"" FROM ""Block"" WHERE ""ID"" IN ( SELECT ""BlockID"" FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""Metadataset"" WHERE ""ID"" IN ( SELECT ""MetadataID"" FROM ""FileLookup"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" IN ( SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN ({0}))))))))" + - @")", - string.Join(",", items.Select(x => "?")) - ); - + var sql = FormatInvariant($@"SELECT DISTINCT ""Name"" FROM ( +SELECT ""Name"" FROM ""Remotevolume"" WHERE ""ID"" IN ( SELECT ""VolumeID"" FROM ""Block"" WHERE ""ID"" IN ( SELECT ""BlockID"" FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""FileLookup"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" IN ( SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN ({string.Join(",", items.Select(x => "?"))}))))))) +UNION +SELECT ""Name"" FROM ""Remotevolume"" WHERE ""ID"" IN ( SELECT ""VolumeID"" FROM ""Block"" WHERE ""ID"" IN ( SELECT ""BlockID"" FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""Metadataset"" WHERE ""ID"" IN ( SELECT ""MetadataID"" FROM ""FileLookup"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" IN ( SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN ({string.Join(",", items.Select(x => "?"))})))))))) +)"); var it = new List(items); it.AddRange(items); - using(var cmd = m_connection.CreateCommand()) - using(var rd = cmd.ExecuteReader(sql, it.ToArray())) + using (var cmd = m_connection.CreateCommand()) + using (var rd = cmd.ExecuteReader(sql, it.ToArray())) while (rd.Read()) - yield return new ListResultRemoteVolume() { + yield return new ListResultRemoteVolume() + { Name = Convert.ToString(rd.GetValue(0)) }; } diff --git a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs index a72ba7df5..75dd81aae 100644 --- a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs @@ -27,7 +27,7 @@ namespace Duplicati.Library.Main.Database { internal class LocalListBrokenFilesDatabase : LocalDatabase { - private const string BLOCK_VOLUME_IDS = @"SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = '{2}'"; + private static readonly string BLOCK_VOLUME_IDS = FormatInvariant($@"SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks.ToString()}'"); // Invalid blocksets include those that: // - Have BlocksetEntries with unknown/invalid blocks (meaning the data to rebuild the blockset isn't available) @@ -35,9 +35,9 @@ namespace Duplicati.Library.Main.Database // - Have BlocklistHash entries with unknown/invalid blocks (meaning the data which defines the list of hashes that makes up the blockset isn't available) // - Are defined in the Blockset table but have no entries in the BlocksetEntries table (this can happen during recreate if Files volumes reference blocksets that are not found in any Index files) // However, blocksets with a length of 0 are excluded from this check, as the corresponding blocks for these are not needed. - private const string BROKEN_FILE_IDS = @" + private static readonly string BROKEN_FILE_IDS = FormatInvariant($@" SELECT DISTINCT ""ID"" FROM ( - SELECT ""ID"" AS ""ID"", ""BlocksetID"" AS ""BlocksetID"" FROM ""FileLookup"" WHERE ""BlocksetID"" != {0} AND ""BlocksetID"" != {1} + SELECT ""ID"" AS ""ID"", ""BlocksetID"" AS ""BlocksetID"" FROM ""FileLookup"" WHERE ""BlocksetID"" != {FOLDER_BLOCKSET_ID} AND ""BlocksetID"" != {SYMLINK_BLOCKSET_ID} UNION SELECT ""A"".""ID"" AS ""ID"", ""B"".""BlocksetID"" AS ""BlocksetID"" FROM ""FileLookup"" A LEFT JOIN ""Metadataset"" B ON ""A"".""MetadataID"" = ""B"".""ID"" ) @@ -47,26 +47,22 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" NOT IN (SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" IN - (" + BLOCK_VOLUME_IDS + @")) + ({BLOCK_VOLUME_IDS})) UNION SELECT ""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Hash"" NOT IN (SELECT ""Hash"" FROM ""Block"" WHERE ""VolumeID"" IN - (" + BLOCK_VOLUME_IDS + @")) + ({BLOCK_VOLUME_IDS})) UNION SELECT ""A"".""ID"" AS ""BlocksetID"" FROM ""Blockset"" A LEFT JOIN ""BlocksetEntry"" B ON ""A"".""ID"" = ""B"".""BlocksetID"" WHERE ""A"".""Length"" > 0 AND ""B"".""BlocksetID"" IS NULL ) WHERE ""BlocksetID"" NOT IN (SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" == 0) ) -"; - private const string BROKEN_FILE_SETS = @"SELECT DISTINCT ""B"".""Timestamp"", ""A"".""FilesetID"", COUNT(""A"".""FileID"") AS ""FileCount"" FROM ""FilesetEntry"" A, ""Fileset"" B WHERE ""A"".""FilesetID"" = ""B"".""ID"" AND ""A"".""FileID"" IN (" + BROKEN_FILE_IDS + @")"; +"); + private static readonly string BROKEN_FILE_SETS = FormatInvariant($@"SELECT DISTINCT ""B"".""Timestamp"", ""A"".""FilesetID"", COUNT(""A"".""FileID"") AS ""FileCount"" FROM ""FilesetEntry"" A, ""Fileset"" B WHERE ""A"".""FilesetID"" = ""B"".""ID"" AND ""A"".""FileID"" IN ({BROKEN_FILE_IDS})"); - private const string BROKEN_FILE_NAMES = @" -SELECT ""A"".""Path"", ""B"".""Length"" FROM ""File"" A, ""Blockset"" B WHERE ""A"".""BlocksetID"" = ""B"".""ID"" AND ""A"".""ID"" IN (" -+ BROKEN_FILE_IDS + -@") AND ""A"".""ID"" IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = ?)"; + private static readonly string BROKEN_FILE_NAMES = FormatInvariant($@"SELECT ""A"".""Path"", ""B"".""Length"" FROM ""File"" A, ""Blockset"" B WHERE ""A"".""BlocksetID"" = ""B"".""ID"" AND ""A"".""ID"" IN ({BROKEN_FILE_IDS}) AND ""A"".""ID"" IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = ?)"); - private const string INSERT_BROKEN_IDS = @"INSERT INTO ""{3}"" (""{4}"") " + BROKEN_FILE_IDS - + @" AND ""ID"" IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = ?)"; + private static string INSERT_BROKEN_IDS(string tablename, string IDfieldname) => FormatInvariant($@"INSERT INTO ""{tablename}"" (""{IDfieldname}"") {BROKEN_FILE_IDS} AND ""ID"" IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = ?)"); public LocalListBrokenFilesDatabase(string path) : base(path, "ListBrokenFiles", false) @@ -82,10 +78,10 @@ SELECT ""A"".""Path"", ""B"".""Length"" FROM ""File"" A, ""Blockset"" B WHERE "" public IEnumerable> GetBrokenFilesets(DateTime time, long[] versions, System.Data.IDbTransaction transaction) { - var query = FormatInvariant(BROKEN_FILE_SETS, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, RemoteVolumeType.Blocks.ToString()); + var query = BROKEN_FILE_SETS; var clause = GetFilelistWhereClause(time, versions); if (!string.IsNullOrWhiteSpace(clause.Item1)) - query += @" AND ""A"".""FilesetID"" IN (SELECT ""ID"" FROM ""Fileset"" " + clause.Item1 + ")"; + query += FormatInvariant($@" AND ""A"".""FilesetID"" IN (SELECT ""ID"" FROM ""Fileset"" {clause.Item1})"); query += @" GROUP BY ""A"".""FilesetID"""; @@ -98,7 +94,7 @@ SELECT ""A"".""Path"", ""B"".""Length"" FROM ""File"" A, ""Blockset"" B WHERE "" public IEnumerable> GetBrokenFilenames(long filesetid, System.Data.IDbTransaction transaction) { using (var cmd = Connection.CreateCommand(transaction)) - foreach (var rd in cmd.ExecuteReaderEnumerable(FormatInvariant(BROKEN_FILE_NAMES, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, RemoteVolumeType.Blocks.ToString()), filesetid)) + foreach (var rd in cmd.ExecuteReaderEnumerable(BROKEN_FILE_NAMES, filesetid)) if (!rd.IsDBNull(0)) yield return new Tuple(rd.ConvertValueToString(0), rd.ConvertValueToInt64(1)); } @@ -106,7 +102,7 @@ SELECT ""A"".""Path"", ""B"".""Length"" FROM ""File"" A, ""Blockset"" B WHERE "" public void InsertBrokenFileIDsIntoTable(long filesetid, string tablename, string IDfieldname, System.Data.IDbTransaction transaction) { using (var cmd = Connection.CreateCommand(transaction)) - cmd.ExecuteNonQuery(FormatInvariant(INSERT_BROKEN_IDS, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, RemoteVolumeType.Blocks.ToString(), tablename, IDfieldname), filesetid); + cmd.ExecuteNonQuery(INSERT_BROKEN_IDS(tablename, IDfieldname), filesetid); } public void RemoveMissingBlocks(IEnumerable names, System.Data.IDbTransaction transaction) @@ -121,8 +117,8 @@ SELECT ""A"".""Path"", ""B"".""Length"" FROM ""File"" A, ""Blockset"" B WHERE "" var volidstable = "DelVolSetIds-" + temptransguid; // Create and fill a temp table with the volids to delete. We avoid using too many parameters that way. - deletecmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMP TABLE ""{0}"" (""ID"" INTEGER PRIMARY KEY)", volidstable)); - deletecmd.CommandText = FormatInvariant(@"INSERT OR IGNORE INTO ""{0}"" (""ID"") VALUES (?)", volidstable); + deletecmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMP TABLE ""{volidstable}"" (""ID"" INTEGER PRIMARY KEY)")); + deletecmd.CommandText = FormatInvariant($@"INSERT OR IGNORE INTO ""{volidstable}"" (""ID"") VALUES (?)"); deletecmd.Parameters.Clear(); deletecmd.AddParameters(1); foreach (var name in names) @@ -131,22 +127,22 @@ SELECT ""A"".""Path"", ""B"".""Length"" FROM ""File"" A, ""Blockset"" B WHERE "" deletecmd.SetParameterValue(0, volumeid); deletecmd.ExecuteNonQuery(); } - var volIdsSubQuery = FormatInvariant(@"SELECT ""ID"" FROM ""{0}"" ", volidstable); + var volIdsSubQuery = FormatInvariant($@"SELECT ""ID"" FROM ""{volidstable}"" "); deletecmd.Parameters.Clear(); - deletecmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN ({0}) OR ""IndexVolumeID"" IN ({0})", volIdsSubQuery)); - deletecmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""Block"" WHERE ""VolumeID"" IN ({0})", volIdsSubQuery)); - deletecmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""DeletedBlock"" WHERE ""VolumeID"" IN ({0})", volIdsSubQuery)); - deletecmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""DuplicateBlock"" WHERE ""VolumeID"" IN ({0})", volIdsSubQuery)); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN ({volIdsSubQuery}) OR ""IndexVolumeID"" IN ({volIdsSubQuery})")); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""Block"" WHERE ""VolumeID"" IN ({volIdsSubQuery})")); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""DeletedBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery})")); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""DuplicateBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery})")); // Clean up temp tables for subqueries. We truncate content and then try to delete. // Drop in try-block, as it fails in nested transactions (SQLite problem) // System.Data.SQLite.SQLiteException (0x80004005): database table is locked - deletecmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""{0}"" ", volidstable)); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{volidstable}"" ")); try { deletecmd.CommandTimeout = 2; - deletecmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", volidstable)); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{volidstable}"" ")); } catch { /* Ignore, will be deleted on close anyway. */ } } diff --git a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs index 031d7c3cc..f58c856dc 100644 --- a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs @@ -32,13 +32,13 @@ namespace Duplicati.Library.Main.Database { ShouldCloseConnection = true; } - + public interface IStorageHelper : IDisposable { void AddElement(string path, string filehash, string metahash, long size, Library.Interface.ListChangesElementType type, bool asNew); - + void AddFromDb(long filesetId, bool asNew, Library.Utility.IFilter filter); - + IChangeCountReport CreateChangeCountReport(); IChangeSizeReport CreateChangeSizeReport(); IEnumerable> CreateChangedFileReport(); @@ -49,11 +49,11 @@ namespace Duplicati.Library.Main.Database long AddedFolders { get; } long AddedSymlinks { get; } long AddedFiles { get; } - + long DeletedFolders { get; } long DeletedSymlinks { get; } long DeletedFiles { get; } - + long ModifiedFolders { get; } long ModifiedSymlinks { get; } long ModifiedFiles { get; } @@ -63,21 +63,21 @@ namespace Duplicati.Library.Main.Database { long AddedSize { get; } long DeletedSize { get; } - + long PreviousSize { get; } long CurrentSize { get; } } - + internal class ChangeCountReport : IChangeCountReport { - public long AddedFolders { get; internal set; } - public long AddedSymlinks { get; internal set; } + public long AddedFolders { get; internal set; } + public long AddedSymlinks { get; internal set; } public long AddedFiles { get; internal set; } - + public long DeletedFolders { get; internal set; } - public long DeletedSymlinks { get; internal set; } - public long DeletedFiles { get; internal set; } - + public long DeletedSymlinks { get; internal set; } + public long DeletedFiles { get; internal set; } + public long ModifiedFolders { get; internal set; } public long ModifiedSymlinks { get; internal set; } public long ModifiedFiles { get; internal set; } @@ -87,66 +87,66 @@ namespace Duplicati.Library.Main.Database { public long AddedSize { get; internal set; } public long DeletedSize { get; internal set; } - + public long PreviousSize { get; internal set; } public long CurrentSize { get; internal set; } } - + private class StorageHelper : IStorageHelper { private readonly System.Data.IDbConnection m_connection; private System.Data.IDbTransaction m_transaction; - + private System.Data.IDbCommand m_insertPreviousElementCommand; private System.Data.IDbCommand m_insertCurrentElementCommand; - + private string m_previousTable; private string m_currentTable; - + public StorageHelper(System.Data.IDbConnection con) { m_connection = con; m_previousTable = "Previous-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); m_currentTable = "Current-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - + m_transaction = m_connection.BeginTransaction(); - - using(var cmd = m_connection.CreateCommand()) + + using (var cmd = m_connection.CreateCommand()) { cmd.Transaction = m_transaction; - - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) ", m_previousTable)); - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) ", m_currentTable)); + + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_previousTable}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) ")); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_currentTable}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) ")); } - + m_insertPreviousElementCommand = m_connection.CreateCommand(); m_insertPreviousElementCommand.Transaction = m_transaction; - m_insertPreviousElementCommand.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (?,?,?,?,?)", m_previousTable); + m_insertPreviousElementCommand.CommandText = FormatInvariant($@"INSERT INTO ""{m_previousTable}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (?,?,?,?,?)"); m_insertPreviousElementCommand.AddParameters(5); - + m_insertCurrentElementCommand = m_connection.CreateCommand(); m_insertCurrentElementCommand.Transaction = m_transaction; - m_insertCurrentElementCommand.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (?,?,?,?,?)", m_currentTable); + m_insertCurrentElementCommand.CommandText = FormatInvariant($@"INSERT INTO ""{m_currentTable}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (?,?,?,?,?)"); m_insertCurrentElementCommand.AddParameters(5); } - + public void AddFromDb(long filesetId, bool asNew, Library.Utility.IFilter filter) { var tablename = asNew ? m_currentTable : m_previousTable; - - var folders = FormatInvariant(@"SELECT ""File"".""Path"" AS ""Path"", NULL AS ""FileHash"", ""Blockset"".""Fullhash"" AS ""MetaHash"", -1 AS ""Size"", {0} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" = -100 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" ", (int)Library.Interface.ListChangesElementType.Folder); - var symlinks = FormatInvariant(@"SELECT ""File"".""Path"" AS ""Path"", NULL AS ""FileHash"", ""Blockset"".""Fullhash"" AS ""MetaHash"", -1 AS ""Size"", {0} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" = -200 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" ", (int)Library.Interface.ListChangesElementType.Symlink); - var files = FormatInvariant(@"SELECT ""File"".""Path"" AS ""Path"", ""FB"".""FullHash"" AS ""FileHash"", ""MB"".""Fullhash"" AS ""MetaHash"", ""FB"".""Length"" AS ""Size"", {0} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" MB, ""Blockset"" FB WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" >= 0 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""MB"".""ID"" AND ""File"".""BlocksetID"" = ""FB"".""ID"" ", (int)Library.Interface.ListChangesElementType.File); + + var folders = FormatInvariant($@"SELECT ""File"".""Path"" AS ""Path"", NULL AS ""FileHash"", ""Blockset"".""Fullhash"" AS ""MetaHash"", -1 AS ""Size"", {(int)Library.Interface.ListChangesElementType.Folder} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" = -100 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" "); + var symlinks = FormatInvariant($@"SELECT ""File"".""Path"" AS ""Path"", NULL AS ""FileHash"", ""Blockset"".""Fullhash"" AS ""MetaHash"", -1 AS ""Size"", {(int)Library.Interface.ListChangesElementType.Symlink} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" = -200 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" "); + var files = FormatInvariant($@"SELECT ""File"".""Path"" AS ""Path"", ""FB"".""FullHash"" AS ""FileHash"", ""MB"".""Fullhash"" AS ""MetaHash"", ""FB"".""Length"" AS ""Size"", {(int)Library.Interface.ListChangesElementType.File} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" MB, ""Blockset"" FB WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" >= 0 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""MB"".""ID"" AND ""File"".""BlocksetID"" = ""FB"".""ID"" "); var combined = "(" + folders + " UNION " + symlinks + " UNION " + files + ")"; - - - using(var cmd = m_connection.CreateCommand()) + + + using (var cmd = m_connection.CreateCommand()) { cmd.Transaction = m_transaction; if (filter == null || filter.Empty) { // Simple case, select everything - cmd.ExecuteNonQuery(FormatInvariant(@"INSERT INTO ""{0}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") SELECT ""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"" FROM {1} A WHERE ""A"".""FilesetID"" = ? ", tablename, combined), filesetId); + cmd.ExecuteNonQuery(FormatInvariant($@"INSERT INTO ""{tablename}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") SELECT ""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"" FROM {combined} A WHERE ""A"".""FilesetID"" = ? "), filesetId); } else if (Library.Utility.Utility.IsFSCaseSensitive && filter is FilterExpression expression && expression.Type == Duplicati.Library.Utility.FilterType.Simple) { @@ -155,11 +155,11 @@ namespace Duplicati.Library.Main.Database // SQLite only supports ASCII compares var p = expression.GetSimpleList(); var filenamestable = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL) ", filenamestable)); - cmd.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" (""Path"") VALUES (?)", filenamestable); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{filenamestable}"" (""Path"" TEXT NOT NULL) ")); + cmd.CommandText = FormatInvariant($@"INSERT INTO ""{filenamestable}"" (""Path"") VALUES (?)"); cmd.AddParameter(); - - foreach(var s in p) + + foreach (var s in p) { cmd.SetParameterValue(0, s); cmd.ExecuteNonQuery(); @@ -169,27 +169,27 @@ namespace Duplicati.Library.Main.Database if (expression.Result) { // Include filter - whereClause = FormatInvariant(@"""A"".""FilesetID"" = ? AND ""A"".""Path"" IN (SELECT DISTINCT ""Path"" FROM ""{0}"")", filenamestable); + whereClause = FormatInvariant($@"""A"".""FilesetID"" = ? AND ""A"".""Path"" IN (SELECT DISTINCT ""Path"" FROM ""{filenamestable}"")"); } else { // Exclude filter - whereClause = FormatInvariant(@"""A"".""FilesetID"" = ? AND ""A"".""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""{0}"")", filenamestable); + whereClause = FormatInvariant($@"""A"".""FilesetID"" = ? AND ""A"".""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""{filenamestable}"")"); } - cmd.ExecuteNonQuery(FormatInvariant(@"INSERT INTO ""{0}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") SELECT ""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"" FROM {1} A WHERE {2} ", tablename, combined, whereClause), filesetId); - cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", filenamestable)); + cmd.ExecuteNonQuery(FormatInvariant($@"INSERT INTO ""{tablename}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") SELECT ""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"" FROM {combined} A WHERE {whereClause} "), filesetId); + cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{filenamestable}"" ")); } else { // Do row-wise iteration object[] values = new object[5]; - using(var cmd2 = m_connection.CreateCommand()) + using (var cmd2 = m_connection.CreateCommand()) { - cmd2.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (?,?,?,?,?)", tablename); + cmd2.CommandText = FormatInvariant($@"INSERT INTO ""{tablename}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (?,?,?,?,?)"); cmd2.AddParameters(5); cmd2.Transaction = m_transaction; - - using(var rd = cmd.ExecuteReader(FormatInvariant(@"SELECT ""A"".""Path"", ""A"".""FileHash"", ""A"".""MetaHash"", ""A"".""Size"", ""A"".""Type"" FROM {0} A WHERE ""A"".""FilesetID"" = ?", combined), filesetId)) + + using (var rd = cmd.ExecuteReader(FormatInvariant($@"SELECT ""A"".""Path"", ""A"".""FileHash"", ""A"".""MetaHash"", ""A"".""Size"", ""A"".""Type"" FROM {combined} A WHERE ""A"".""FilesetID"" = ?"), filesetId)) while (rd.Read()) { rd.GetValues(values); @@ -207,7 +207,7 @@ namespace Duplicati.Library.Main.Database } } } - + public void AddElement(string path, string filehash, string metahash, long size, Library.Interface.ListChangesElementType type, bool asNew) { var cmd = asNew ? m_insertCurrentElementCommand : m_insertPreviousElementCommand; @@ -221,7 +221,7 @@ namespace Duplicati.Library.Main.Database private static IEnumerable ReaderToStringList(System.Data.IDataReader rd) { - using(rd) + using (rd) while (rd.Read()) { var v = rd.GetValue(0); @@ -231,13 +231,13 @@ namespace Duplicati.Library.Main.Database yield return v.ToString(); } } - + private Tuple GetSqls(bool allTypes) { return new Tuple( - FormatInvariant(@"SELECT ""Path"" FROM ""{0}"" WHERE {2} ""{0}"".""Path"" NOT IN (SELECT ""Path"" FROM ""{1}"")", m_currentTable, m_previousTable, FormatInvariant(allTypes ? "" : @" ""{0}"".""Type"" = ? AND ", m_currentTable)), - FormatInvariant(@"SELECT ""Path"" FROM ""{0}"" WHERE {2} ""{0}"".""Path"" NOT IN (SELECT ""Path"" FROM ""{1}"")", m_previousTable, m_currentTable, FormatInvariant(allTypes ? "" : @" ""{0}"".""Type"" = ? AND ", m_previousTable)), - FormatInvariant(@"SELECT ""{0}"".""Path"" FROM ""{0}"",""{1}"" WHERE {2} ""{0}"".""Path"" = ""{1}"".""Path"" AND (""{0}"".""FileHash"" != ""{1}"".""FileHash"" OR ""{0}"".""MetaHash"" != ""{1}"".""MetaHash"" OR ""{0}"".""Type"" != ""{1}"".""Type"") ", m_currentTable, m_previousTable, FormatInvariant(allTypes ? "" : @" ""{0}"".""Type"" = ? AND ", m_currentTable)) + FormatInvariant($@"SELECT ""Path"" FROM ""{m_currentTable}"" WHERE {(allTypes ? "" : FormatInvariant(@$" ""{m_currentTable}"".""Type"" = ? AND "))} ""{m_currentTable}"".""Path"" NOT IN (SELECT ""Path"" FROM ""{m_previousTable}"")"), + FormatInvariant($@"SELECT ""Path"" FROM ""{m_previousTable}"" WHERE {(allTypes ? "" : FormatInvariant(@$" ""{m_previousTable}"".""Type"" = ? AND "))} ""{m_previousTable}"".""Path"" NOT IN (SELECT ""Path"" FROM ""{m_currentTable}"")"), + FormatInvariant($@"SELECT ""{m_currentTable}"".""Path"" FROM ""{m_currentTable}"",""{m_previousTable}"" WHERE {(allTypes ? "" : FormatInvariant($@" ""{m_currentTable}"".""Type"" = ? AND "))} ""{m_currentTable}"".""Path"" = ""{m_previousTable}"".""Path"" AND (""{m_currentTable}"".""FileHash"" != ""{m_previousTable}"".""FileHash"" OR ""{m_currentTable}"".""MetaHash"" != ""{m_previousTable}"".""MetaHash"" OR ""{m_currentTable}"".""Type"" != ""{m_previousTable}"".""Type"") ") ); } @@ -247,34 +247,34 @@ namespace Duplicati.Library.Main.Database var added = sqls.Item1; var deleted = sqls.Item2; //var modified = sqls.Item3; - - using(var cmd = m_connection.CreateCommand()) + + using (var cmd = m_connection.CreateCommand()) { cmd.Transaction = m_transaction; - + var result = new ChangeSizeReport(); - - result.PreviousSize = cmd.ExecuteScalarInt64(FormatInvariant(@"SELECT SUM(""Size"") FROM ""{0}"" ", m_previousTable), 0); - result.CurrentSize = cmd.ExecuteScalarInt64(FormatInvariant(@"SELECT SUM(""Size"") FROM ""{0}"" ", m_currentTable), 0); - - result.AddedSize = cmd.ExecuteScalarInt64(FormatInvariant(@"SELECT SUM(""Size"") FROM ""{0}"" WHERE ""{0}"".""Path"" IN ({1}) ", m_currentTable, added), 0); - result.DeletedSize = cmd.ExecuteScalarInt64(FormatInvariant(@"SELECT SUM(""Size"") FROM ""{0}"" WHERE ""{0}"".""Path"" IN ({1}) ", m_previousTable, deleted), 0); - + + result.PreviousSize = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT SUM(""Size"") FROM ""{m_previousTable}"" "), 0); + result.CurrentSize = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT SUM(""Size"") FROM ""{m_currentTable}"" "), 0); + + result.AddedSize = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT SUM(""Size"") FROM ""{m_currentTable}"" WHERE ""{m_currentTable}"".""Path"" IN ({added}) "), 0); + result.DeletedSize = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT SUM(""Size"") FROM ""{m_previousTable}"" WHERE ""{m_previousTable}"".""Path"" IN ({deleted}) "), 0); + return result; } } - + public IChangeCountReport CreateChangeCountReport() { var sqls = GetSqls(false); var added = @"SELECT COUNT(*) FROM (" + sqls.Item1 + ")"; var deleted = @"SELECT COUNT(*) FROM (" + sqls.Item2 + ")"; var modified = @"SELECT COUNT(*) FROM (" + sqls.Item3 + ")"; - - using(var cmd = m_connection.CreateCommand()) + + using (var cmd = m_connection.CreateCommand()) { cmd.Transaction = m_transaction; - + var result = new ChangeCountReport(); result.AddedFolders = cmd.ExecuteScalarInt64(added, 0, (int)Library.Interface.ListChangesElementType.Folder); result.AddedSymlinks = cmd.ExecuteScalarInt64(added, 0, (int)Library.Interface.ListChangesElementType.Symlink); @@ -283,78 +283,78 @@ namespace Duplicati.Library.Main.Database result.DeletedFolders = cmd.ExecuteScalarInt64(deleted, 0, (int)Library.Interface.ListChangesElementType.Folder); result.DeletedSymlinks = cmd.ExecuteScalarInt64(deleted, 0, (int)Library.Interface.ListChangesElementType.Symlink); result.DeletedFiles = cmd.ExecuteScalarInt64(deleted, 0, (int)Library.Interface.ListChangesElementType.File); - + result.ModifiedFolders = cmd.ExecuteScalarInt64(modified, 0, (int)Library.Interface.ListChangesElementType.Folder); result.ModifiedSymlinks = cmd.ExecuteScalarInt64(modified, 0, (int)Library.Interface.ListChangesElementType.Symlink); result.ModifiedFiles = cmd.ExecuteScalarInt64(modified, 0, (int)Library.Interface.ListChangesElementType.File); - + return result; - } + } } - + public IEnumerable> CreateChangedFileReport() { var sqls = GetSqls(false); var added = sqls.Item1; var deleted = sqls.Item2; var modified = sqls.Item3; - - using(var cmd = m_connection.CreateCommand()) + + using (var cmd = m_connection.CreateCommand()) { cmd.Transaction = m_transaction; - foreach(var s in ReaderToStringList(cmd.ExecuteReader(added, (int)Library.Interface.ListChangesElementType.Folder))) - yield return new Tuple(Library.Interface.ListChangesChangeType.Added, Library.Interface.ListChangesElementType.Folder, s); - foreach(var s in ReaderToStringList(cmd.ExecuteReader(added, (int)Library.Interface.ListChangesElementType.Symlink))) + foreach (var s in ReaderToStringList(cmd.ExecuteReader(added, (int)Library.Interface.ListChangesElementType.Folder))) + yield return new Tuple(Library.Interface.ListChangesChangeType.Added, Library.Interface.ListChangesElementType.Folder, s); + foreach (var s in ReaderToStringList(cmd.ExecuteReader(added, (int)Library.Interface.ListChangesElementType.Symlink))) yield return new Tuple(Library.Interface.ListChangesChangeType.Added, Library.Interface.ListChangesElementType.Symlink, s); - foreach(var s in ReaderToStringList(cmd.ExecuteReader(added, (int)Library.Interface.ListChangesElementType.File))) + foreach (var s in ReaderToStringList(cmd.ExecuteReader(added, (int)Library.Interface.ListChangesElementType.File))) yield return new Tuple(Library.Interface.ListChangesChangeType.Added, Library.Interface.ListChangesElementType.File, s); - - foreach(var s in ReaderToStringList(cmd.ExecuteReader(deleted, (int)Library.Interface.ListChangesElementType.Folder))) + + foreach (var s in ReaderToStringList(cmd.ExecuteReader(deleted, (int)Library.Interface.ListChangesElementType.Folder))) yield return new Tuple(Library.Interface.ListChangesChangeType.Deleted, Library.Interface.ListChangesElementType.Folder, s); - foreach(var s in ReaderToStringList(cmd.ExecuteReader(deleted, (int)Library.Interface.ListChangesElementType.Symlink))) + foreach (var s in ReaderToStringList(cmd.ExecuteReader(deleted, (int)Library.Interface.ListChangesElementType.Symlink))) yield return new Tuple(Library.Interface.ListChangesChangeType.Deleted, Library.Interface.ListChangesElementType.Symlink, s); - foreach(var s in ReaderToStringList(cmd.ExecuteReader(deleted, (int)Library.Interface.ListChangesElementType.File))) + foreach (var s in ReaderToStringList(cmd.ExecuteReader(deleted, (int)Library.Interface.ListChangesElementType.File))) yield return new Tuple(Library.Interface.ListChangesChangeType.Deleted, Library.Interface.ListChangesElementType.File, s); - - foreach(var s in ReaderToStringList(cmd.ExecuteReader(modified, (int)Library.Interface.ListChangesElementType.Folder))) + + foreach (var s in ReaderToStringList(cmd.ExecuteReader(modified, (int)Library.Interface.ListChangesElementType.Folder))) yield return new Tuple(Library.Interface.ListChangesChangeType.Modified, Library.Interface.ListChangesElementType.Folder, s); - foreach(var s in ReaderToStringList(cmd.ExecuteReader(modified, (int)Library.Interface.ListChangesElementType.Symlink))) + foreach (var s in ReaderToStringList(cmd.ExecuteReader(modified, (int)Library.Interface.ListChangesElementType.Symlink))) yield return new Tuple(Library.Interface.ListChangesChangeType.Modified, Library.Interface.ListChangesElementType.Symlink, s); - foreach(var s in ReaderToStringList(cmd.ExecuteReader(modified, (int)Library.Interface.ListChangesElementType.File))) + foreach (var s in ReaderToStringList(cmd.ExecuteReader(modified, (int)Library.Interface.ListChangesElementType.File))) yield return new Tuple(Library.Interface.ListChangesChangeType.Modified, Library.Interface.ListChangesElementType.File, s); } } - + public void Dispose() { if (m_insertPreviousElementCommand != null) { try { m_insertPreviousElementCommand.Dispose(); } - catch {} + catch { } finally { m_insertPreviousElementCommand = null; } } if (m_insertCurrentElementCommand != null) { try { m_insertCurrentElementCommand.Dispose(); } - catch {} + catch { } finally { m_insertCurrentElementCommand = null; } } - + if (m_transaction != null) { try { m_transaction.Rollback(); } - catch {} + catch { } finally { m_previousTable = null; - m_currentTable= null; + m_currentTable = null; m_transaction = null; } } } } - + public IStorageHelper CreateStorageHelper() { return new StorageHelper(m_connection); diff --git a/Duplicati/Library/Main/Database/LocalListDatabase.cs b/Duplicati/Library/Main/Database/LocalListDatabase.cs index 6ab0d1abf..36b8b4958 100644 --- a/Duplicati/Library/Main/Database/LocalListDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListDatabase.cs @@ -33,13 +33,13 @@ namespace Duplicati.Library.Main.Database { ShouldCloseConnection = true; } - + public interface IFileversion { string Path { get; } IEnumerable Sizes { get; } } - + public interface IFileset { long Version { get; } @@ -48,7 +48,7 @@ namespace Duplicati.Library.Main.Database long FileCount { get; } long FileSizes { get; } } - + public interface IFileSets : IDisposable { IEnumerable Sets { get; } @@ -56,7 +56,7 @@ namespace Duplicati.Library.Main.Database IEnumerable SelectFiles(Library.Utility.IFilter filter); IEnumerable GetLargestPrefix(Library.Utility.IFilter filter); IEnumerable SelectFolderContents(Library.Utility.IFilter filter); - void TakeFirst (); + void TakeFirst(); } private class FileSets : IFileSets @@ -64,7 +64,7 @@ namespace Duplicati.Library.Main.Database private readonly System.Data.IDbConnection m_connection; private string m_tablename; private readonly KeyValuePair[] m_filesets; - + public FileSets(LocalListDatabase owner, DateTime time, long[] versions) { m_connection = owner.m_connection; @@ -73,14 +73,14 @@ namespace Duplicati.Library.Main.Database var tmp = owner.GetFilelistWhereClause(time, versions, m_filesets); string query = tmp.Item1; var args = tmp.Item2; - - using(var cmd = m_connection.CreateCommand()) + + using (var cmd = m_connection.CreateCommand()) { - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" AS SELECT DISTINCT ""ID"" AS ""FilesetID"", ""IsFullBackup"" AS ""IsFullBackup"" , ""Timestamp"" AS ""Timestamp"" FROM ""Fileset"" " + query, m_tablename), args); - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE INDEX ""{0}_FilesetIDTimestampIndex"" ON ""{0}"" (""FilesetID"", ""Timestamp"" DESC)", m_tablename)); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tablename}"" AS SELECT DISTINCT ""ID"" AS ""FilesetID"", ""IsFullBackup"" AS ""IsFullBackup"" , ""Timestamp"" AS ""Timestamp"" FROM ""Fileset"" {query}"), args); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""{m_tablename}_FilesetIDTimestampIndex"" ON ""{m_tablename}"" (""FilesetID"", ""Timestamp"" DESC)")); } } - + private class Fileset : IFileset { public long Version { get; private set; } @@ -88,7 +88,7 @@ namespace Duplicati.Library.Main.Database public DateTime Time { get; private set; } public long FileCount { get; private set; } public long FileSizes { get; private set; } - + public Fileset(long version, int isFullBackup, DateTime time, long filecount, long filesizes) { this.Version = version; @@ -98,25 +98,25 @@ namespace Duplicati.Library.Main.Database this.FileSizes = filesizes; } } - + private class Fileversion : IFileversion { private readonly System.Data.IDataReader m_reader; public string Path { get; private set; } public bool More { get; private set; } - - public Fileversion(System.Data.IDataReader reader) - { - m_reader = reader; + + public Fileversion(System.Data.IDataReader reader) + { + m_reader = reader; this.Path = reader.GetValue(0).ToString(); this.More = true; } - + public IEnumerable Sizes { get { - while(this.More && this.Path == m_reader.GetValue(0).ToString()) + while (this.More && this.Path == m_reader.GetValue(0).ToString()) { yield return m_reader.ConvertValueToInt64(1, -1); this.More = m_reader.Read(); @@ -124,7 +124,7 @@ namespace Duplicati.Library.Main.Database } } } - + private class FileversionFixed : IFileversion { public string Path { get; internal set; } @@ -138,40 +138,40 @@ namespace Duplicati.Library.Main.Database private IEnumerable GetLargestPrefix(Library.Utility.IFilter filter, string prefixrule) { - using(var tmpnames = new FilteredFilenameTable(m_connection, filter, null)) - using(var cmd = m_connection.CreateCommand()) + using (var tmpnames = new FilteredFilenameTable(m_connection, filter, null)) + using (var cmd = m_connection.CreateCommand()) { //First we trim the filelist to exclude filenames not found in any of the filesets - cmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""{0}"" WHERE ""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""File"", ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""FilesetEntry"".""FilesetID"" IN (SELECT ""FilesetID"" FROM ""{1}"") ) ", tmpnames.Tablename, m_tablename)); + cmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{tmpnames.Tablename}"" WHERE ""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""File"", ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""FilesetEntry"".""FilesetID"" IN (SELECT ""FilesetID"" FROM ""{m_tablename}"") ) ")); //If we have a prefix rule, apply it if (!string.IsNullOrWhiteSpace(prefixrule)) - cmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""{0}"" WHERE SUBSTR(""Path"", 1, {1}) != ?", tmpnames.Tablename, prefixrule.Length), prefixrule); + cmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{tmpnames.Tablename}"" WHERE SUBSTR(""Path"", 1, {prefixrule.Length}) != ?"), prefixrule); // Then we recursively find the largest prefix - cmd.CommandText = FormatInvariant(@"SELECT ""Path"" FROM ""{0}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1", tmpnames.Tablename); + cmd.CommandText = FormatInvariant($@"SELECT ""Path"" FROM ""{tmpnames.Tablename}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1"); var v0 = cmd.ExecuteScalar(); string maxpath = ""; if (v0 != null) maxpath = v0.ToString(); var dirsep = Util.GuessDirSeparator(maxpath); - - cmd.CommandText = FormatInvariant(@"SELECT COUNT(*) FROM ""{0}""", tmpnames.Tablename); + + cmd.CommandText = FormatInvariant($@"SELECT COUNT(*) FROM ""{tmpnames.Tablename}"""); var filecount = cmd.ExecuteScalarInt64(0); long foundfiles = -1; - + //TODO: Handle FS case-sensitive? - cmd.CommandText = FormatInvariant(@"SELECT COUNT(*) FROM ""{0}"" WHERE SUBSTR(""Path"", 1, ?) = ?", tmpnames.Tablename); + cmd.CommandText = FormatInvariant($@"SELECT COUNT(*) FROM ""{tmpnames.Tablename}"" WHERE SUBSTR(""Path"", 1, ?) = ?"); cmd.AddParameter(); cmd.AddParameter(); - + while (filecount != foundfiles && maxpath.Length > 0) { var mp = Util.AppendDirSeparator(maxpath, dirsep); cmd.SetParameterValue(0, mp.Length); cmd.SetParameterValue(1, mp); - + foundfiles = cmd.ExecuteScalarInt64(0); if (filecount != foundfiles) @@ -188,7 +188,7 @@ namespace Duplicati.Library.Main.Database // Special handling for Windows and multi-drive/UNC backups as they do not have a single common root if (string.IsNullOrWhiteSpace(maxpath) && string.IsNullOrWhiteSpace(prefixrule)) { - var paths = cmd.ExecuteReaderEnumerable(FormatInvariant(@"SELECT Path FROM ""{0}""", tmpnames.Tablename)).Select(x => x.ConvertValueToString(0)).ToArray(); + var paths = cmd.ExecuteReaderEnumerable(FormatInvariant($@"SELECT Path FROM ""{tmpnames.Tablename}""")).Select(x => x.ConvertValueToString(0)).ToArray(); var roots = paths.Select(x => x.Substring(0, 1)).Distinct().Where(x => x != "\\").ToArray(); //unc path like \\server.domain\ @@ -204,14 +204,14 @@ namespace Duplicati.Library.Main.Database }; } } - + private IEnumerable SelectFolderEntries(System.Data.IDbCommand cmd, string prefix, string table) { if (!string.IsNullOrEmpty(prefix)) prefix = Util.AppendDirSeparator(prefix, Util.GuessDirSeparator(prefix)); - + var ppl = prefix.Length; - using(var rd = cmd.ExecuteReader(FormatInvariant(@"SELECT DISTINCT ""Path"" FROM ""{0}"" ", table))) + using (var rd = cmd.ExecuteReader(FormatInvariant($@"SELECT DISTINCT ""Path"" FROM ""{table}"" "))) while (rd.Read()) { var s = rd.GetString(0); @@ -227,7 +227,7 @@ namespace Duplicati.Library.Main.Database yield return prefix + s; } } - + public IEnumerable SelectFolderContents(Library.Utility.IFilter filter) { var tbname = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); @@ -245,13 +245,13 @@ namespace Duplicati.Library.Main.Database if (pathprefix.Length > 0 || dirsep == "/") pathprefix = Util.AppendDirSeparator(pathprefix, dirsep); - - using(var tmpnames = new FilteredFilenameTable(m_connection, new Library.Utility.FilterExpression(new string[] { pathprefix + "*" }, true), null)) - using(var cmd = m_connection.CreateCommand()) + + using (var tmpnames = new FilteredFilenameTable(m_connection, new Library.Utility.FilterExpression(new string[] { pathprefix + "*" }, true), null)) + using (var cmd = m_connection.CreateCommand()) { //First we trim the filelist to exclude filenames not found in any of the filesets - cmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""{0}"" WHERE ""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""File"", ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""FilesetEntry"".""FilesetID"" IN (SELECT ""FilesetID"" FROM ""{1}"") ) ", tmpnames.Tablename, m_tablename)); - + cmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{tmpnames.Tablename}"" WHERE ""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""File"", ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""FilesetEntry"".""FilesetID"" IN (SELECT ""FilesetID"" FROM ""{m_tablename}"") ) ")); + // If we had instr support this would work: /*var distinctPaths = @"SELECT DISTINCT :1 || " + @"CASE(INSTR(SUBSTR(""Path"", :2), '/')) " + @@ -261,33 +261,33 @@ namespace Duplicati.Library.Main.Database @" FROM (" + cartesianPathFileset + @")";*/ // Instead we manually iterate the paths - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL)", tbname)); - - using(var c2 = m_connection.CreateCommand()) + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tbname}"" (""Path"" TEXT NOT NULL)")); + + using (var c2 = m_connection.CreateCommand()) { - c2.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" (""Path"") VALUES (?)", tbname); + c2.CommandText = FormatInvariant($@"INSERT INTO ""{tbname}"" (""Path"") VALUES (?)"); c2.AddParameter(); - - foreach(var n in SelectFolderEntries(cmd, pathprefix, tmpnames.Tablename).Distinct()) + + foreach (var n in SelectFolderEntries(cmd, pathprefix, tmpnames.Tablename).Distinct()) { c2.SetParameterValue(0, n); c2.ExecuteNonQuery(); } - c2.ExecuteNonQuery(FormatInvariant(@"CREATE INDEX ""{0}_PathIndex"" ON ""{0}"" (""Path"")", tbname)); + c2.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""{tbname}_PathIndex"" ON ""{tbname}"" (""Path"")")); } - - //Then we select the matching results - var filesets = FormatInvariant(@"SELECT ""FilesetID"", ""Timestamp"" FROM ""{0}"" ORDER BY ""Timestamp"" DESC", m_tablename); - var cartesianPathFileset = FormatInvariant(@"SELECT ""A"".""Path"", ""B"".""FilesetID"" FROM ""{0}"" A, (" + filesets + @") B ORDER BY ""A"".""Path"" ASC, ""B"".""Timestamp"" DESC", tbname, m_tablename); - var filesWithSizes = FormatInvariant(@"SELECT ""Length"", ""FilesetEntry"".""FilesetID"", ""File"".""Path"" FROM ""Blockset"", ""FilesetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND FilesetEntry.""FilesetID"" IN (SELECT DISTINCT ""FilesetID"" FROM ""{0}"") ", m_tablename); + //Then we select the matching results + var filesets = FormatInvariant($@"SELECT ""FilesetID"", ""Timestamp"" FROM ""{m_tablename}"" ORDER BY ""Timestamp"" DESC"); + var cartesianPathFileset = FormatInvariant($@"SELECT ""A"".""Path"", ""B"".""FilesetID"" FROM ""{tbname}"" A, ({filesets}) B ORDER BY ""A"".""Path"" ASC, ""B"".""Timestamp"" DESC"); + + var filesWithSizes = FormatInvariant($@"SELECT ""Length"", ""FilesetEntry"".""FilesetID"", ""File"".""Path"" FROM ""Blockset"", ""FilesetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND FilesetEntry.""FilesetID"" IN (SELECT DISTINCT ""FilesetID"" FROM ""{m_tablename}"") "); var query = @"SELECT ""C"".""Path"", ""D"".""Length"", ""C"".""FilesetID"" FROM (" + cartesianPathFileset + @") C LEFT OUTER JOIN (" + filesWithSizes + @") D ON ""C"".""FilesetID"" = ""D"".""FilesetID"" AND ""C"".""Path"" = ""D"".""Path"""; - - cmd.AddParameter(pathprefix, "1"); + + cmd.AddParameter(pathprefix, "1"); cmd.AddParameter(pathprefix.Length + 1, "2"); - - using(var rd = cmd.ExecuteReader(query)) + + using (var rd = cmd.ExecuteReader(query)) if (rd.Read()) { bool more; @@ -303,39 +303,39 @@ namespace Duplicati.Library.Main.Database { more = rd.Read(); } - - } while(more); + + } while (more); } - } + } } finally { try { - using(var c = m_connection.CreateCommand()) - c.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}""", tbname)); + using (var c = m_connection.CreateCommand()) + c.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{tbname}""")); } catch { } } } - + public IEnumerable SelectFiles(Library.Utility.IFilter filter) { - using(var tmpnames = new FilteredFilenameTable(m_connection, filter, null)) - using(var cmd = m_connection.CreateCommand()) + using (var tmpnames = new FilteredFilenameTable(m_connection, filter, null)) + using (var cmd = m_connection.CreateCommand()) { //First we trim the filelist to exclude filenames not found in any of the filesets - cmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""{0}"" WHERE ""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""File"", ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""FilesetEntry"".""FilesetID"" IN (SELECT ""FilesetID"" FROM ""{1}"") ) ", tmpnames.Tablename, m_tablename)); - + cmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{tmpnames.Tablename}"" WHERE ""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""File"", ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""FilesetEntry"".""FilesetID"" IN (SELECT ""FilesetID"" FROM ""{m_tablename}"") ) ")); + //Then we select the matching results - var filesets = FormatInvariant(@"SELECT ""FilesetID"", ""Timestamp"" FROM ""{0}"" ORDER BY ""Timestamp"" DESC", m_tablename); - var cartesianPathFileset = FormatInvariant(@"SELECT ""A"".""Path"", ""B"".""FilesetID"" FROM ""{0}"" A, (" + filesets + @") B ORDER BY ""A"".""Path"" ASC, ""B"".""Timestamp"" DESC", tmpnames.Tablename, m_tablename); - var filesWithSizes = FormatInvariant(@"SELECT ""Length"", ""FilesetEntry"".""FilesetID"", ""File"".""Path"" FROM ""Blockset"", ""FilesetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND FilesetEntry.""FilesetID"" IN (SELECT DISTINCT ""FilesetID"" FROM ""{0}"") ", m_tablename); + var filesets = FormatInvariant($@"SELECT ""FilesetID"", ""Timestamp"" FROM ""{m_tablename}"" ORDER BY ""Timestamp"" DESC"); + var cartesianPathFileset = FormatInvariant($@"SELECT ""A"".""Path"", ""B"".""FilesetID"" FROM ""{tmpnames.Tablename}"" A, ({filesets}) B ORDER BY ""A"".""Path"" ASC, ""B"".""Timestamp"" DESC"); + var filesWithSizes = FormatInvariant($@"SELECT ""Length"", ""FilesetEntry"".""FilesetID"", ""File"".""Path"" FROM ""Blockset"", ""FilesetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND FilesetEntry.""FilesetID"" IN (SELECT DISTINCT ""FilesetID"" FROM ""{m_tablename}"") "); var query = @"SELECT ""C"".""Path"", ""D"".""Length"", ""C"".""FilesetID"" FROM (" + cartesianPathFileset + @") C LEFT OUTER JOIN (" + filesWithSizes + @") D ON ""C"".""FilesetID"" = ""D"".""FilesetID"" AND ""C"".""Path"" = ""D"".""Path"""; - using(var rd = cmd.ExecuteReader(query)) - if(rd.Read()) + using (var rd = cmd.ExecuteReader(query)) + if (rd.Read()) { bool more; do @@ -343,15 +343,15 @@ namespace Duplicati.Library.Main.Database var f = new Fileversion(rd); yield return f; more = f.More; - } while(more); + } while (more); } } } - + public void TakeFirst() { - using(var cmd = m_connection.CreateCommand()) - cmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""{0}"" WHERE ""FilesetID"" NOT IN (SELECT ""FilesetID"" FROM ""{0}"" ORDER BY ""Timestamp"" DESC LIMIT 1 )", m_tablename)); + using (var cmd = m_connection.CreateCommand()) + cmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{m_tablename}"" WHERE ""FilesetID"" NOT IN (SELECT ""FilesetID"" FROM ""{m_tablename}"" ORDER BY ""Timestamp"" DESC LIMIT 1 )")); } public IEnumerable QuickSets @@ -380,8 +380,8 @@ namespace Duplicati.Library.Main.Database } } - public IEnumerable Sets - { + public IEnumerable Sets + { get { var dict = new Dictionary(); @@ -389,7 +389,7 @@ namespace Duplicati.Library.Main.Database { dict[m_filesets[i].Key] = i; } - + var summation = FormatInvariant( $@"SELECT ""A"".""FilesetID"" AS ""FilesetID"", COUNT(*) AS ""FileCount"", SUM(""C"".""Length"") AS ""FileSizes"" FROM ""FilesetEntry"" A, ""File"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" AND ""A"".""FilesetID"" IN (SELECT DISTINCT ""FilesetID"" FROM ""{m_tablename}"") GROUP BY ""A"".""FilesetID"" "); @@ -413,30 +413,30 @@ namespace Duplicati.Library.Main.Database } } } - + public void Dispose() { if (m_tablename != null) { - try + try { using (var cmd = m_connection.CreateCommand()) { - cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", m_tablename)); + cmd.ExecuteNonQuery(FormatInvariant(@$"DROP TABLE IF EXISTS ""{m_tablename}"" ")); } } - catch {} + catch { } finally { m_tablename = null; } } - + } } - + public IFileSets SelectFileSets(DateTime time, long[] versions) { return new FileSets(this, time, versions); } - + } } diff --git a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs index becb8515d..0528a69e3 100644 --- a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs @@ -95,14 +95,14 @@ namespace Duplicati.Library.Main.Database m_tablename = "TempDeletedFilesTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); using (var cmd = m_connection.CreateCommand(m_transaction)) - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (""FileID"" INTEGER PRIMARY KEY) ", m_tablename)); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tablename}"" (""FileID"" INTEGER PRIMARY KEY) ")); } public void ApplyFilter(Action filtercommand) { using (var cmd = m_connection.CreateCommand(m_transaction)) filtercommand(cmd, ParentID, m_tablename); - + PostFilterChecks(); } @@ -117,8 +117,8 @@ namespace Duplicati.Library.Main.Database var filenamestable = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); using (var cmd = m_connection.CreateCommand(m_transaction)) { - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL) ", filenamestable)); - cmd.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" (""Path"") VALUES (?)", filenamestable); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{filenamestable}"" (""Path"" TEXT NOT NULL) ")); + cmd.CommandText = FormatInvariant($@"INSERT INTO ""{filenamestable}"" (""Path"") VALUES (?)"); cmd.AddParameter(); foreach (var s in p) @@ -127,8 +127,8 @@ namespace Duplicati.Library.Main.Database cmd.ExecuteNonQuery(); } - cmd.ExecuteNonQuery(FormatInvariant(@"INSERT INTO ""{0}"" (""FileID"") SELECT DISTINCT ""A"".""FileID"" FROM ""FilesetEntry"" A, ""File"" B WHERE ""A"".""FilesetID"" = ? AND ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""Path"" IN ""{1}""", m_tablename, filenamestable), ParentID); - cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", filenamestable)); + cmd.ExecuteNonQuery(FormatInvariant($@"INSERT INTO ""{m_tablename}"" (""FileID"") SELECT DISTINCT ""A"".""FileID"" FROM ""FilesetEntry"" A, ""File"" B WHERE ""A"".""FilesetID"" = ? AND ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""Path"" IN ""{filenamestable}"""), ParentID); + cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{filenamestable}"" ")); } } else @@ -138,7 +138,7 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand(m_transaction)) using (var cmd2 = m_connection.CreateCommand(m_transaction)) { - cmd2.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" (""FileID"") VALUES (?)", m_tablename); + cmd2.CommandText = FormatInvariant($@"INSERT INTO ""{m_tablename}"" (""FileID"") VALUES (?)"); cmd2.AddParameters(1); using (var rd = cmd.ExecuteReader(@"SELECT ""B"".""Path"", ""A"".""FileID"" FROM ""FilesetEntry"" A, ""File"" B WHERE ""A"".""FilesetID"" = ? AND ""A"".""FileID"" = ""B"".""ID"" ", ParentID)) @@ -161,9 +161,9 @@ namespace Duplicati.Library.Main.Database { using (var cmd = m_connection.CreateCommand(m_transaction)) { - RemovedFileCount = cmd.ExecuteScalarInt64(FormatInvariant(@"SELECT COUNT(*) FROM ""{0}""", m_tablename), 0); - RemovedFileSize = cmd.ExecuteScalarInt64(FormatInvariant(@"SELECT SUM(""C"".""Length"") FROM ""{0}"" A, ""FileLookup"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" ", m_tablename), 0); - var filesetcount = cmd.ExecuteScalarInt64(FormatInvariant(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetID"" = " + ParentID), 0); + RemovedFileCount = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM ""{m_tablename}"""), 0); + RemovedFileSize = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT SUM(""C"".""Length"") FROM ""{m_tablename}"" A, ""FileLookup"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" "), 0); + var filesetcount = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetID"" = {ParentID}"), 0); if (filesetcount == RemovedFileCount) throw new Duplicati.Library.Interface.UserInformationException($"Refusing to purge {RemovedFileCount} files from fileset with ID {ParentID}, as that would remove the entire fileset.\nTo delete a fileset, use the \"delete\" command.", "PurgeWouldRemoveEntireFileset"); } @@ -176,15 +176,15 @@ namespace Duplicati.Library.Main.Database m_parentdb.UpdateFullBackupStateInFileset(filesetid, isFullBackup); using (var cmd = m_connection.CreateCommand(m_transaction)) - cmd.ExecuteNonQuery(FormatInvariant(@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") SELECT ?, ""FileID"", ""LastModified"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = ? AND ""FileID"" NOT IN ""{0}"" ", m_tablename), filesetid, ParentID); + cmd.ExecuteNonQuery(FormatInvariant($@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") SELECT ?, ""FileID"", ""LastModified"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = ? AND ""FileID"" NOT IN ""{m_tablename}"" "), filesetid, ParentID); return new Tuple(remotevolid, filesetid); } - + public IEnumerable> ListAllDeletedFiles() { using (var cmd = m_connection.CreateCommand(m_transaction)) - using (var rd = cmd.ExecuteReader(FormatInvariant(@"SELECT ""B"".""Path"", ""C"".""Length"" FROM ""{0}"" A, ""File"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" ", m_tablename))) + using (var rd = cmd.ExecuteReader(FormatInvariant($@"SELECT ""B"".""Path"", ""C"".""Length"" FROM ""{m_tablename}"" A, ""File"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" "))) while (rd.Read()) yield return new KeyValuePair(rd.ConvertValueToString(0), rd.ConvertValueToInt64(1)); } @@ -194,7 +194,7 @@ namespace Duplicati.Library.Main.Database try { using (var cmd = m_connection.CreateCommand(m_transaction)) - cmd.ExecuteNonQuery(@"DROP TABLE IF EXISTS ""{0}""", m_tablename); + cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tablename}""")); } catch { @@ -202,5 +202,5 @@ namespace Duplicati.Library.Main.Database } } - } + } } diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index a7e91308a..ffbc4438e 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -75,13 +75,13 @@ namespace Duplicati.Library.Main.Database // {1} --> BlockHash-Size // {2} --> Temp-Table // {3} --> FullBlocklist-BlockCount [equals ({0} / {1}), if SQLite pays respect to ints] - private const string SELECT_BLOCKLIST_ENTRIES = - @" + private static string SELECT_BLOCKLIST_ENTRIES(long blocksize, long blockhashsize, string temptable, long fullBlockListBlockCount) => + FormatInvariant($@" SELECT ""E"".""BlocksetID"", - ""F"".""Index"" + (""E"".""BlocklistIndex"" * {3}) AS ""FullIndex"", + ""F"".""Index"" + (""E"".""BlocklistIndex"" * {fullBlockListBlockCount}) AS ""FullIndex"", ""F"".""BlockHash"", - MIN({0}, ""E"".""Length"" - ((""F"".""Index"" + (""E"".""BlocklistIndex"" * {3})) * {0})) AS ""BlockSize"", + MIN({blocksize}, ""E"".""Length"" - ((""F"".""Index"" + (""E"".""BlocklistIndex"" * {fullBlockListBlockCount})) * {blocksize})) AS ""BlockSize"", ""E"".""Hash"", ""E"".""BlocklistSize"", ""E"".""BlocklistHash"" @@ -92,7 +92,7 @@ namespace Duplicati.Library.Main.Database SELECT ""A"".""BlocksetID"", ""A"".""Index"" AS ""BlocklistIndex"", - MIN({3} * {1}, (((""B"".""Length"" + {0} - 1) / {0}) - (""A"".""Index"" * ({3}))) * {1}) AS ""BlocklistSize"", + MIN({fullBlockListBlockCount} * {blockhashsize}, (((""B"".""Length"" + {blocksize} - 1) / {blocksize}) - (""A"".""Index"" * ({fullBlockListBlockCount}))) * {blockhashsize}) AS ""BlocklistSize"", ""A"".""Hash"" AS ""BlocklistHash"", ""B"".""Length"" FROM @@ -107,13 +107,13 @@ namespace Duplicati.Library.Main.Database AND ""C"".""BlocklistSize"" = ""D"".""Size"" ) E, - ""{2}"" F + ""{temptable}"" F WHERE ""F"".""BlocklistHash"" = ""E"".""Hash"" ORDER BY ""E"".""BlocksetID"", ""FullIndex"" -"; +"); public LocalRecreateDatabase(LocalDatabase parentdb, Options options) : base(parentdb) @@ -123,12 +123,12 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) { - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (""BlockListHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""Index"" INTEGER NOT NULL)", m_tempblocklist)); - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE INDEX ""Index_{0}"" ON ""{0}"" (""BlockListHash"");", m_tempblocklist)); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tempblocklist}"" (""BlockListHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""Index"" INTEGER NOT NULL)")); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""Index_{m_tempblocklist}"" ON ""{m_tempblocklist}"" (""BlockListHash"");")); - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (""FileHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""BlockSize"" INTEGER NOT NULL)", m_tempsmalllist)); - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE UNIQUE INDEX ""Index_File_{0}"" ON ""{0}"" (""FileHash"", ""BlockSize"");", m_tempsmalllist)); - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE UNIQUE INDEX ""Index_Block_{0}"" ON ""{0}"" (""BlockHash"", ""BlockSize"");", m_tempsmalllist)); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tempsmalllist}"" (""FileHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""BlockSize"" INTEGER NOT NULL)")); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE UNIQUE INDEX ""Index_File_{m_tempsmalllist}"" ON ""{m_tempsmalllist}"" (""FileHash"", ""BlockSize"");")); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE UNIQUE INDEX ""Index_Block_{m_tempsmalllist}"" ON ""{m_tempsmalllist}"" (""BlockHash"", ""BlockSize"");")); } m_insertFileCommand = m_connection.CreateCommand(); @@ -165,10 +165,10 @@ namespace Duplicati.Library.Main.Database m_updateBlockVolumeCommand.CommandText = @"UPDATE ""Block"" SET ""VolumeID"" = ? WHERE ""Hash"" = ? AND ""Size"" = ?"; m_updateBlockVolumeCommand.AddParameters(3); - m_insertTempBlockListHash.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" (""BlocklistHash"", ""BlockHash"", ""Index"") VALUES (?,?,?) ", m_tempblocklist); + m_insertTempBlockListHash.CommandText = FormatInvariant($@"INSERT INTO ""{m_tempblocklist}"" (""BlocklistHash"", ""BlockHash"", ""Index"") VALUES (?,?,?) "); m_insertTempBlockListHash.AddParameters(3); - m_insertSmallBlockset.CommandText = FormatInvariant(@"INSERT OR IGNORE INTO ""{0}"" (""FileHash"", ""BlockHash"", ""BlockSize"") VALUES (?,?,?) ", m_tempsmalllist); + m_insertSmallBlockset.CommandText = FormatInvariant($@"INSERT OR IGNORE INTO ""{m_tempsmalllist}"" (""FileHash"", ""BlockHash"", ""BlockSize"") VALUES (?,?,?) "); m_insertSmallBlockset.AddParameters(3); m_findBlocksetCommand.CommandText = @"SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" = ? AND ""FullHash"" = ? "; @@ -180,7 +180,7 @@ namespace Duplicati.Library.Main.Database m_findFilesetCommand.CommandText = @"SELECT ""ID"" FROM ""FileLookup"" WHERE ""PrefixID"" = ? AND ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ? "; m_findFilesetCommand.AddParameters(4); - m_findTempBlockListHashCommand.CommandText = FormatInvariant(@"SELECT DISTINCT ""BlockListHash"" FROM ""{0}"" WHERE ""BlockListHash"" = ? ", m_tempblocklist); + m_findTempBlockListHashCommand.CommandText = FormatInvariant($@"SELECT DISTINCT ""BlockListHash"" FROM ""{m_tempblocklist}"" WHERE ""BlockListHash"" = ? "); m_findTempBlockListHashCommand.AddParameters(1); m_findHashBlockCommand.CommandText = @"SELECT ""VolumeID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ? "; @@ -201,27 +201,16 @@ namespace Duplicati.Library.Main.Database //Update all small blocklists and matching blocks - var selectSmallBlocks = FormatInvariant(@"SELECT ""BlockHash"", ""BlockSize"" FROM ""{0}""", m_tempsmalllist); + var selectSmallBlocks = FormatInvariant($@"SELECT ""BlockHash"", ""BlockSize"" FROM ""{m_tempsmalllist}"""); - var selectBlockHashes = FormatInvariant( - @"SELECT ""BlockHash"" AS ""FullHash"", ""BlockSize"" AS ""Length"" FROM ( " + - SELECT_BLOCKLIST_ENTRIES + - @" )", - blocksize, - hashsize, - m_tempblocklist, - blocksize / hashsize - ); + var selectBlockHashes = FormatInvariant($@"SELECT ""BlockHash"" AS ""FullHash"", ""BlockSize"" AS ""Length"" FROM ( {SELECT_BLOCKLIST_ENTRIES(blocksize, hashsize, m_tempblocklist, blocksize / hashsize)} )"); var selectAllBlocks = @"SELECT DISTINCT ""FullHash"", ""Length"" FROM (" + selectBlockHashes + " UNION " + selectSmallBlocks + " )"; - var selectNewBlocks = FormatInvariant( - @"SELECT ""FullHash"" AS ""Hash"", ""Length"" AS ""Size"", -1 AS ""VolumeID"" " + - @" FROM (SELECT ""A"".""FullHash"", ""A"".""Length"", CASE WHEN ""B"".""Hash"" IS NULL THEN '' ELSE ""B"".""Hash"" END AS ""Hash"", CASE WHEN ""B"".""Size"" is NULL THEN -1 ELSE ""B"".""Size"" END AS ""Size"" FROM ({0}) A" + - @" LEFT OUTER JOIN ""Block"" B ON ""B"".""Hash"" = ""A"".""FullHash"" AND ""B"".""Size"" = ""A"".""Length"" )" + - @" WHERE ""FullHash"" != ""Hash"" AND ""Length"" != ""Size"" ", - selectAllBlocks - ); + var selectNewBlocks = FormatInvariant($@"SELECT ""FullHash"" AS ""Hash"", ""Length"" AS ""Size"", -1 AS ""VolumeID"" +FROM (SELECT ""A"".""FullHash"", ""A"".""Length"", CASE WHEN ""B"".""Hash"" IS NULL THEN '' ELSE ""B"".""Hash"" END AS ""Hash"", CASE WHEN ""B"".""Size"" is NULL THEN -1 ELSE ""B"".""Size"" END AS ""Size"" FROM ({selectAllBlocks}) A +LEFT OUTER JOIN ""Block"" B ON ""B"".""Hash"" = ""A"".""FullHash"" AND ""B"".""Size"" = ""A"".""Length"" ) +WHERE ""FullHash"" != ""Hash"" AND ""Length"" != ""Size"" "); var insertBlocksCommand = @"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") " + @@ -231,25 +220,13 @@ namespace Duplicati.Library.Main.Database cmd.ExecuteNonQuery(insertBlocksCommand); var selectBlocklistBlocksetEntries = FormatInvariant( - @"SELECT ""E"".""BlocksetID"" AS ""BlocksetID"", ""D"".""FullIndex"" AS ""Index"", ""F"".""ID"" AS ""BlockID"" FROM ( " + - SELECT_BLOCKLIST_ENTRIES + - @") D, ""BlocklistHash"" E, ""Block"" F, ""Block"" G WHERE ""D"".""BlocksetID"" = ""E"".""BlocksetID"" AND ""D"".""BlocklistHash"" = ""E"".""Hash"" AND ""D"".""BlocklistSize"" = ""G"".""Size"" AND ""D"".""BlocklistHash"" = ""G"".""Hash"" AND ""D"".""Blockhash"" = ""F"".""Hash"" AND ""D"".""BlockSize"" = ""F"".""Size"" ", - // TODO: The BlocklistHash join seems to be unnecessary, but the join might be required to work around a from really old versions of Duplicati // this could be used instead //@") D, ""Block"" WHERE ""BlockQuery"".""BlockHash"" = ""Block"".""Hash"" AND ""BlockQuery"".""BlockSize"" = ""Block"".""Size"" "; + $@"SELECT ""E"".""BlocksetID"" AS ""BlocksetID"", ""D"".""FullIndex"" AS ""Index"", ""F"".""ID"" AS ""BlockID"" FROM ( {SELECT_BLOCKLIST_ENTRIES(blocksize, hashsize, m_tempblocklist, blocksize / hashsize)}) D, ""BlocklistHash"" E, ""Block"" F, ""Block"" G WHERE ""D"".""BlocksetID"" = ""E"".""BlocksetID"" AND ""D"".""BlocklistHash"" = ""E"".""Hash"" AND ""D"".""BlocklistSize"" = ""G"".""Size"" AND ""D"".""BlocklistHash"" = ""G"".""Hash"" AND ""D"".""Blockhash"" = ""F"".""Hash"" AND ""D"".""BlockSize"" = ""F"".""Size"" " + ); - blocksize, - hashsize, - m_tempblocklist, - blocksize / hashsize - ); - - var selectBlocksetEntries = FormatInvariant( - @"SELECT ""Blockset"".""ID"" AS ""BlocksetID"", 0 AS ""Index"", ""Block"".""ID"" AS ""BlockID"" FROM ""Blockset"", ""Block"", ""{1}"" S WHERE ""Blockset"".""Fullhash"" = ""S"".""FileHash"" AND ""S"".""BlockHash"" = ""Block"".""Hash"" AND ""S"".""BlockSize"" = ""Block"".""Size"" AND ""Blockset"".""Length"" = ""S"".""BlockSize"" AND ""Blockset"".""Length"" <= {0} ", - blocksize, - m_tempsmalllist - ); + var selectBlocksetEntries = FormatInvariant($@"SELECT ""Blockset"".""ID"" AS ""BlocksetID"", 0 AS ""Index"", ""Block"".""ID"" AS ""BlockID"" FROM ""Blockset"", ""Block"", ""{m_tempsmalllist}"" S WHERE ""Blockset"".""Fullhash"" = ""S"".""FileHash"" AND ""S"".""BlockHash"" = ""Block"".""Hash"" AND ""S"".""BlockSize"" = ""Block"".""Size"" AND ""Blockset"".""Length"" = ""S"".""BlockSize"" AND ""Blockset"".""Length"" <= {blocksize} "); var selectAllBlocksetEntries = selectBlocklistBlocksetEntries + @@ -274,8 +251,8 @@ namespace Duplicati.Library.Main.Database using (var fixcmd = m_connection.CreateCommand()) { - fixcmd.ExecuteNonQuery(FormatInvariant(@"CREATE TABLE ""{0}-Failure"" AS SELECT * FROM ""{0}"" ", m_tempblocklist)); - fixcmd.ExecuteNonQuery(FormatInvariant(@"CREATE TABLE ""{0}-Failure"" AS SELECT * FROM ""{0}"" ", m_tempsmalllist)); + fixcmd.ExecuteNonQuery(FormatInvariant($@"CREATE TABLE ""{m_tempblocklist}-Failure"" AS SELECT * FROM ""{m_tempblocklist}"" ")); + fixcmd.ExecuteNonQuery(FormatInvariant($@"CREATE TABLE ""{m_tempsmalllist}-Failure"" AS SELECT * FROM ""{m_tempsmalllist}"" ")); } throw new Exception("The recreate failed, please create a bug-report from this database and send it to the developers for further analysis"); @@ -306,68 +283,50 @@ namespace Duplicati.Library.Main.Database { cmd.Transaction = transaction; - var insertBlocksCommand = FormatInvariant( - @"INSERT INTO BLOCK (Hash, Size, VolumeID) " + - @"SELECT DISTINCT BlockHash AS Hash, BlockSize AS Size, -1 AS VolumeID FROM " + - @"(" + - @"SELECT NB.BlockHash, " + - @"MIN({0}, BS.Length - ((NB.""Index"" + (BH.""Index"" * {1})) * {0})) AS BlockSize " + - @"FROM (" + - @" SELECT TBL.BlockListHash, TBL.BlockHash, TBL.""Index"" FROM {2} TBL " + - @" LEFT OUTER JOIN Block B ON (B.Hash = TBL.BlockHash) " + - @" WHERE B.Hash IS NULL " + - @" ) NB " + - @"JOIN BlocklistHash BH ON (BH.Hash = NB.BlocklistHash) " + - @"JOIN Blockset BS ON (BS.ID = BH.Blocksetid) " + - @"", - blocksize, - blocksize / hashsize, - m_tempblocklist - ); + var insertBlocksCommand = FormatInvariant($@"INSERT INTO BLOCK (Hash, Size, VolumeID) +SELECT DISTINCT BlockHash AS Hash, BlockSize AS Size, -1 AS VolumeID FROM +( +SELECT NB.BlockHash, +MIN({blocksize}, BS.Length - ((NB.""Index"" + (BH.""Index"" * {blocksize / hashsize})) * {blocksize})) AS BlockSize +FROM ( + SELECT TBL.BlockListHash, TBL.BlockHash, TBL.""Index"" FROM {m_tempblocklist} TBL + LEFT OUTER JOIN Block B ON (B.Hash = TBL.BlockHash) + WHERE B.Hash IS NULL + ) NB +JOIN BlocklistHash BH ON (BH.Hash = NB.BlocklistHash) +JOIN Blockset BS ON (BS.ID = BH.Blocksetid) "); if (!hashOnly) { - insertBlocksCommand += FormatInvariant( - @" UNION " + - @"" + - @"SELECT TS.BlockHash, TS.BlockSize FROM " + - @"{0} TS " + - @"WHERE NOT EXISTS (SELECT ""X"" FROM Block AS B WHERE " + - @" B.Hash = TS.BlockHash AND " + - @" B.Size = TS.BlockSize) " + - @")", - m_tempsmalllist - ); + insertBlocksCommand += FormatInvariant($@" UNION +SELECT TS.BlockHash, TS.BlockSize FROM +{m_tempsmalllist} TS +WHERE NOT EXISTS (SELECT ""X"" FROM Block AS B WHERE + B.Hash = TS.BlockHash AND + B.Size = TS.BlockSize) +)"); } - var insertBlocksetEntriesCommand = FormatInvariant( - @"INSERT INTO BlocksetEntry (BlocksetID, ""Index"", BlockID) " + - @"SELECT DISTINCT BH.blocksetid, (BH.""Index"" * {0})+TBL.""Index"" as FullIndex, BK.ID AS BlockID " + - @"FROM {1} TBL " + - @" JOIN blocklisthash BH ON (BH.hash = TBL.blocklisthash) " + - @" JOIN block BK ON (BK.Hash = TBL.BlockHash) " + - @" LEFT OUTER JOIN BlocksetEntry BE ON (BE.BlockSetID = BH.BlocksetID AND BE.""Index"" = (BH.""Index"" * {0})+TBL.""Index"") " + - @"WHERE " + - @"BE.BlockSetID IS NULL ", - blocksize / hashsize, - m_tempblocklist - ); + var insertBlocksetEntriesCommand = FormatInvariant($@"INSERT INTO BlocksetEntry (BlocksetID, ""Index"", BlockID) +SELECT DISTINCT BH.blocksetid, (BH.""Index"" * {blocksize / hashsize})+TBL.""Index"" as FullIndex, BK.ID AS BlockID +FROM {m_tempblocklist} TBL + JOIN blocklisthash BH ON (BH.hash = TBL.blocklisthash) + JOIN block BK ON (BK.Hash = TBL.BlockHash) + LEFT OUTER JOIN BlocksetEntry BE ON (BE.BlockSetID = BH.BlocksetID AND BE.""Index"" = (BH.""Index"" * {blocksize / hashsize})+TBL.""Index"") +WHERE +BE.BlockSetID IS NULL "); if (!hashOnly) { - insertBlocksetEntriesCommand += FormatInvariant( - @" UNION " + - @"SELECT BS.ID AS BlocksetID, 0 AS ""Index"", BL.ID AS BlockID " + - @"FROM {1} TS " + - @" JOIN Blockset BS ON (BS.FullHash = TS.FileHash AND " + - @" BS.Length = TS.BlockSize AND " + - @" BS.Length <= {0}) " + - @" JOIN Block BL ON (BL.Hash = TS.BlockHash AND " + - @" BL.Size = TS.BlockSize) " + - @" LEFT OUTER JOIN BlocksetEntry BE ON (BE.BlocksetID = BS.ID AND BE.""Index"" = 0) " + - @"WHERE " + - @"BE.BlocksetID IS NULL ", - blocksize, - m_tempsmalllist - ); + insertBlocksetEntriesCommand += FormatInvariant($@" UNION +SELECT BS.ID AS BlocksetID, 0 AS ""Index"", BL.ID AS BlockID +FROM {m_tempsmalllist} TS + JOIN Blockset BS ON (BS.FullHash = TS.FileHash AND + BS.Length = TS.BlockSize AND + BS.Length <= {blocksize}) + JOIN Block BL ON (BL.Hash = TS.BlockHash AND + BL.Size = TS.BlockSize) + LEFT OUTER JOIN BlocksetEntry BE ON (BE.BlocksetID = BS.ID AND BE.""Index"" = 0) +WHERE +BE.BlocksetID IS NULL "); } try @@ -383,8 +342,8 @@ namespace Duplicati.Library.Main.Database using (var fixcmd = m_connection.CreateCommand()) { - fixcmd.ExecuteNonQuery(FormatInvariant(@"CREATE TABLE ""{0}_Failure"" AS SELECT * FROM ""{0}"" ", m_tempblocklist)); - fixcmd.ExecuteNonQuery(FormatInvariant(@"CREATE TABLE ""{0}_Failure"" AS SELECT * FROM ""{0}"" ", m_tempsmalllist)); + fixcmd.ExecuteNonQuery(FormatInvariant($@"CREATE TABLE ""{m_tempblocklist}_Failure"" AS SELECT * FROM ""{m_tempblocklist}"" ")); + fixcmd.ExecuteNonQuery(FormatInvariant($@"CREATE TABLE ""{m_tempsmalllist}_Failure"" AS SELECT * FROM ""{m_tempsmalllist}"" ")); } throw new Exception("The recreate failed, please create a bug-report from this database and send it to the developers for further analysis"); @@ -531,7 +490,7 @@ namespace Duplicati.Library.Main.Database m_updateBlockVolumeCommand.SetParameterValue(2, size); var c = m_updateBlockVolumeCommand.ExecuteNonQuery(); if (c != 1) - throw new Exception(FormatInvariant("Failed to update table, found {0} entries for key {1} with size {2}", c, hash, size)); + throw new Exception($"Failed to update table, found {c} entries for key {hash} with size {size}"); return true; } @@ -600,30 +559,22 @@ namespace Duplicati.Library.Main.Database var selectCommand = @"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"", ""RemoteVolume"".""ID"" FROM ""RemoteVolume"""; var missingBlocklistEntries = - FormatInvariant( - @"SELECT ""BlocklistHash"".""Hash"" FROM ""BlocklistHash"" LEFT OUTER JOIN ""BlocksetEntry"" ON ""BlocksetEntry"".""Index"" = (""BlocklistHash"".""Index"" * {0}) AND ""BlocksetEntry"".""BlocksetID"" = ""BlocklistHash"".""BlocksetID"" WHERE ""BlocksetEntry"".""BlocksetID"" IS NULL", - blocksize / hashsize - ); + FormatInvariant($@"SELECT ""BlocklistHash"".""Hash"" FROM ""BlocklistHash"" LEFT OUTER JOIN ""BlocksetEntry"" ON ""BlocksetEntry"".""Index"" = (""BlocklistHash"".""Index"" * {blocksize / hashsize}) AND ""BlocksetEntry"".""BlocksetID"" = ""BlocklistHash"".""BlocksetID"" WHERE ""BlocksetEntry"".""BlocksetID"" IS NULL"); var missingBlockInfo = @"SELECT ""VolumeID"" FROM ""Block"" WHERE ""VolumeID"" < 0 AND SIZE > 0"; var missingBlocklistVolumes = FormatInvariant( - @"SELECT ""VolumeID"" FROM ""Block"", (" + - missingBlocklistEntries + - @") A WHERE ""A"".""Hash"" = ""Block"".""Hash"" " + $@"SELECT ""VolumeID"" FROM ""Block"", ({missingBlocklistEntries}) A WHERE ""A"".""Hash"" = ""Block"".""Hash"" " ); - var countMissingInformation = FormatInvariant( - @"SELECT COUNT(*) FROM (SELECT DISTINCT ""VolumeID"" FROM ({0} UNION {1}))", - missingBlockInfo, - missingBlocklistVolumes); + var countMissingInformation = FormatInvariant($@"SELECT COUNT(*) FROM (SELECT DISTINCT ""VolumeID"" FROM ({missingBlockInfo} UNION {missingBlocklistVolumes}))"); if (passNo == 0) { // On the first pass, we select all the volumes we know we need, // which may be an empty list - cmd.CommandText = FormatInvariant(selectCommand + @" WHERE ""ID"" IN ({0})", missingBlocklistVolumes); + cmd.CommandText = selectCommand + FormatInvariant($@" WHERE ""ID"" IN ({missingBlocklistVolumes})"); // Reset the list m_proccessedVolumes = new Dictionary(); @@ -642,7 +593,7 @@ namespace Duplicati.Library.Main.Database var mentionedVolumes = @"SELECT DISTINCT ""VolumeID"" FROM ""Block"" "; - cmd.CommandText = FormatInvariant(selectCommand + @" WHERE ""ID"" NOT IN ({0}) AND ""Type"" = ? ", mentionedVolumes); + cmd.CommandText = selectCommand + FormatInvariant($@" WHERE ""ID"" NOT IN ({mentionedVolumes}) AND ""Type"" = ? "); cmd.AddParameter(RemoteVolumeType.Blocks.ToString()); } else @@ -764,15 +715,15 @@ DELETE FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND "" using (var cmd = m_connection.CreateCommand(tr.Parent)) { // 1. Select blocks not used by any file and not as a blocklist into temporary table - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tmptablename}"" AS SELECT ""Block"".""ID"", ""Block"".""Hash"", ""Block"".""Size"", ""Block"".""VolumeID"" FROM ""Block"" WHERE ""Block"".""ID"" NOT IN (SELECT ""BlocksetEntry"".""BlockID"" FROM ""BlocksetEntry"") - AND ""Block"".""Hash"" NOT IN (SELECT ""BlocklistHash"".""Hash"" FROM ""BlocklistHash"")", tmptablename)); + AND ""Block"".""Hash"" NOT IN (SELECT ""BlocklistHash"".""Hash"" FROM ""BlocklistHash"")")); // 2. Insert blocks into DeletedBlock table - cmd.ExecuteNonQuery(FormatInvariant(@"INSERT INTO ""DeletedBlock"" (""Hash"", ""Size"", ""VolumeID"") SELECT ""Hash"", ""Size"", ""VolumeID"" FROM ""{0}""", tmptablename)); + cmd.ExecuteNonQuery(FormatInvariant($@"INSERT INTO ""DeletedBlock"" (""Hash"", ""Size"", ""VolumeID"") SELECT ""Hash"", ""Size"", ""VolumeID"" FROM ""{tmptablename}""")); // 3. Remove blocks from Block table - cmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""Block"" WHERE ""ID"" IN (SELECT ""ID"" FROM ""{0}"")", tmptablename)); - cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}""", tmptablename)); + cmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""Block"" WHERE ""ID"" IN (SELECT ""ID"" FROM ""{tmptablename}"")")); + cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{tmptablename}""")); tr.Commit(); } @@ -786,7 +737,7 @@ DELETE FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND "" if (m_tempblocklist != null) try { - cmd.CommandText = FormatInvariant(@"DROP TABLE IF EXISTS ""{0}""", m_tempblocklist); + cmd.CommandText = FormatInvariant(@$"DROP TABLE IF EXISTS ""{m_tempblocklist}"""); cmd.ExecuteNonQuery(); } catch (Exception ex) @@ -801,7 +752,7 @@ DELETE FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND "" if (m_tempsmalllist != null) try { - cmd.CommandText = FormatInvariant(@"DROP TABLE IF EXISTS ""{0}""", m_tempsmalllist); + cmd.CommandText = FormatInvariant(@$"DROP TABLE IF EXISTS ""{m_tempsmalllist}"""); cmd.ExecuteNonQuery(); } catch (Exception ex) diff --git a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs index aadd51747..bc9aee475 100644 --- a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs @@ -46,7 +46,7 @@ namespace Duplicati.Library.Main.Database { var filesetid = cmd.ExecuteScalarInt64(@"SELECT ""Fileset"".""ID"" FROM ""Fileset"",""RemoteVolume"" WHERE ""Fileset"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""RemoteVolume"".""Name"" = ?", -1, filelist); if (filesetid == -1) - throw new Exception(FormatInvariant("No such remote file: {0}", filelist)); + throw new Exception($"No such remote file: {filelist}"); return filesetid; } @@ -156,19 +156,19 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) { cmd.Transaction = m_transaction.Parent; - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Restored"" INTEGER NOT NULL) ", tablename)); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" (""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Restored"" INTEGER NOT NULL) ")); m_tablename = tablename; - var blockCount = cmd.ExecuteNonQuery(FormatInvariant(@"INSERT INTO ""{0}"" (""Hash"", ""Size"", ""Restored"") SELECT DISTINCT ""Block"".""Hash"", ""Block"".""Size"", 0 AS ""Restored"" FROM ""Block"",""Remotevolume"" WHERE ""Block"".""VolumeID"" = ""Remotevolume"".""ID"" AND ""Remotevolume"".""Name"" = ? ", m_tablename), volumename); + var blockCount = cmd.ExecuteNonQuery(FormatInvariant($@"INSERT INTO ""{m_tablename}"" (""Hash"", ""Size"", ""Restored"") SELECT DISTINCT ""Block"".""Hash"", ""Block"".""Size"", 0 AS ""Restored"" FROM ""Block"",""Remotevolume"" WHERE ""Block"".""VolumeID"" = ""Remotevolume"".""ID"" AND ""Remotevolume"".""Name"" = ? "), volumename); if (blockCount == 0) throw new Exception($"Unexpected empty block volume: {0}"); - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE UNIQUE INDEX ""{0}-Ix"" ON ""{0}"" (""Hash"", ""Size"", ""Restored"")", tablename)); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE UNIQUE INDEX ""{tablename}-Ix"" ON ""{tablename}"" (""Hash"", ""Size"", ""Restored"")")); } m_insertCommand = m_connection.CreateCommand(); m_insertCommand.Transaction = m_transaction.Parent; - m_insertCommand.CommandText = FormatInvariant(@"UPDATE ""{0}"" SET ""Restored"" = ? WHERE ""Hash"" = ? AND ""Size"" = ? AND ""Restored"" = ? ", tablename); + m_insertCommand.CommandText = FormatInvariant($@"UPDATE ""{tablename}"" SET ""Restored"" = ? WHERE ""Hash"" = ? AND ""Size"" = ? AND ""Restored"" = ? "); m_insertCommand.AddParameters(4); } @@ -184,7 +184,7 @@ namespace Duplicati.Library.Main.Database public IEnumerable GetSourceFilesWithBlocks(long blocksize) { using (var cmd = m_connection.CreateCommand(m_transaction.Parent)) - using (var rd = cmd.ExecuteReader(FormatInvariant(@"SELECT DISTINCT ""{0}"".""Hash"", ""{0}"".""Size"", ""File"".""Path"", ""BlocksetEntry"".""Index"" * {1} FROM ""{0}"", ""Block"", ""BlocksetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""{0}"".""Hash"" = ""Block"".""Hash"" AND ""{0}"".""Size"" = ""Block"".""Size"" AND ""{0}"".""Restored"" = ? ", m_tablename, blocksize), 0)) + using (var rd = cmd.ExecuteReader(FormatInvariant($@"SELECT DISTINCT ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path"", ""BlocksetEntry"".""Index"" * {blocksize} FROM ""{m_tablename}"", ""Block"", ""BlocksetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""{m_tablename}"".""Hash"" = ""Block"".""Hash"" AND ""{m_tablename}"".""Size"" = ""Block"".""Size"" AND ""{m_tablename}"".""Restored"" = ? "), 0)) if (rd.Read()) { var bs = new BlockWithSources(rd); @@ -196,26 +196,25 @@ namespace Duplicati.Library.Main.Database public IEnumerable> GetMissingBlocks() { using (var cmd = m_connection.CreateCommand(m_transaction.Parent)) - foreach (var rd in cmd.ExecuteReaderEnumerable(FormatInvariant(@"SELECT ""{0}"".""Hash"", ""{0}"".""Size"" FROM ""{0}"" WHERE ""{0}"".""Restored"" = ? ", m_tablename), 0)) + foreach (var rd in cmd.ExecuteReaderEnumerable(FormatInvariant($@"SELECT ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"" FROM ""{m_tablename}"" WHERE ""{m_tablename}"".""Restored"" = ? "), 0)) yield return new KeyValuePair(rd.ConvertValueToString(0), rd.ConvertValueToInt64(1)); } public IEnumerable GetFilesetsUsingMissingBlocks() { - var blocks = @"SELECT DISTINCT ""FileLookup"".""ID"" AS ID FROM ""{0}"", ""Block"", ""Blockset"", ""BlocksetEntry"", ""FileLookup"" WHERE ""Block"".""Hash"" = ""{0}"".""Hash"" AND ""Block"".""Size"" = ""{0}"".""Size"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" "; - var blocklists = @"SELECT DISTINCT ""FileLookup"".""ID"" AS ID FROM ""{0}"", ""Block"", ""Blockset"", ""BlocklistHash"", ""FileLookup"" WHERE ""Block"".""Hash"" = ""{0}"".""Hash"" AND ""Block"".""Size"" = ""{0}"".""Size"" AND ""BlocklistHash"".""Hash"" = ""Block"".""Hash"" AND ""BlocklistHash"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" "; - - var cmdtxt = @"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""FilesetEntry"", ""Fileset"" WHERE ""RemoteVolume"".""ID"" = ""Fileset"".""VolumeID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""RemoteVolume"".""Type"" = ? AND ""FilesetEntry"".""FileID"" IN (SELECT DISTINCT ""ID"" FROM ( " + blocks + " UNION " + blocklists + " ))"; + var blocks = FormatInvariant($@"SELECT DISTINCT ""FileLookup"".""ID"" AS ID FROM ""{m_tablename}"", ""Block"", ""Blockset"", ""BlocksetEntry"", ""FileLookup"" WHERE ""Block"".""Hash"" = ""{m_tablename}"".""Hash"" AND ""Block"".""Size"" = ""{m_tablename}"".""Size"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" "); + var blocklists = FormatInvariant($@"SELECT DISTINCT ""FileLookup"".""ID"" AS ID FROM ""{m_tablename}"", ""Block"", ""Blockset"", ""BlocklistHash"", ""FileLookup"" WHERE ""Block"".""Hash"" = ""{m_tablename}"".""Hash"" AND ""Block"".""Size"" = ""{m_tablename}"".""Size"" AND ""BlocklistHash"".""Hash"" = ""Block"".""Hash"" AND ""BlocklistHash"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" "); + var cmdtxt = FormatInvariant($@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""FilesetEntry"", ""Fileset"" WHERE ""RemoteVolume"".""ID"" = ""Fileset"".""VolumeID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""RemoteVolume"".""Type"" = ? AND ""FilesetEntry"".""FileID"" IN (SELECT DISTINCT ""ID"" FROM ( {blocks} UNION {blocklists} ))"); using (var cmd = m_connection.CreateCommand(m_transaction.Parent)) - foreach (var rd in cmd.ExecuteReaderEnumerable(FormatInvariant(cmdtxt, m_tablename), RemoteVolumeType.Files.ToString())) + foreach (var rd in cmd.ExecuteReaderEnumerable(cmdtxt, RemoteVolumeType.Files.ToString())) yield return new RemoteVolume(rd.GetString(0), rd.ConvertValueToString(1), rd.ConvertValueToInt64(2)); } public IEnumerable GetMissingBlockSources() { using (var cmd = m_connection.CreateCommand(m_transaction.Parent)) - foreach (var rd in cmd.ExecuteReaderEnumerable(FormatInvariant(@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""Block"", ""{0}"" WHERE ""Block"".""Hash"" = ""{0}"".""Hash"" AND ""Block"".""Size"" = ""{0}"".""Size"" AND ""Block"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""Remotevolume"".""Name"" != ? ", m_tablename), m_volumename)) + foreach (var rd in cmd.ExecuteReaderEnumerable(FormatInvariant($@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""Block"", ""{m_tablename}"" WHERE ""Block"".""Hash"" = ""{m_tablename}"".""Hash"" AND ""Block"".""Size"" = ""{m_tablename}"".""Size"" AND ""Block"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""Remotevolume"".""Name"" != ? "), m_volumename)) yield return new RemoteVolume(rd.GetString(0), rd.ConvertValueToString(1), rd.ConvertValueToInt64(2)); } @@ -226,7 +225,7 @@ namespace Duplicati.Library.Main.Database try { using (var cmd = m_connection.CreateCommand(m_transaction.Parent)) - cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", m_transaction)); + cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tablename}"" "), m_transaction); } catch { } finally { m_tablename = null; } @@ -265,36 +264,36 @@ namespace Duplicati.Library.Main.Database var tablename = "TmpFile-" + Guid.NewGuid().ToString("N"); - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" AS SELECT * FROM ""File""", tablename)); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" AS SELECT * FROM ""File""")); var sql = @"SELECT ""A"".""ID"", ""B"".""BlocksetID"" FROM (SELECT MIN(""ID"") AS ""ID"", COUNT(""ID"") AS ""Duplicates"" FROM ""Metadataset"" GROUP BY ""BlocksetID"") ""A"", ""Metadataset"" ""B"" WHERE ""A"".""Duplicates"" > 1 AND ""A"".""ID"" = ""B"".""ID"""; using (var c2 = m_connection.CreateCommand(tr)) { - c2.CommandText = FormatInvariant(@"UPDATE ""{0}"" SET ""MetadataID"" = ? WHERE ""MetadataID"" IN (SELECT ""ID"" FROM ""Metadataset"" WHERE ""BlocksetID"" = ?)", tablename); + c2.CommandText = FormatInvariant($@"UPDATE ""{tablename}"" SET ""MetadataID"" = ? WHERE ""MetadataID"" IN (SELECT ""ID"" FROM ""Metadataset"" WHERE ""BlocksetID"" = ?)"); c2.CommandText += @"; DELETE FROM ""Metadataset"" WHERE ""BlocksetID"" = ? AND ""ID"" != ?"; using (var rd = cmd.ExecuteReader(sql)) while (rd.Read()) c2.ExecuteNonQuery(null, rd.GetValue(0), rd.GetValue(1), rd.GetValue(1), rd.GetValue(0)); } - sql = FormatInvariant(@"SELECT ""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""Entries"" FROM ( - SELECT MIN(""ID"") AS ""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Entries"" FROM ""{0}"" GROUP BY ""Path"", ""BlocksetID"", ""MetadataID"") - WHERE ""Entries"" > 1 ORDER BY ""ID""", tablename); + sql = FormatInvariant($@"SELECT ""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""Entries"" FROM ( + SELECT MIN(""ID"") AS ""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Entries"" FROM ""{tablename}"" GROUP BY ""Path"", ""BlocksetID"", ""MetadataID"") + WHERE ""Entries"" > 1 ORDER BY ""ID"""); using (var c2 = m_connection.CreateCommand()) { c2.Transaction = tr; - c2.CommandText = FormatInvariant(@"UPDATE ""FilesetEntry"" SET ""FileID"" = ? WHERE ""FileID"" IN (SELECT ""ID"" FROM ""{0}"" WHERE ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ?)", tablename); - c2.CommandText += FormatInvariant(@"; DELETE FROM ""{0}"" WHERE ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ? AND ""ID"" != ?", tablename); + c2.CommandText = FormatInvariant($@"UPDATE ""FilesetEntry"" SET ""FileID"" = ? WHERE ""FileID"" IN (SELECT ""ID"" FROM ""{tablename}"" WHERE ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ?)"); + c2.CommandText += FormatInvariant($@"; DELETE FROM ""{tablename}"" WHERE ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ? AND ""ID"" != ?"); foreach (var rd in cmd.ExecuteReaderEnumerable(sql)) c2.ExecuteNonQuery(null, rd.GetValue(0), rd.GetValue(1), rd.GetValue(2), rd.GetValue(3), rd.GetValue(1), rd.GetValue(2), rd.GetValue(3), rd.GetValue(0)); } - cmd.ExecuteNonQuery(FormatInvariant(@"DELETE FROM ""FileLookup"" WHERE ""ID"" NOT IN (SELECT ""ID"" FROM ""{0}"") ", tablename)); - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE INDEX ""{0}-Ix"" ON ""{0}"" (""ID"", ""MetadataID"")", tablename)); - cmd.ExecuteNonQuery(FormatInvariant(@"UPDATE ""FileLookup"" SET ""MetadataID"" = (SELECT ""MetadataID"" FROM ""{0}"" A WHERE ""A"".""ID"" = ""FileLookup"".""ID"") ", tablename)); - cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE ""{0}"" ", tablename)); + cmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""FileLookup"" WHERE ""ID"" NOT IN (SELECT ""ID"" FROM ""{tablename}"") ")); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""{tablename}-Ix"" ON ""{tablename}"" (""ID"", ""MetadataID"")")); + cmd.ExecuteNonQuery(FormatInvariant($@"UPDATE ""FileLookup"" SET ""MetadataID"" = (SELECT ""MetadataID"" FROM ""{tablename}"" A WHERE ""A"".""ID"" = ""FileLookup"".""ID"") ")); + cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE ""{tablename}"" ")); cmd.CommandText = sql_count; x = cmd.ExecuteScalarInt64(0); @@ -353,7 +352,7 @@ namespace Duplicati.Library.Main.Database { var hashsize = blockhasher.HashSize / 8; - var sql = FormatInvariant(@"SELECT * FROM (SELECT ""N"".""BlocksetID"", ((""N"".""BlockCount"" + {0} - 1) / {0}) AS ""BlocklistHashCountExpected"", CASE WHEN ""G"".""BlocklistHashCount"" IS NULL THEN 0 ELSE ""G"".""BlocklistHashCount"" END AS ""BlocklistHashCountActual"" FROM (SELECT ""BlocksetID"", COUNT(*) AS ""BlockCount"" FROM ""BlocksetEntry"" GROUP BY ""BlocksetID"") ""N"" LEFT OUTER JOIN (SELECT ""BlocksetID"", COUNT(*) AS ""BlocklistHashCount"" FROM ""BlocklistHash"" GROUP BY ""BlocksetID"") ""G"" ON ""N"".""BlocksetID"" = ""G"".""BlocksetID"" WHERE ""N"".""BlockCount"" > 1) WHERE ""BlocklistHashCountExpected"" != ""BlocklistHashCountActual""", blocksize / hashsize); + var sql = FormatInvariant($@"SELECT * FROM (SELECT ""N"".""BlocksetID"", ((""N"".""BlockCount"" + {blocksize / hashsize} - 1) / {blocksize / hashsize}) AS ""BlocklistHashCountExpected"", CASE WHEN ""G"".""BlocklistHashCount"" IS NULL THEN 0 ELSE ""G"".""BlocklistHashCount"" END AS ""BlocklistHashCountActual"" FROM (SELECT ""BlocksetID"", COUNT(*) AS ""BlockCount"" FROM ""BlocksetEntry"" GROUP BY ""BlocksetID"") ""N"" LEFT OUTER JOIN (SELECT ""BlocksetID"", COUNT(*) AS ""BlocklistHashCount"" FROM ""BlocklistHash"" GROUP BY ""BlocksetID"") ""G"" ON ""N"".""BlocksetID"" = ""G"".""BlocksetID"" WHERE ""N"".""BlockCount"" > 1) WHERE ""BlocklistHashCountExpected"" != ""BlocklistHashCountActual"""); var countsql = @"SELECT COUNT(*) FROM (" + sql + @")"; var itemswithnoblocklisthash = cmd.ExecuteScalarInt64(countsql, 0); @@ -507,8 +506,8 @@ namespace Duplicati.Library.Main.Database { var tablename = "ProbeBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)", tablename)); - cmd.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" (""Hash"", ""Size"") VALUES (?, ?)", tablename); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" (""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)")); + cmd.CommandText = FormatInvariant($@"INSERT INTO ""{tablename}"" (""Hash"", ""Size"") VALUES (?, ?)"); cmd.AddParameters(2); foreach (var kp in blocks) @@ -519,9 +518,9 @@ namespace Duplicati.Library.Main.Database } var id = cmd.ExecuteScalarInt64(@"SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" = ?", -1, filename); - var aliens = cmd.ExecuteScalarInt64(FormatInvariant(@"SELECT COUNT(*) FROM (SELECT ""A"".""VolumeID"" FROM ""{0}"" B LEFT OUTER JOIN ""Block"" A ON ""A"".""Hash"" = ""B"".""Hash"" AND ""A"".""Size"" = ""B"".""Size"") WHERE ""VolumeID"" != ? ", tablename), 0, id); + var aliens = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM (SELECT ""A"".""VolumeID"" FROM ""{tablename}"" B LEFT OUTER JOIN ""Block"" A ON ""A"".""Hash"" = ""B"".""Hash"" AND ""A"".""Size"" = ""B"".""Size"") WHERE ""VolumeID"" != ? "), 0, id); - cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", tablename)); + cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{tablename}"" ")); if (aliens != 0) throw new Exception($"Not all blocks were found in {filename}"); @@ -532,7 +531,7 @@ namespace Duplicati.Library.Main.Database { using (var cmd = m_connection.CreateCommand()) { - var query = FormatInvariant(@" + var query = FormatInvariant($@" SELECT ""C"".""Hash"", ""C"".""Size"" @@ -558,14 +557,13 @@ WHERE AND ""A"".""BlockID"" = ""C"".""ID"" AND - ""A"".""Index"" >= ""B"".""BlocklistHashIndex"" * ({0} / {1}) + ""A"".""Index"" >= ""B"".""BlocklistHashIndex"" * ({blocksize} / {blockhashlength}) AND - ""A"".""Index"" < (""B"".""BlocklistHashIndex"" + 1) * ({0} / {1}) + ""A"".""Index"" < (""B"".""BlocklistHashIndex"" + 1) * ({blocksize} / {blockhashlength}) ORDER BY ""A"".""Index"" -" - , blocksize, blockhashlength); +"); using (var en = blocklist.GetEnumerator()) { diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index a1aca90e3..23ebb2be8 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -89,26 +89,26 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) { // How to handle METADATA? - cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", m_fileprogtable)); - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (" - + @" ""FileId"" INTEGER PRIMARY KEY " - + @", ""TotalBlocks"" INTEGER NOT NULL, ""TotalSize"" INTEGER NOT NULL " - + @", ""BlocksRestored"" INTEGER NOT NULL, ""SizeRestored"" INTEGER NOT NULL " - + @")", m_fileprogtable)); + cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_fileprogtable}"" ")); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_fileprogtable}"" ( +""FileId"" INTEGER PRIMARY KEY, +""TotalBlocks"" INTEGER NOT NULL, ""TotalSize"" INTEGER NOT NULL, +""BlocksRestored"" INTEGER NOT NULL, ""SizeRestored"" INTEGER NOT NULL +)")); - cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", m_totalprogtable)); - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (" - + @" ""TotalFiles"" INTEGER NOT NULL, ""TotalBlocks"" INTEGER NOT NULL, ""TotalSize"" INTEGER NOT NULL " - + @", ""FilesFullyRestored"" INTEGER NOT NULL, ""FilesPartiallyRestored"" INTEGER NOT NULL " - + @", ""BlocksRestored"" INTEGER NOT NULL, ""SizeRestored"" INTEGER NOT NULL " - + @")", m_totalprogtable)); + cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_totalprogtable}"" ")); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_totalprogtable}"" ( +""TotalFiles"" INTEGER NOT NULL, ""TotalBlocks"" INTEGER NOT NULL, ""TotalSize"" INTEGER NOT NULL, +""FilesFullyRestored"" INTEGER NOT NULL, ""FilesPartiallyRestored"" INTEGER NOT NULL, +""BlocksRestored"" INTEGER NOT NULL, ""SizeRestored"" INTEGER NOT NULL +)")); if (createFilesNewlyDoneTracker) { - cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", m_filesnewlydonetable)); - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (" - + @" ""ID"" INTEGER PRIMARY KEY " - + @")", m_filesnewlydonetable)); + cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_filesnewlydonetable}"" ")); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_filesnewlydonetable}"" ( +""ID"" INTEGER PRIMARY KEY +)")); } try @@ -118,31 +118,28 @@ namespace Duplicati.Library.Main.Database // Note: FilesNewlyDone is NOT initialized, as in initialization nothing is really new. string sql; - sql = FormatInvariant( - @" INSERT INTO ""{0}"" (""FileId"", ""TotalBlocks"", ""TotalSize"", ""BlocksRestored"", ""SizeRestored"") " - + @" SELECT ""F"".""ID"", IFNULL(COUNT(""B"".""ID""), 0), IFNULL(SUM(""B"".""Size""), 0)" - + @" , IFNULL(COUNT(CASE ""B"".""Restored"" WHEN 1 THEN ""B"".""ID"" ELSE NULL END), 0) " - + @" , IFNULL(SUM(CASE ""B"".""Restored"" WHEN 1 THEN ""B"".""Size"" ELSE 0 END), 0) " - + @" FROM ""{1}"" ""F"" LEFT JOIN ""{2}"" ""B""" // allow for empty files (no data Blocks) - + @" ON ""B"".""FileID"" = ""F"".""ID"" " - + @" WHERE ""B"".""Metadata"" IS NOT 1 " // Use "IS" because of Left Join - + @" GROUP BY ""F"".""ID"" " - , m_fileprogtable, m_tempfiletable, m_tempblocktable); + // We use a LEFT JOIN to allow for empty files (no data Blocks) + sql = FormatInvariant($@" INSERT INTO ""{m_fileprogtable}"" (""FileId"", ""TotalBlocks"", ""TotalSize"", ""BlocksRestored"", ""SizeRestored"") +SELECT ""F"".""ID"", IFNULL(COUNT(""B"".""ID""), 0), IFNULL(SUM(""B"".""Size""), 0) + , IFNULL(COUNT(CASE ""B"".""Restored"" WHEN 1 THEN ""B"".""ID"" ELSE NULL END), 0) + , IFNULL(SUM(CASE ""B"".""Restored"" WHEN 1 THEN ""B"".""Size"" ELSE 0 END), 0) + FROM ""{m_tempfiletable}"" ""F"" LEFT JOIN ""{m_tempblocktable}"" ""B"" + ON ""B"".""FileID"" = ""F"".""ID"" + WHERE ""B"".""Metadata"" IS NOT 1 + GROUP BY ""F"".""ID"" "); // Will be one row per file. cmd.ExecuteNonQuery(sql); - sql = FormatInvariant( - @"INSERT INTO ""{0}"" (" - + @" ""TotalFiles"", ""TotalBlocks"", ""TotalSize"" " - + @", ""FilesFullyRestored"", ""FilesPartiallyRestored"", ""BlocksRestored"", ""SizeRestored""" - + @" ) " - + @" SELECT IFNULL(COUNT(""P"".""FileId""), 0), IFNULL(SUM(""P"".""TotalBlocks""), 0), IFNULL(SUM(""P"".""TotalSize""), 0) " - + @" , IFNULL(COUNT(CASE WHEN ""P"".""BlocksRestored"" = ""P"".""TotalBlocks"" THEN 1 ELSE NULL END), 0) " - + @" , IFNULL(COUNT(CASE WHEN ""P"".""BlocksRestored"" BETWEEN 1 AND ""P"".""TotalBlocks"" - 1 THEN 1 ELSE NULL END), 0) " - + @" , IFNULL(SUM(""P"".""BlocksRestored""), 0), IFNULL(SUM(""P"".""SizeRestored""), 0) " - + @" FROM ""{1}"" ""P"" " - , m_totalprogtable, m_fileprogtable); + sql = FormatInvariant($@"INSERT INTO ""{m_totalprogtable}"" ( + ""TotalFiles"", ""TotalBlocks"", ""TotalSize"" +, ""FilesFullyRestored"", ""FilesPartiallyRestored"", ""BlocksRestored"", ""SizeRestored"" + ) + SELECT IFNULL(COUNT(""P"".""FileId""), 0), IFNULL(SUM(""P"".""TotalBlocks""), 0), IFNULL(SUM(""P"".""TotalSize""), 0) + , IFNULL(COUNT(CASE WHEN ""P"".""BlocksRestored"" = ""P"".""TotalBlocks"" THEN 1 ELSE NULL END), 0) + , IFNULL(COUNT(CASE WHEN ""P"".""BlocksRestored"" BETWEEN 1 AND ""P"".""TotalBlocks"" - 1 THEN 1 ELSE NULL END), 0) + , IFNULL(SUM(""P"".""BlocksRestored""), 0), IFNULL(SUM(""P"".""SizeRestored""), 0) + FROM ""{m_fileprogtable}"" ""P"" "); // Will result in a single line (no support to also track metadata) cmd.ExecuteNonQuery(sql); @@ -154,44 +151,38 @@ namespace Duplicati.Library.Main.Database // with additional INSERT and DELETE triggers. // A trigger to update the file-stat entry each time a block changes restoration state. - sql = FormatInvariant( - @"CREATE TEMPORARY TRIGGER ""TrackRestoredBlocks_{1}"" AFTER UPDATE OF ""Restored"" ON ""{1}"" " - + @" WHEN OLD.""Restored"" != NEW.""Restored"" AND NEW.""Metadata"" = 0 " - + @" BEGIN UPDATE ""{0}"" " - + @" SET ""BlocksRestored"" = ""{0}"".""BlocksRestored"" + (NEW.""Restored"" - OLD.""Restored"") " - + @" , ""SizeRestored"" = ""{0}"".""SizeRestored"" + ((NEW.""Restored"" - OLD.""Restored"") * NEW.Size) " - + @" WHERE ""{0}"".""FileId"" = NEW.""FileID"" " - + @" ; END " - , m_fileprogtable, m_tempblocktable); + sql = FormatInvariant($@"CREATE TEMPORARY TRIGGER ""TrackRestoredBlocks_{m_tempblocktable}"" AFTER UPDATE OF ""Restored"" ON ""{m_tempblocktable}"" +WHEN OLD.""Restored"" != NEW.""Restored"" AND NEW.""Metadata"" = 0 +BEGIN UPDATE ""{m_fileprogtable}"" + SET ""BlocksRestored"" = ""{m_fileprogtable}"".""BlocksRestored"" + (NEW.""Restored"" - OLD.""Restored"") + , ""SizeRestored"" = ""{m_fileprogtable}"".""SizeRestored"" + ((NEW.""Restored"" - OLD.""Restored"") * NEW.Size) + WHERE ""{m_fileprogtable}"".""FileId"" = NEW.""FileID"" +; END "); cmd.ExecuteNonQuery(sql); // A trigger to update total stats each time a file stat changed (nested triggering by file-stats) - sql = FormatInvariant( - @"CREATE TEMPORARY TRIGGER ""UpdateTotalStats_{1}"" AFTER UPDATE ON ""{1}"" " - + @" BEGIN UPDATE ""{0}"" " - + @" SET ""FilesFullyRestored"" = ""{0}"".""FilesFullyRestored"" " - + @" + (CASE WHEN NEW.""BlocksRestored"" = NEW.""TotalBlocks"" THEN 1 ELSE 0 END) " - + @" - (CASE WHEN OLD.""BlocksRestored"" = OLD.""TotalBlocks"" THEN 1 ELSE 0 END) " - + @" , ""FilesPartiallyRestored"" = ""{0}"".""FilesPartiallyRestored"" " - + @" + (CASE WHEN NEW.""BlocksRestored"" BETWEEN 1 AND NEW.""TotalBlocks"" - 1 THEN 1 ELSE 0 END) " - + @" - (CASE WHEN OLD.""BlocksRestored"" BETWEEN 1 AND OLD.""TotalBlocks"" - 1 THEN 1 ELSE 0 END) " - + @" , ""BlocksRestored"" = ""{0}"".""BlocksRestored"" + NEW.""BlocksRestored"" - OLD.""BlocksRestored"" " // simple delta - + @" , ""SizeRestored"" = ""{0}"".""SizeRestored"" + NEW.""SizeRestored"" - OLD.""SizeRestored"" " // simple delta - + @" ; END " - , m_totalprogtable, m_fileprogtable); + sql = FormatInvariant($@"CREATE TEMPORARY TRIGGER ""UpdateTotalStats_{m_fileprogtable}"" AFTER UPDATE ON ""{m_fileprogtable}"" +BEGIN UPDATE ""{m_totalprogtable}"" + SET ""FilesFullyRestored"" = ""{m_totalprogtable}"".""FilesFullyRestored"" + + (CASE WHEN NEW.""BlocksRestored"" = NEW.""TotalBlocks"" THEN 1 ELSE 0 END) + - (CASE WHEN OLD.""BlocksRestored"" = OLD.""TotalBlocks"" THEN 1 ELSE 0 END) + , ""FilesPartiallyRestored"" = ""{m_totalprogtable}"".""FilesPartiallyRestored"" + + (CASE WHEN NEW.""BlocksRestored"" BETWEEN 1 AND NEW.""TotalBlocks"" - 1 THEN 1 ELSE 0 END) + - (CASE WHEN OLD.""BlocksRestored"" BETWEEN 1 AND OLD.""TotalBlocks"" - 1 THEN 1 ELSE 0 END) + , ""BlocksRestored"" = ""{m_totalprogtable}"".""BlocksRestored"" + NEW.""BlocksRestored"" - OLD.""BlocksRestored"" + , ""SizeRestored"" = ""{m_totalprogtable}"".""SizeRestored"" + NEW.""SizeRestored"" - OLD.""SizeRestored"" +; END"); cmd.ExecuteNonQuery(sql); if (createFilesNewlyDoneTracker) { // A trigger checking if a file is done (all blocks restored in file-stat) (nested triggering by file-stats) - sql = FormatInvariant( - @"CREATE TEMPORARY TRIGGER ""UpdateFilesNewlyDone_{1}"" AFTER UPDATE OF ""BlocksRestored"", ""TotalBlocks"" ON ""{1}"" " - + @" WHEN NEW.""BlocksRestored"" = NEW.""TotalBlocks"" " - + @" BEGIN " - + @" INSERT OR IGNORE INTO ""{0}"" (""ID"") VALUES (NEW.""FileId""); " - + @" END " - , m_filesnewlydonetable, m_fileprogtable); + sql = FormatInvariant($@"CREATE TEMPORARY TRIGGER ""UpdateFilesNewlyDone_{m_fileprogtable}"" AFTER UPDATE OF ""BlocksRestored"", ""TotalBlocks"" ON ""{m_fileprogtable}"" +WHEN NEW.""BlocksRestored"" = NEW.""TotalBlocks"" +BEGIN + INSERT OR IGNORE INTO ""{m_filesnewlydonetable}"" (""ID"") VALUES (NEW.""FileId""); +END "); cmd.ExecuteNonQuery(sql); } @@ -232,17 +223,17 @@ namespace Duplicati.Library.Main.Database cmd.Parameters.Clear(); - cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", m_tempfiletable)); - cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", m_tempblocktable)); - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (""ID"" INTEGER PRIMARY KEY, ""Path"" TEXT NOT NULL, ""BlocksetID"" INTEGER NOT NULL, ""MetadataID"" INTEGER NOT NULL, ""TargetPath"" TEXT NULL, ""DataVerified"" BOOLEAN NOT NULL, ""LatestBlocksetId"" INTEGER, ""LocalSourceExists"" BOOLEAN) ", m_tempfiletable)); - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (""ID"" INTEGER PRIMARY KEY, ""FileID"" INTEGER NOT NULL, ""Index"" INTEGER NOT NULL, ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Restored"" BOOLEAN NOT NULL, ""Metadata"" BOOLEAN NOT NULL, ""VolumeID"" INTEGER NOT NULL, ""BlockID"" INTEGER NOT NULL)", m_tempblocktable)); + cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tempfiletable}"" ")); + cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tempblocktable}"" ")); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tempfiletable}"" (""ID"" INTEGER PRIMARY KEY, ""Path"" TEXT NOT NULL, ""BlocksetID"" INTEGER NOT NULL, ""MetadataID"" INTEGER NOT NULL, ""TargetPath"" TEXT NULL, ""DataVerified"" BOOLEAN NOT NULL, ""LatestBlocksetId"" INTEGER, ""LocalSourceExists"" BOOLEAN) ")); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tempblocktable}"" (""ID"" INTEGER PRIMARY KEY, ""FileID"" INTEGER NOT NULL, ""Index"" INTEGER NOT NULL, ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Restored"" BOOLEAN NOT NULL, ""Metadata"" BOOLEAN NOT NULL, ""VolumeID"" INTEGER NOT NULL, ""BlockID"" INTEGER NOT NULL)")); // TODO: Optimize to use the path prefix if (filter == null || filter.Empty) { // Simple case, restore everything - cmd.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" (""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") SELECT ""File"".""ID"", ""File"".""Path"", ""File"".""BlocksetID"", ""File"".""MetadataID"", 0 FROM ""File"", ""FilesetEntry"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FilesetEntry"".""FilesetID"" = ? ", m_tempfiletable); + cmd.CommandText = FormatInvariant($@"INSERT INTO ""{m_tempfiletable}"" (""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") SELECT ""File"".""ID"", ""File"".""Path"", ""File"".""BlocksetID"", ""File"".""MetadataID"", 0 FROM ""File"", ""FilesetEntry"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FilesetEntry"".""FilesetID"" = ? "); cmd.AddParameter(filesetId); cmd.ExecuteNonQuery(); } @@ -256,8 +247,8 @@ namespace Duplicati.Library.Main.Database var p = expression.GetSimpleList(); var m_filenamestable = "Filenames-" + m_temptabsetguid; cmd.Transaction = tr; - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL) ", m_filenamestable)); - cmd.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" (""Path"") VALUES (?)", m_filenamestable); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_filenamestable}"" (""Path"" TEXT NOT NULL) ")); + cmd.CommandText = FormatInvariant($@"INSERT INTO ""{m_filenamestable}"" (""Path"") VALUES (?)"); cmd.AddParameter(); foreach (var s in p) @@ -266,7 +257,7 @@ namespace Duplicati.Library.Main.Database cmd.ExecuteNonQuery(); } - cmd.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" (""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") SELECT ""File"".""ID"", ""File"".""Path"", ""File"".""BlocksetID"", ""File"".""MetadataID"", 0 FROM ""File"", ""FilesetEntry"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FilesetEntry"".""FilesetID"" = ? AND ""Path"" IN (SELECT DISTINCT ""Path"" FROM ""{1}"") ", m_tempfiletable, m_filenamestable); + cmd.CommandText = FormatInvariant($@"INSERT INTO ""{m_tempfiletable}"" (""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") SELECT ""File"".""ID"", ""File"".""Path"", ""File"".""BlocksetID"", ""File"".""MetadataID"", 0 FROM ""File"", ""FilesetEntry"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FilesetEntry"".""FilesetID"" = ? AND ""Path"" IN (SELECT DISTINCT ""Path"" FROM ""{m_filenamestable}"") "); cmd.SetParameterValue(0, filesetId); var c = cmd.ExecuteNonQuery(); @@ -277,7 +268,7 @@ namespace Duplicati.Library.Main.Database var sb = new StringBuilder(); sb.AppendLine(); - using (var rd = cmd.ExecuteReader(FormatInvariant(@"SELECT ""Path"" FROM ""{0}"" WHERE ""Path"" NOT IN (SELECT ""Path"" FROM ""{1}"")", m_filenamestable, m_tempfiletable))) + using (var rd = cmd.ExecuteReader(FormatInvariant($@"SELECT ""Path"" FROM ""{m_filenamestable}"" WHERE ""Path"" NOT IN (SELECT ""Path"" FROM ""{m_tempfiletable}"")"))) while (rd.Read()) sb.AppendLine(rd.GetValue(0).ToString()); @@ -286,7 +277,7 @@ namespace Duplicati.Library.Main.Database cmd.Parameters.Clear(); } - cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", m_filenamestable)); + cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_filenamestable}"" ")); using (new Logging.Timer(LOGTAG, "CommitPrepareFileset", "CommitPrepareFileset")) tr.Commit(); @@ -302,7 +293,7 @@ namespace Duplicati.Library.Main.Database object[] values = new object[4]; using (var cmd2 = m_connection.CreateCommand()) { - cmd2.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" (""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") VALUES (?,?,?,?,0)", m_tempfiletable); + cmd2.CommandText = FormatInvariant($@"INSERT INTO ""{m_tempfiletable}"" (""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") VALUES (?,?,?,?,0)"); cmd2.AddParameter(); cmd2.AddParameter(); cmd2.AddParameter(); @@ -325,11 +316,11 @@ namespace Duplicati.Library.Main.Database } //creating indexes after insertion is much faster - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE INDEX ""{0}_ID"" ON ""{0}"" (""ID"")", m_tempfiletable)); - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE INDEX ""{0}_TargetPath"" ON ""{0}"" (""TargetPath"")", m_tempfiletable)); - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE INDEX ""{0}_Path"" ON ""{0}"" (""Path"")", m_tempfiletable)); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""{m_tempfiletable}_ID"" ON ""{m_tempfiletable}"" (""ID"")")); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""{m_tempfiletable}_TargetPath"" ON ""{m_tempfiletable}"" (""TargetPath"")")); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""{m_tempfiletable}_Path"" ON ""{m_tempfiletable}"" (""Path"")")); - using (var rd = cmd.ExecuteReader(FormatInvariant(@"SELECT COUNT(DISTINCT ""{0}"".""Path""), SUM(""Blockset"".""Length"") FROM ""{0}"", ""Blockset"" WHERE ""{0}"".""BlocksetID"" = ""Blockset"".""ID"" ", m_tempfiletable))) + using (var rd = cmd.ExecuteReader(FormatInvariant($@"SELECT COUNT(DISTINCT ""{m_tempfiletable}"".""Path""), SUM(""Blockset"".""Length"") FROM ""{m_tempfiletable}"", ""Blockset"" WHERE ""{m_tempfiletable}"".""BlocksetID"" = ""Blockset"".""ID"" "))) { var filecount = 0L; var filesize = 0L; @@ -356,7 +347,7 @@ namespace Duplicati.Library.Main.Database { using (var cmd = m_connection.CreateCommand()) { - cmd.CommandText = FormatInvariant(@"SELECT ""Path"" FROM ""{0}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1", m_tempfiletable); + cmd.CommandText = FormatInvariant($@"SELECT ""Path"" FROM ""{m_tempfiletable}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1"); var v0 = cmd.ExecuteScalar(); if (v0 == null || v0 == DBNull.Value) return null; @@ -369,7 +360,7 @@ namespace Duplicati.Library.Main.Database { using (var cmd = m_connection.CreateCommand()) { - cmd.CommandText = FormatInvariant(@"SELECT ""Path"" FROM ""{0}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1", m_tempfiletable); + cmd.CommandText = FormatInvariant($@"SELECT ""Path"" FROM ""{m_tempfiletable}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1"); var v0 = cmd.ExecuteScalar(); string maxpath = ""; if (v0 != null && v0 != DBNull.Value) @@ -377,12 +368,12 @@ namespace Duplicati.Library.Main.Database var dirsep = Util.GuessDirSeparator(maxpath); - cmd.CommandText = FormatInvariant(@"SELECT COUNT(*) FROM ""{0}""", m_tempfiletable); + cmd.CommandText = FormatInvariant($@"SELECT COUNT(*) FROM ""{m_tempfiletable}"""); var filecount = cmd.ExecuteScalarInt64(-1); long foundfiles = -1; //TODO: Handle FS case-sensitive? - cmd.CommandText = FormatInvariant(@"SELECT COUNT(*) FROM ""{0}"" WHERE SUBSTR(""Path"", 1, ?) = ?", m_tempfiletable); + cmd.CommandText = FormatInvariant($@"SELECT COUNT(*) FROM ""{m_tempfiletable}"" WHERE SUBSTR(""Path"", 1, ?) = ?"); cmd.AddParameter(); cmd.AddParameter(); @@ -423,19 +414,19 @@ namespace Duplicati.Library.Main.Database if ((OperatingSystem.IsMacOS() || OperatingSystem.IsLinux()) && dirsep == "\\") { // For Win -> Linux, we remove the colon from the drive letter, and use the drive letter as root folder - cmd.ExecuteNonQuery(FormatInvariant(@"UPDATE ""{0}"" SET ""Targetpath"" = CASE WHEN SUBSTR(""Path"", 2, 1) == ':' THEN '\\' || SUBSTR(""Path"", 1, 1) || SUBSTR(""Path"", 3) ELSE ""Path"" END", m_tempfiletable)); - cmd.ExecuteNonQuery(FormatInvariant(@"UPDATE ""{0}"" SET ""Targetpath"" = CASE WHEN SUBSTR(""Path"", 1, 2) == '\\' THEN '\\' || SUBSTR(""Path"", 2) ELSE ""Path"" END", m_tempfiletable)); + cmd.ExecuteNonQuery(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""Targetpath"" = CASE WHEN SUBSTR(""Path"", 2, 1) == ':' THEN '\\' || SUBSTR(""Path"", 1, 1) || SUBSTR(""Path"", 3) ELSE ""Path"" END")); + cmd.ExecuteNonQuery(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""Targetpath"" = CASE WHEN SUBSTR(""Path"", 1, 2) == '\\' THEN '\\' || SUBSTR(""Path"", 2) ELSE ""Path"" END")); } else if (OperatingSystem.IsWindows() && dirsep == "/") { // For Linux -> Win, we use the temporary folder's drive as the root path - cmd.ExecuteNonQuery(FormatInvariant(@"UPDATE ""{0}"" SET ""Targetpath"" = CASE WHEN SUBSTR(""Path"", 1, 1) == '/' THEN ? || SUBSTR(""Path"", 2) ELSE ""Path"" END", m_tempfiletable), Util.AppendDirSeparator(System.IO.Path.GetPathRoot(Library.Utility.TempFolder.SystemTempPath)).Replace("\\", "/")); + cmd.ExecuteNonQuery(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""Targetpath"" = CASE WHEN SUBSTR(""Path"", 1, 1) == '/' THEN ? || SUBSTR(""Path"", 2) ELSE ""Path"" END"), Util.AppendDirSeparator(System.IO.Path.GetPathRoot(Library.Utility.TempFolder.SystemTempPath)).Replace("\\", "/")); } else { // Same OS, just use the path directly - cmd.ExecuteNonQuery(FormatInvariant(@"UPDATE ""{0}"" SET ""Targetpath"" = ""Path"" ", m_tempfiletable)); + cmd.ExecuteNonQuery(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""Targetpath"" = ""Path"" ")); } } else @@ -445,32 +436,32 @@ namespace Duplicati.Library.Main.Database //Special case, restoring to new folder, but files are from different drives (no shared root on Windows) // We use the format / / - cmd.ExecuteNonQuery(FormatInvariant(@"UPDATE ""{0}"" SET ""TargetPath"" = CASE WHEN SUBSTR(""Path"", 2, 1) == ':' THEN SUBSTR(""Path"", 1, 1) || SUBSTR(""Path"", 3) ELSE ""Path"" END", m_tempfiletable)); + cmd.ExecuteNonQuery(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = CASE WHEN SUBSTR(""Path"", 2, 1) == ':' THEN SUBSTR(""Path"", 1, 1) || SUBSTR(""Path"", 3) ELSE ""Path"" END")); // For UNC paths, we use \\server\folder -> / / - cmd.ExecuteNonQuery(FormatInvariant(@"UPDATE ""{0}"" SET ""TargetPath"" = CASE WHEN SUBSTR(""Path"", 1, 2) == '\\' THEN SUBSTR(""Path"", 2) ELSE ""TargetPath"" END", m_tempfiletable)); + cmd.ExecuteNonQuery(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = CASE WHEN SUBSTR(""Path"", 1, 2) == '\\' THEN SUBSTR(""Path"", 2) ELSE ""TargetPath"" END")); } else { largest_prefix = Util.AppendDirSeparator(largest_prefix, dirsep); - cmd.ExecuteNonQuery(FormatInvariant(@"UPDATE ""{0}"" SET ""TargetPath"" = SUBSTR(""Path"", ?)", m_tempfiletable), largest_prefix.Length + 1); + cmd.ExecuteNonQuery(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = SUBSTR(""Path"", ?)"), largest_prefix.Length + 1); } } // Cross-os path remapping support if ((OperatingSystem.IsMacOS() || OperatingSystem.IsLinux()) && dirsep == "\\") // For Win paths on Linux - cmd.ExecuteNonQuery(FormatInvariant(@"UPDATE ""{0}"" SET ""TargetPath"" = REPLACE(""TargetPath"", '\', '/')", m_tempfiletable)); + cmd.ExecuteNonQuery(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = REPLACE(""TargetPath"", '\', '/')")); else if (OperatingSystem.IsWindows() && dirsep == "/") // For Linux paths on Windows - cmd.ExecuteNonQuery(FormatInvariant(@"UPDATE ""{0}"" SET ""TargetPath"" = REPLACE(REPLACE(""TargetPath"", '\', '_'), '/', '\')", m_tempfiletable)); + cmd.ExecuteNonQuery(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = REPLACE(REPLACE(""TargetPath"", '\', '_'), '/', '\')")); if (!string.IsNullOrEmpty(destination)) { // Paths are now relative with target-os naming system // so we prefix them with the target path - cmd.ExecuteNonQuery(FormatInvariant(@"UPDATE ""{0}"" SET ""TargetPath"" = ? || ""TargetPath"" ", m_tempfiletable), Util.AppendDirSeparator(destination)); + cmd.ExecuteNonQuery(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = ? || ""TargetPath"" "), Util.AppendDirSeparator(destination)); } @@ -481,23 +472,23 @@ namespace Duplicati.Library.Main.Database { using (var cmd = m_connection.CreateCommand()) { - cmd.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" (""FileID"", ""Index"", ""Hash"", ""Size"", ""Restored"", ""Metadata"", ""VolumeId"", ""BlockId"") SELECT DISTINCT ""{1}"".""ID"", ""BlocksetEntry"".""Index"", ""Block"".""Hash"", ""Block"".""Size"", 0, 0, ""Block"".""VolumeID"", ""Block"".""ID"" FROM ""{1}"", ""BlocksetEntry"", ""Block"" WHERE ""{1}"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" ", m_tempblocktable, m_tempfiletable); + cmd.CommandText = FormatInvariant($@"INSERT INTO ""{m_tempblocktable}"" (""FileID"", ""Index"", ""Hash"", ""Size"", ""Restored"", ""Metadata"", ""VolumeId"", ""BlockId"") SELECT DISTINCT ""{m_tempfiletable}"".""ID"", ""BlocksetEntry"".""Index"", ""Block"".""Hash"", ""Block"".""Size"", 0, 0, ""Block"".""VolumeID"", ""Block"".""ID"" FROM ""{m_tempfiletable}"", ""BlocksetEntry"", ""Block"" WHERE ""{m_tempfiletable}"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" "); var p1 = cmd.ExecuteNonQuery(); int p2 = 0; if (!skipMetadata) { - cmd.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" (""FileID"", ""Index"", ""Hash"", ""Size"", ""Restored"", ""Metadata"", ""VolumeId"", ""BlockId"") SELECT DISTINCT ""{1}"".""ID"", ""BlocksetEntry"".""Index"", ""Block"".""Hash"", ""Block"".""Size"", 0, 1, ""Block"".""VolumeID"", ""Block"".""ID"" FROM ""{1}"", ""BlocksetEntry"", ""Block"", ""Metadataset"" WHERE ""{1}"".""MetadataID"" = ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" ", m_tempblocktable, m_tempfiletable); + cmd.CommandText = FormatInvariant($@"INSERT INTO ""{m_tempblocktable}"" (""FileID"", ""Index"", ""Hash"", ""Size"", ""Restored"", ""Metadata"", ""VolumeId"", ""BlockId"") SELECT DISTINCT ""{m_tempfiletable}"".""ID"", ""BlocksetEntry"".""Index"", ""Block"".""Hash"", ""Block"".""Size"", 0, 1, ""Block"".""VolumeID"", ""Block"".""ID"" FROM ""{m_tempfiletable}"", ""BlocksetEntry"", ""Block"", ""Metadataset"" WHERE ""{m_tempfiletable}"".""MetadataID"" = ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" "); p2 = cmd.ExecuteNonQuery(); } //creating indexes after insertion is much faster - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE INDEX ""{0}_HashSizeIndex"" ON ""{0}"" (""Hash"", ""Size"")", m_tempblocktable)); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""{m_tempblocktable}_HashSizeIndex"" ON ""{m_tempblocktable}"" (""Hash"", ""Size"")")); // better suited to speed up commit on UpdateBlocks - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE INDEX ""{0}_FileIdIndexIndex"" ON ""{0}"" (""FileId"", ""Index"")", m_tempblocktable)); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""{m_tempblocktable}_FileIdIndexIndex"" ON ""{m_tempblocktable}"" (""FileId"", ""Index"")")); - var size = cmd.ExecuteScalarInt64(FormatInvariant(@"SELECT SUM(""Size"") FROM ""{0}"" ", m_tempblocktable), 0); + var size = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT SUM(""Size"") FROM ""{m_tempblocktable}"" "), 0); Logging.Log.WriteVerboseMessage(LOGTAG, "RestoreSourceSize", "Restore list contains {0} blocks with a total size of {1}", p1 + p2, Library.Utility.Utility.FormatSizeString(size)); } } @@ -505,7 +496,7 @@ namespace Duplicati.Library.Main.Database public void UpdateTargetPath(long ID, string newname) { using (var cmd = m_connection.CreateCommand()) - cmd.ExecuteNonQuery(FormatInvariant(@"UPDATE ""{0}"" SET ""TargetPath"" = ? WHERE ""ID"" = ?", m_tempfiletable), newname, ID); + cmd.ExecuteNonQuery(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = ? WHERE ""ID"" = ?"), newname, ID); } public interface IExistingFileBlock @@ -614,7 +605,7 @@ namespace Duplicati.Library.Main.Database { using (var cmd = connection.CreateCommand()) { - cmd.CommandText = FormatInvariant(@"SELECT ""{0}"".""TargetPath"", ""Blockset"".""FullHash"", ""{0}"".""ID"", ""Blockset"".""Length"", ""Block"".""Hash"", ""BlocksetEntry"".""Index"", ""Block"".""Size"" FROM ""{0}"", ""Blockset"", ""BlocksetEntry"", ""Block"" WHERE ""{0}"".""BlocksetID"" = ""Blockset"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""{0}"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" ORDER BY ""{0}"".""TargetPath"", ""BlocksetEntry"".""Index""", tablename); + cmd.CommandText = FormatInvariant($@"SELECT ""{tablename}"".""TargetPath"", ""Blockset"".""FullHash"", ""{tablename}"".""ID"", ""Blockset"".""Length"", ""Block"".""Hash"", ""BlocksetEntry"".""Index"", ""Block"".""Size"" FROM ""{tablename}"", ""Blockset"", ""BlocksetEntry"", ""Block"" WHERE ""{tablename}"".""BlocksetID"" = ""Blockset"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""{tablename}"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" ORDER BY ""{tablename}"".""TargetPath"", ""BlocksetEntry"".""Index"""); using (var rd = cmd.ExecuteReader()) if (rd.Read()) { @@ -716,7 +707,7 @@ namespace Duplicati.Library.Main.Database using (var cmd = connection.CreateCommand()) { // TODO: Skip metadata as required - cmd.CommandText = FormatInvariant(@"SELECT DISTINCT ""A"".""TargetPath"", ""A"".""ID"", ""B"".""Hash"", (""B"".""Index"" * {2}), ""B"".""Index"", ""B"".""Size"", ""C"".""Path"", (""D"".""Index"" * {2}), ""E"".""Size"", ""B"".""Metadata"" FROM ""{0}"" ""A"", ""{1}"" ""B"", ""File"" ""C"", ""BlocksetEntry"" ""D"", ""Block"" E WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""C"".""BlocksetID"" = ""D"".""BlocksetID"" AND ""D"".""BlockID"" = ""E"".""ID"" AND ""B"".""Hash"" = ""E"".""Hash"" AND ""B"".""Size"" = ""E"".""Size"" AND ""B"".""Restored"" = 0", filetablename, blocktablename, blocksize); + cmd.CommandText = FormatInvariant($@"SELECT DISTINCT ""A"".""TargetPath"", ""A"".""ID"", ""B"".""Hash"", (""B"".""Index"" * {blocksize}), ""B"".""Index"", ""B"".""Size"", ""C"".""Path"", (""D"".""Index"" * {blocksize}), ""E"".""Size"", ""B"".""Metadata"" FROM ""{filetablename}"" ""A"", ""{blocktablename}"" ""B"", ""File"" ""C"", ""BlocksetEntry"" ""D"", ""Block"" E WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""C"".""BlocksetID"" = ""D"".""BlocksetID"" AND ""D"".""BlockID"" = ""E"".""ID"" AND ""B"".""Hash"" = ""E"".""Hash"" AND ""B"".""Size"" = ""E"".""Size"" AND ""B"".""Restored"" = 0"); using (var rd = cmd.ExecuteReader()) { if (rd.Read()) @@ -757,16 +748,14 @@ namespace Duplicati.Library.Main.Database // Now it is likely to restore all files from front to back. Large files will always be done last. // One could also use like the average block number in a volume, that needs to be measured. - cmd.CommandText = FormatInvariant( - @"SELECT ""RV"".""Name"", ""RV"".""Hash"", ""RV"".""Size"", ""BB"".""MaxIndex"" " - + @" FROM ""RemoteVolume"" ""RV"" INNER JOIN " - + @" (SELECT ""TB"".""VolumeID"", MAX(""TB"".""Index"") as ""MaxIndex"" " - + @" FROM ""{0}"" ""TB"" " - + @" WHERE ""TB"".""Restored"" = 0 " - + @" GROUP BY ""TB"".""VolumeID"" " - + @" ) as ""BB"" ON ""RV"".""ID"" = ""BB"".""VolumeID"" " - + @" ORDER BY ""BB"".""MaxIndex"" " - , m_tempblocktable); + cmd.CommandText = FormatInvariant($@"SELECT ""RV"".""Name"", ""RV"".""Hash"", ""RV"".""Size"", ""BB"".""MaxIndex"" +FROM ""RemoteVolume"" ""RV"" INNER JOIN + (SELECT ""TB"".""VolumeID"", MAX(""TB"".""Index"") as ""MaxIndex"" + FROM ""{m_tempblocktable}"" ""TB"" + WHERE ""TB"".""Restored"" = 0 + GROUP BY ""TB"".""VolumeID"" + ) as ""BB"" ON ""RV"".""ID"" = ""BB"".""VolumeID"" +ORDER BY ""BB"".""MaxIndex"" "); using (var rd = cmd.ExecuteReader()) { @@ -809,11 +798,11 @@ namespace Duplicati.Library.Main.Database using (var c = m_connection.CreateCommand()) { m_tmptable = "VolumeFiles-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - c.CommandText = FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" ( ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL )", m_tmptable); + c.CommandText = FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tmptable}"" ( ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL )"); c.ExecuteNonQuery(); - c.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" (""Hash"", ""Size"") VALUES (?,?)", m_tmptable); + c.CommandText = FormatInvariant($@"INSERT INTO ""{m_tmptable}"" (""Hash"", ""Size"") VALUES (?,?)"); c.AddParameters(2); foreach (var s in curvolume.Blocks) { @@ -832,7 +821,7 @@ namespace Duplicati.Library.Main.Database if (m_tmptable != null) using (var c = m_connection.CreateCommand()) { - c.CommandText = FormatInvariant(@"DROP TABLE IF EXISTS ""{0}""", m_tmptable); + c.CommandText = FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tmptable}"""); c.ExecuteNonQuery(); } } @@ -877,13 +866,11 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) { // The IN-clause with subquery enables SQLite to use indexes better. Three way join (A,B,C) is slow here! - cmd.CommandText = FormatInvariant( - @" SELECT DISTINCT ""A"".""TargetPath"", ""BB"".""FileID"", (""BB"".""Index"" * {3}), ""BB"".""Size"", ""BB"".""Hash"" " - + @" FROM ""{0}"" ""A"", ""{1}"" ""BB"" " - + @" WHERE ""A"".""ID"" = ""BB"".""FileID"" AND ""BB"".""Restored"" = 0 AND ""BB"".""Metadata"" = {4}" - + @" AND ""BB"".""ID"" IN (SELECT ""B"".""ID"" FROM ""{1}"" ""B"", ""{2}"" ""C"" WHERE ""B"".""Hash"" = ""C"".""Hash"" AND ""B"".""Size"" = ""C"".""Size"") " - + @" ORDER BY ""A"".""TargetPath"", ""BB"".""Index""" - , m_filetablename, m_blocktablename, m_tmptable, m_blocksize, "0"); + cmd.CommandText = FormatInvariant($@" SELECT DISTINCT ""A"".""TargetPath"", ""BB"".""FileID"", (""BB"".""Index"" * {m_blocksize}), ""BB"".""Size"", ""BB"".""Hash"" +FROM ""{m_filetablename}"" ""A"", ""{m_blocktablename}"" ""BB"" +WHERE ""A"".""ID"" = ""BB"".""FileID"" AND ""BB"".""Restored"" = 0 AND ""BB"".""Metadata"" = {"0"} +AND ""BB"".""ID"" IN (SELECT ""B"".""ID"" FROM ""{m_blocktablename}"" ""B"", ""{m_tmptable}"" ""C"" WHERE ""B"".""Hash"" = ""C"".""Hash"" AND ""B"".""Size"" = ""C"".""Size"") +ORDER BY ""A"".""TargetPath"", ""BB"".""Index"""); using (var rd = cmd.ExecuteReader()) { if (rd.Read()) @@ -912,13 +899,11 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) { // The IN-clause with subquery enables SQLite to use indexes better. Three way join (A,B,C) is slow here! - cmd.CommandText = FormatInvariant( - @" SELECT DISTINCT ""A"".""TargetPath"", ""BB"".""FileID"", (""BB"".""Index"" * {3}), ""BB"".""Size"", ""BB"".""Hash"" " - + @" FROM ""{0}"" ""A"", ""{1}"" ""BB"" " - + @" WHERE ""A"".""ID"" = ""BB"".""FileID"" AND ""BB"".""Restored"" = 0 AND ""BB"".""Metadata"" = {4}" - + @" AND ""BB"".""ID"" IN (SELECT ""B"".""ID"" FROM ""{1}"" ""B"", ""{2}"" ""C"" WHERE ""B"".""Hash"" = ""C"".""Hash"" AND ""B"".""Size"" = ""C"".""Size"") " - + @" ORDER BY ""A"".""TargetPath"", ""BB"".""Index""" - , m_filetablename, m_blocktablename, m_tmptable, m_blocksize, "1"); + cmd.CommandText = FormatInvariant($@" SELECT DISTINCT ""A"".""TargetPath"", ""BB"".""FileID"", (""BB"".""Index"" * {m_blocksize}), ""BB"".""Size"", ""BB"".""Hash"" + FROM ""{m_filetablename}"" ""A"", ""{m_blocktablename}"" ""BB"" +WHERE ""A"".""ID"" = ""BB"".""FileID"" AND ""BB"".""Restored"" = 0 AND ""BB"".""Metadata"" = {"1"} + AND ""BB"".""ID"" IN (SELECT ""B"".""ID"" FROM ""{m_blocktablename}"" ""B"", ""{m_tmptable}"" ""C"" WHERE ""B"".""Hash"" = ""C"".""Hash"" AND ""B"".""Size"" = ""C"".""Size"") +ORDER BY ""A"".""TargetPath"", ""BB"".""Index"""); using (var rd = cmd.ExecuteReader()) { if (rd.Read()) @@ -985,7 +970,7 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) { cmd.AddParameter(!onlyNonVerified); - using (var rd = cmd.ExecuteReader(FormatInvariant(@"SELECT ""{0}"".""ID"", ""{0}"".""TargetPath"", ""Blockset"".""FullHash"", ""Blockset"".""Length"" FROM ""{0}"",""Blockset"" WHERE ""{0}"".""BlocksetID"" = ""Blockset"".""ID"" AND ""{0}"".""DataVerified"" <= ?", m_tempfiletable))) + using (var rd = cmd.ExecuteReader(FormatInvariant($@"SELECT ""{m_tempfiletable}"".""ID"", ""{m_tempfiletable}"".""TargetPath"", ""Blockset"".""FullHash"", ""Blockset"".""Length"" FROM ""{m_tempfiletable}"",""Blockset"" WHERE ""{m_tempfiletable}"".""BlocksetID"" = ""Blockset"".""ID"" AND ""{m_tempfiletable}"".""DataVerified"" <= ?"))) while (rd.Read()) yield return new FileToRestore(rd.ConvertValueToInt64(0), rd.ConvertValueToString(1), rd.ConvertValueToString(2), rd.ConvertValueToInt64(3)); } @@ -1118,7 +1103,7 @@ namespace Duplicati.Library.Main.Database if (m_tempfiletable != null) try { - cmd.CommandText = FormatInvariant(@"DROP TABLE IF EXISTS ""{0}""", m_tempfiletable); + cmd.CommandText = FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tempfiletable}"""); cmd.ExecuteNonQuery(); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } @@ -1127,7 +1112,7 @@ namespace Duplicati.Library.Main.Database if (m_tempblocktable != null) try { - cmd.CommandText = FormatInvariant(@"DROP TABLE IF EXISTS ""{0}""", m_tempblocktable); + cmd.CommandText = FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tempblocktable}"""); cmd.ExecuteNonQuery(); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } @@ -1136,7 +1121,7 @@ namespace Duplicati.Library.Main.Database if (m_latestblocktable != null) try { - cmd.CommandText = FormatInvariant(@"DROP TABLE IF EXISTS ""{0}""", m_latestblocktable); + cmd.CommandText = FormatInvariant($@"DROP TABLE IF EXISTS ""{m_latestblocktable}"""); cmd.ExecuteNonQuery(); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } @@ -1145,7 +1130,7 @@ namespace Duplicati.Library.Main.Database if (m_fileprogtable != null) try { - cmd.CommandText = FormatInvariant(@"DROP TABLE IF EXISTS ""{0}""", m_fileprogtable); + cmd.CommandText = FormatInvariant($@"DROP TABLE IF EXISTS ""{m_fileprogtable}"""); cmd.ExecuteNonQuery(); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } @@ -1154,7 +1139,7 @@ namespace Duplicati.Library.Main.Database if (m_totalprogtable != null) try { - cmd.CommandText = FormatInvariant(@"DROP TABLE IF EXISTS ""{0}""", m_totalprogtable); + cmd.CommandText = FormatInvariant($@"DROP TABLE IF EXISTS ""{m_totalprogtable}"""); cmd.ExecuteNonQuery(); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } @@ -1163,7 +1148,7 @@ namespace Duplicati.Library.Main.Database if (m_filesnewlydonetable != null) try { - cmd.CommandText = FormatInvariant(@"DROP TABLE IF EXISTS ""{0}""", m_filesnewlydonetable); + cmd.CommandText = FormatInvariant($@"DROP TABLE IF EXISTS ""{m_filesnewlydonetable}"""); cmd.ExecuteNonQuery(); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } @@ -1216,35 +1201,27 @@ namespace Duplicati.Library.Main.Database m_blocktablename = blocktablename; m_filetablename = filetablename; - m_insertblockCommand.CommandText = FormatInvariant( - @"UPDATE ""{0}"" SET ""Restored"" = 1 " - + @" WHERE ""FileID"" = ? AND ""Index"" = ? AND ""Hash"" = ? AND ""Size"" = ? AND ""Metadata"" = ? AND ""Restored"" = 0 " - , m_blocktablename); + m_insertblockCommand.CommandText = FormatInvariant($@"UPDATE ""{m_blocktablename}"" SET ""Restored"" = 1 +WHERE ""FileID"" = ? AND ""Index"" = ? AND ""Hash"" = ? AND ""Size"" = ? AND ""Metadata"" = ? AND ""Restored"" = 0 "); m_insertblockCommand.AddParameters(5); - m_resetfileCommand.CommandText = FormatInvariant( - @"UPDATE ""{0}"" SET ""Restored"" = 0 WHERE ""FileID"" = ? " - , m_blocktablename); + m_resetfileCommand.CommandText = FormatInvariant($@"UPDATE ""{m_blocktablename}"" SET ""Restored"" = 0 WHERE ""FileID"" = ? "); m_resetfileCommand.AddParameters(1); - m_updateAsRestoredCommand.CommandText = FormatInvariant( - @"UPDATE ""{0}"" SET ""Restored"" = 1 WHERE ""FileID"" = ? AND ""Metadata"" <= ? " - , m_blocktablename); + m_updateAsRestoredCommand.CommandText = FormatInvariant($@"UPDATE ""{m_blocktablename}"" SET ""Restored"" = 1 WHERE ""FileID"" = ? AND ""Metadata"" <= ? "); m_updateAsRestoredCommand.AddParameters(2); - m_updateFileAsDataVerifiedCommand.CommandText = FormatInvariant( - @"UPDATE ""{0}"" SET ""DataVerified"" = 1 WHERE ""ID"" = ?" - , m_filetablename); + m_updateFileAsDataVerifiedCommand.CommandText = FormatInvariant($@"UPDATE ""{m_filetablename}"" SET ""DataVerified"" = 1 WHERE ""ID"" = ?"); m_updateFileAsDataVerifiedCommand.AddParameters(1); if (statstablename != null) { // Fields in Stats: TotalFiles, TotalBlocks, TotalSize // FilesFullyRestored, FilesPartiallyRestored, BlocksRestored, SizeRestored - m_statUpdateCommand.CommandText = FormatInvariant(@"SELECT SUM(""FilesFullyRestored""), SUM(""SizeRestored"") FROM ""{0}"" ", statstablename); + m_statUpdateCommand.CommandText = FormatInvariant($@"SELECT SUM(""FilesFullyRestored""), SUM(""SizeRestored"") FROM ""{statstablename}"" "); } else // very slow fallback if stats tables were not created - m_statUpdateCommand.CommandText = FormatInvariant(@"SELECT COUNT(DISTINCT ""FileID""), SUM(""Size"") FROM ""{0}"" WHERE ""Restored"" = 1 ", m_blocktablename); + m_statUpdateCommand.CommandText = FormatInvariant($@"SELECT COUNT(DISTINCT ""FileID""), SUM(""Size"") FROM ""{m_blocktablename}"" WHERE ""Restored"" = 1 "); } @@ -1370,7 +1347,7 @@ namespace Duplicati.Library.Main.Database public IEnumerable GetTargetFolders() { using (var cmd = m_connection.CreateCommand()) - using (var rd = cmd.ExecuteReader(FormatInvariant(@"SELECT ""TargetPath"" FROM ""{0}"" WHERE ""BlocksetID"" == ?", m_tempfiletable), FOLDER_BLOCKSET_ID)) + using (var rd = cmd.ExecuteReader(FormatInvariant($@"SELECT ""TargetPath"" FROM ""{m_tempfiletable}"" WHERE ""BlocksetID"" == ?"), FOLDER_BLOCKSET_ID)) while (rd.Read()) yield return rd.ConvertValueToString(0); } @@ -1435,11 +1412,11 @@ namespace Duplicati.Library.Main.Database { using (var cmd = m_connection.CreateCommand()) { - cmd.CommandText = FormatInvariant(@"UPDATE ""{0}"" SET ""LocalSourceExists"" = 1 WHERE Path = ?", m_tempfiletable); + cmd.CommandText = FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""LocalSourceExists"" = 1 WHERE Path = ?"); cmd.AddParameters(1); cmd.Transaction = transaction; - var fileset = FormatInvariant(@"SELECT DISTINCT ""{0}"".""Path"" FROM ""{0}""", m_tempfiletable); + var fileset = FormatInvariant($@"SELECT DISTINCT ""{m_tempfiletable}"".""Path"" FROM ""{m_tempfiletable}"""); using (var rd = cmdReader.ExecuteReader(fileset)) { while (rd.Read()) @@ -1458,7 +1435,7 @@ namespace Duplicati.Library.Main.Database } //This localSourceExists index will make the query engine to start by searching FileSet table. As the result is ordered by FileSet.ID, we will get the cursor "instantly" - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE INDEX ""{0}_LocalSourceExists"" ON ""{0}"" (""LocalSourceExists"")", m_tempfiletable)); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""{m_tempfiletable}_LocalSourceExists"" ON ""{m_tempfiletable}"" (""LocalSourceExists"")")); } } transaction.Commit(); @@ -1466,13 +1443,12 @@ namespace Duplicati.Library.Main.Database m_latestblocktable = "LatestBlocksetIds-" + m_temptabsetguid; - var whereclause = FormatInvariant(@" - ""{0}"".""LocalSourceExists"" = 1 AND - ""{1}"".""Restored"" = 0 AND ""{1}"".""Metadata"" = 0 AND - ""{0}"".""TargetPath"" != ""{0}"".""Path""", - m_tempfiletable, m_tempblocktable); + var whereclause = FormatInvariant($@" + ""{m_tempfiletable}"".""LocalSourceExists"" = 1 AND + ""{m_tempblocktable}"".""Restored"" = 0 AND ""{m_tempblocktable}"".""Metadata"" = 0 AND + ""{m_tempfiletable}"".""TargetPath"" != ""{m_tempfiletable}"".""Path"""); - var latestBlocksetIds = FormatInvariant(@" + var latestBlocksetIds = FormatInvariant($@" SELECT ""File"".""Path"" AS ""PATH"", ""File"".""BlocksetID"" AS ""BlocksetID"", @@ -1486,44 +1462,43 @@ namespace Duplicati.Library.Main.Database ""FilesetEntry"".""FilesetID"" = ""Fileset"".""ID"" AND ""File"".""Path"" IN (SELECT DISTINCT - ""{0}"".""Path"" + ""{m_tempfiletable}"".""Path"" FROM - ""{0}"", - ""{1}"" + ""{m_tempfiletable}"", + ""{m_tempblocktable}"" WHERE - ""{0}"".""ID"" = ""{1}"".""FileID"" AND - {2}) - GROUP BY ""File"".""Path""", - m_tempfiletable, m_tempblocktable, whereclause); + ""{m_tempfiletable}"".""ID"" = ""{m_tempblocktable}"".""FileID"" AND + {whereclause}) + GROUP BY ""File"".""Path"""); using (var cmd = m_connection.CreateCommand()) { - cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}"" ", m_latestblocktable)); - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" AS {1}", m_latestblocktable, latestBlocksetIds)); - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE INDEX ""{0}_path"" ON ""{0}"" (""Path"")", m_latestblocktable)); + cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_latestblocktable}"" ")); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_latestblocktable}"" AS {latestBlocksetIds}")); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""{m_latestblocktable}_path"" ON ""{m_latestblocktable}"" (""Path"")")); - cmd.ExecuteNonQuery(FormatInvariant(@"UPDATE ""{0}"" SET LatestBlocksetId = (SELECT BlocksetId FROM ""{1}"" WHERE Path = ""{0}"".Path)", m_tempfiletable, m_latestblocktable)); + cmd.ExecuteNonQuery(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET LatestBlocksetId = (SELECT BlocksetId FROM ""{m_latestblocktable}"" WHERE Path = ""{m_tempfiletable}"".Path)")); } - var sources = FormatInvariant( - @" SELECT DISTINCT " + - @" ""{0}"".""TargetPath""," + - @" ""{0}"".""Path""," + - @" ""{0}"".""ID""," + - @" ""{1}"".""Index""," + - @" ""{1}"".""Hash""," + - @" ""{1}"".""Size""" + - @" FROM" + - @" ""{0}""," + - @" ""{1}""," + - @" ""BlocksetEntry""" + - @" WHERE" + - @" ""{0}"".""ID"" = ""{1}"".""FileID"" AND" + - @" ""BlocksetEntry"".""BlocksetID"" = ""{0}"".""LatestBlocksetID"" AND" + - @" ""BlocksetEntry"".""BlockID"" = ""{1}"".""BlockID"" AND" + - @" ""BlocksetEntry"".""Index"" = ""{1}"".""Index"" AND" + - @" " + whereclause + - @" ORDER BY ""{0}"".""ID"", ""{1}"".""Index"" ", m_tempfiletable, m_tempblocktable); + var sources = FormatInvariant($@" +SELECT DISTINCT + ""{m_tempfiletable}"".""TargetPath"", + ""{m_tempfiletable}"".""Path"", + ""{m_tempfiletable}"".""ID"", + ""{m_tempblocktable}"".""Index"", + ""{m_tempblocktable}"".""Hash"", + ""{m_tempblocktable}"".""Size"" +FROM + ""{m_tempfiletable}"", + ""{m_tempblocktable}"", + ""BlocksetEntry"" +WHERE + ""{m_tempfiletable}"".""ID"" = ""{m_tempblocktable}"".""FileID"" AND + ""BlocksetEntry"".""BlocksetID"" = ""{m_tempfiletable}"".""LatestBlocksetID"" AND + ""BlocksetEntry"".""BlockID"" = ""{m_tempblocktable}"".""BlockID"" AND + ""BlocksetEntry"".""Index"" = ""{m_tempblocktable}"".""Index"" AND + {whereclause} +ORDER BY ""{m_tempfiletable}"".""ID"", ""{m_tempblocktable}"".""Index"" "); using (var cmd = m_connection.CreateCommand()) using (var rd = cmd.ExecuteReader(sources)) diff --git a/Duplicati/Library/Main/Database/LocalTestDatabase.cs b/Duplicati/Library/Main/Database/LocalTestDatabase.cs index e6b8f8b1c..16aadde88 100644 --- a/Duplicati/Library/Main/Database/LocalTestDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalTestDatabase.cs @@ -187,13 +187,13 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) { cmd.Transaction = m_transaction; - cmd.ExecuteNonQuery(FormatInvariant(@"CREATE TEMPORARY TABLE ""{0}"" {1}", tablename, tableFormat)); + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" {tableFormat}")); m_tablename = tablename; } m_insertCommand = m_connection.CreateCommand(); m_insertCommand.Transaction = m_transaction; - m_insertCommand.CommandText = FormatInvariant(@"INSERT INTO ""{0}"" {1}", m_tablename, insertCommand); + m_insertCommand.CommandText = FormatInvariant($@"INSERT INTO ""{m_tablename}"" {insertCommand}"); m_insertCommand.AddParameters(insertArguments); } @@ -205,7 +205,7 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) { cmd.Transaction = m_transaction; - cmd.ExecuteNonQuery(FormatInvariant(@"DROP TABLE IF EXISTS ""{0}""", m_tablename)); + cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tablename}""")); } } catch { } @@ -255,11 +255,11 @@ namespace Duplicati.Library.Main.Database { var cmpName = "CmpTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - var create = @"CREATE TEMPORARY TABLE ""{1}"" AS SELECT ""A"".""Path"" AS ""Path"", CASE WHEN ""B"".""Fullhash"" IS NULL THEN -1 ELSE ""B"".""Length"" END AS ""Size"", ""B"".""Fullhash"" AS ""Hash"", ""C"".""Length"" AS ""Metasize"", ""C"".""Fullhash"" AS ""Metahash"" FROM (SELECT ""File"".""Path"", ""File"".""BlocksetID"" AS ""FileBlocksetID"", ""Metadataset"".""BlocksetID"" AS ""MetadataBlocksetID"" from ""Remotevolume"", ""Fileset"", ""FilesetEntry"", ""File"", ""Metadataset"" WHERE ""Remotevolume"".""Name"" = ? AND ""Fileset"".""VolumeID"" = ""Remotevolume"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""MetadataID"" = ""Metadataset"".""ID"") A LEFT OUTER JOIN ""Blockset"" B ON ""B"".""ID"" = ""A"".""FileBlocksetID"" LEFT OUTER JOIN ""Blockset"" C ON ""C"".""ID""=""A"".""MetadataBlocksetID"" "; - var extra = @"SELECT ? AS ""Type"", ""{0}"".""Path"" AS ""Path"" FROM ""{0}"" WHERE ""{0}"".""Path"" NOT IN ( SELECT ""Path"" FROM ""{1}"" )"; - var missing = @"SELECT ? AS ""Type"", ""Path"" AS ""Path"" FROM ""{1}"" WHERE ""Path"" NOT IN (SELECT ""Path"" FROM ""{0}"")"; - var modified = @"SELECT ? AS ""Type"", ""E"".""Path"" AS ""Path"" FROM ""{0}"" E, ""{1}"" D WHERE ""D"".""Path"" = ""E"".""Path"" AND (""D"".""Size"" != ""E"".""Size"" OR ""D"".""Hash"" != ""E"".""Hash"" OR ""D"".""Metasize"" != ""E"".""Metasize"" OR ""D"".""Metahash"" != ""E"".""Metahash"") "; - var drop = @"DROP TABLE IF EXISTS ""{1}"" "; + var create = FormatInvariant($@"CREATE TEMPORARY TABLE ""{cmpName}"" AS SELECT ""A"".""Path"" AS ""Path"", CASE WHEN ""B"".""Fullhash"" IS NULL THEN -1 ELSE ""B"".""Length"" END AS ""Size"", ""B"".""Fullhash"" AS ""Hash"", ""C"".""Length"" AS ""Metasize"", ""C"".""Fullhash"" AS ""Metahash"" FROM (SELECT ""File"".""Path"", ""File"".""BlocksetID"" AS ""FileBlocksetID"", ""Metadataset"".""BlocksetID"" AS ""MetadataBlocksetID"" from ""Remotevolume"", ""Fileset"", ""FilesetEntry"", ""File"", ""Metadataset"" WHERE ""Remotevolume"".""Name"" = ? AND ""Fileset"".""VolumeID"" = ""Remotevolume"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""MetadataID"" = ""Metadataset"".""ID"") A LEFT OUTER JOIN ""Blockset"" B ON ""B"".""ID"" = ""A"".""FileBlocksetID"" LEFT OUTER JOIN ""Blockset"" C ON ""C"".""ID""=""A"".""MetadataBlocksetID"" "); + var extra = FormatInvariant($@"SELECT ? AS ""Type"", ""{m_tablename}"".""Path"" AS ""Path"" FROM ""{m_tablename}"" WHERE ""{m_tablename}"".""Path"" NOT IN ( SELECT ""Path"" FROM ""{cmpName}"" )"); + var missing = FormatInvariant($@"SELECT ? AS ""Type"", ""Path"" AS ""Path"" FROM ""{cmpName}"" WHERE ""Path"" NOT IN (SELECT ""Path"" FROM ""{m_tablename}"")"); + var modified = FormatInvariant($@"SELECT ? AS ""Type"", ""E"".""Path"" AS ""Path"" FROM ""{m_tablename}"" E, ""{cmpName}"" D WHERE ""D"".""Path"" = ""E"".""Path"" AND (""D"".""Size"" != ""E"".""Size"" OR ""D"".""Hash"" != ""E"".""Hash"" OR ""D"".""Metasize"" != ""E"".""Metasize"" OR ""D"".""Metahash"" != ""E"".""Metahash"") "); + var drop = FormatInvariant($@"DROP TABLE IF EXISTS ""{cmpName}"" "); using (var cmd = m_connection.CreateCommand()) { @@ -267,15 +267,15 @@ namespace Duplicati.Library.Main.Database try { - cmd.ExecuteNonQuery(FormatInvariant(create, m_tablename, cmpName), m_volumename); - using (var rd = cmd.ExecuteReader(FormatInvariant(extra + " UNION " + missing + " UNION " + modified, m_tablename, cmpName), (int)Library.Interface.TestEntryStatus.Extra, (int)Library.Interface.TestEntryStatus.Missing, (int)Library.Interface.TestEntryStatus.Modified)) + cmd.ExecuteNonQuery(create, m_volumename); + using (var rd = cmd.ExecuteReader(FormatInvariant($"{extra} UNION {missing} UNION {modified}"), (int)Library.Interface.TestEntryStatus.Extra, (int)Library.Interface.TestEntryStatus.Missing, (int)Library.Interface.TestEntryStatus.Modified)) while (rd.Read()) yield return new KeyValuePair((Duplicati.Library.Interface.TestEntryStatus)rd.GetInt64(0), rd.GetString(1)); } finally { - try { cmd.ExecuteNonQuery(FormatInvariant(drop, m_tablename, cmpName)); } + try { cmd.ExecuteNonQuery(drop); } catch { } } } @@ -311,11 +311,11 @@ namespace Duplicati.Library.Main.Database public IEnumerable> Compare() { var cmpName = "CmpTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - var create = @"CREATE TEMPORARY TABLE ""{1}"" AS SELECT ""A"".""Name"", ""A"".""Hash"", ""A"".""Size"" FROM ""Remotevolume"" A, ""Remotevolume"" B, ""IndexBlockLink"" WHERE ""B"".""Name"" = ? AND ""A"".""ID"" = ""IndexBlockLink"".""BlockVolumeID"" AND ""B"".""ID"" = ""IndexBlockLink"".""IndexVolumeID"" "; - var extra = @"SELECT ? AS ""Type"", ""{0}"".""Name"" AS ""Name"" FROM ""{0}"" WHERE ""{0}"".""Name"" NOT IN ( SELECT ""Name"" FROM ""{1}"" )"; - var missing = @"SELECT ? AS ""Type"", ""Name"" AS ""Name"" FROM ""{1}"" WHERE ""Name"" NOT IN (SELECT ""Name"" FROM ""{0}"")"; - var modified = @"SELECT ? AS ""Type"", ""E"".""Name"" AS ""Name"" FROM ""{0}"" E, ""{1}"" D WHERE ""D"".""Name"" = ""E"".""Name"" AND (""D"".""Hash"" != ""E"".""Hash"" OR ""D"".""Size"" != ""E"".""Size"") "; - var drop = @"DROP TABLE IF EXISTS ""{1}"" "; + var create = FormatInvariant($@"CREATE TEMPORARY TABLE ""{cmpName}"" AS SELECT ""A"".""Name"", ""A"".""Hash"", ""A"".""Size"" FROM ""Remotevolume"" A, ""Remotevolume"" B, ""IndexBlockLink"" WHERE ""B"".""Name"" = ? AND ""A"".""ID"" = ""IndexBlockLink"".""BlockVolumeID"" AND ""B"".""ID"" = ""IndexBlockLink"".""IndexVolumeID"" "); + var extra = FormatInvariant($@"SELECT ? AS ""Type"", ""{m_tablename}"".""Name"" AS ""Name"" FROM ""{m_tablename}"" WHERE ""{m_tablename}"".""Name"" NOT IN ( SELECT ""Name"" FROM ""{cmpName}"" )"); + var missing = FormatInvariant($@"SELECT ? AS ""Type"", ""Name"" AS ""Name"" FROM ""{cmpName}"" WHERE ""Name"" NOT IN (SELECT ""Name"" FROM ""{m_tablename}"")"); + var modified = FormatInvariant($@"SELECT ? AS ""Type"", ""E"".""Name"" AS ""Name"" FROM ""{m_tablename}"" E, ""{cmpName}"" D WHERE ""D"".""Name"" = ""E"".""Name"" AND (""D"".""Hash"" != ""E"".""Hash"" OR ""D"".""Size"" != ""E"".""Size"") "); + var drop = FormatInvariant($@"DROP TABLE IF EXISTS ""{cmpName}"" "); using (var cmd = m_connection.CreateCommand()) { @@ -323,15 +323,15 @@ namespace Duplicati.Library.Main.Database try { - cmd.ExecuteNonQuery(FormatInvariant(create, m_tablename, cmpName), m_volumename); - using (var rd = cmd.ExecuteReader(FormatInvariant(extra + " UNION " + missing + " UNION " + modified, m_tablename, cmpName), (int)Library.Interface.TestEntryStatus.Extra, (int)Library.Interface.TestEntryStatus.Missing, (int)Library.Interface.TestEntryStatus.Modified)) + cmd.ExecuteNonQuery(create, m_volumename); + using (var rd = cmd.ExecuteReader(FormatInvariant($"{extra} UNION {missing} UNION {modified}"), (int)Library.Interface.TestEntryStatus.Extra, (int)Library.Interface.TestEntryStatus.Missing, (int)Library.Interface.TestEntryStatus.Modified)) while (rd.Read()) yield return new KeyValuePair((Duplicati.Library.Interface.TestEntryStatus)rd.GetInt64(0), rd.GetString(1)); } finally { - try { cmd.ExecuteNonQuery(FormatInvariant(drop, m_tablename, cmpName)); } + try { cmd.ExecuteNonQuery(drop); } catch { } } } @@ -368,11 +368,11 @@ namespace Duplicati.Library.Main.Database var curBlocks = @"SELECT ""Block"".""Hash"" AS ""Hash"", ""Block"".""Size"" AS ""Size"" FROM ""Remotevolume"", ""Block"" WHERE ""Remotevolume"".""Name"" = ? AND ""Remotevolume"".""ID"" = ""Block"".""VolumeID"""; var duplBlocks = @"SELECT ""Block"".""Hash"" AS ""Hash"", ""Block"".""Size"" AS ""Size"" FROM ""DuplicateBlock"", ""Block"" WHERE ""DuplicateBlock"".""VolumeID"" = (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" = ?) AND ""Block"".""ID"" = ""DuplicateBlock"".""BlockID"""; var delBlocks = @"SELECT ""DeletedBlock"".""Hash"" AS ""Hash"", ""DeletedBlock"".""Size"" AS ""Size"" FROM ""DeletedBlock"", ""RemoteVolume"" WHERE ""RemoteVolume"".""Name"" = ? AND ""RemoteVolume"".""ID"" = ""DeletedBlock"".""VolumeID"""; - var create = @"CREATE TEMPORARY TABLE ""{0}"" AS SELECT DISTINCT ""Hash"" AS ""Hash"", ""Size"" AS ""Size"" FROM ({1} UNION {2} UNION {3})"; - var extra = @"SELECT ? AS ""Type"", ""{0}"".""Hash"" AS ""Hash"" FROM ""{0}"" WHERE ""{0}"".""Hash"" NOT IN ( SELECT ""Hash"" FROM ""{1}"" )"; - var missing = @"SELECT ? AS ""Type"", ""Hash"" AS ""Hash"" FROM ""{1}"" WHERE ""Hash"" NOT IN (SELECT ""Hash"" FROM ""{0}"")"; - var modified = @"SELECT ? AS ""Type"", ""E"".""Hash"" AS ""Hash"" FROM ""{0}"" E, ""{1}"" D WHERE ""D"".""Hash"" = ""E"".""Hash"" AND ""D"".""Size"" != ""E"".""Size"" "; - var drop = @"DROP TABLE IF EXISTS ""{1}"" "; + var create = FormatInvariant($@"CREATE TEMPORARY TABLE ""{cmpName}"" AS SELECT DISTINCT ""Hash"" AS ""Hash"", ""Size"" AS ""Size"" FROM ({curBlocks} UNION {delBlocks} UNION {duplBlocks})"); + var extra = FormatInvariant($@"SELECT ? AS ""Type"", ""{m_tablename}"".""Hash"" AS ""Hash"" FROM ""{m_tablename}"" WHERE ""{m_tablename}"".""Hash"" NOT IN ( SELECT ""Hash"" FROM ""{cmpName}"" )"); + var missing = FormatInvariant($@"SELECT ? AS ""Type"", ""Hash"" AS ""Hash"" FROM ""{cmpName}"" WHERE ""Hash"" NOT IN (SELECT ""Hash"" FROM ""{m_tablename}"")"); + var modified = FormatInvariant($@"SELECT ? AS ""Type"", ""E"".""Hash"" AS ""Hash"" FROM ""{m_tablename}"" E, ""{cmpName}"" D WHERE ""D"".""Hash"" = ""E"".""Hash"" AND ""D"".""Size"" != ""E"".""Size"" "); + var drop = FormatInvariant($@"DROP TABLE IF EXISTS ""{cmpName}"" "); using (var cmd = m_connection.CreateCommand()) { @@ -380,15 +380,15 @@ namespace Duplicati.Library.Main.Database try { - cmd.ExecuteNonQuery(FormatInvariant(create, cmpName, curBlocks, delBlocks, duplBlocks), m_volumename, m_volumename, m_volumename); - using (var rd = cmd.ExecuteReader(FormatInvariant(extra + " UNION " + missing + " UNION " + modified, m_tablename, cmpName), (int)Library.Interface.TestEntryStatus.Extra, (int)Library.Interface.TestEntryStatus.Missing, (int)Library.Interface.TestEntryStatus.Modified)) + cmd.ExecuteNonQuery(create, m_volumename, m_volumename, m_volumename); + using (var rd = cmd.ExecuteReader(FormatInvariant($"{extra} UNION {missing} UNION {modified}"), (int)Library.Interface.TestEntryStatus.Extra, (int)Library.Interface.TestEntryStatus.Missing, (int)Library.Interface.TestEntryStatus.Modified)) while (rd.Read()) yield return new KeyValuePair((Duplicati.Library.Interface.TestEntryStatus)rd.GetInt64(0), rd.GetString(1)); } finally { - try { cmd.ExecuteNonQuery(FormatInvariant(drop, m_tablename, cmpName)); } + try { cmd.ExecuteNonQuery(drop); } catch { } } } From bb1aa8e1f762bebeca458b697546c6979f6e290b Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 14 Mar 2025 14:34:56 +0100 Subject: [PATCH 354/362] Cosmetic cleanup Removed superfluous namespaces. Using `var` most places. Using extension methods for common setup of commands. --- .../Library/Main/Database/ExtensionMethods.cs | 82 +++-- .../Library/Main/Database/LocalDatabase.cs | 311 +++++++----------- .../Main/Database/LocalDeleteDatabase.cs | 66 ++-- .../Database/LocalListAffectedDatabase.cs | 16 +- .../Database/LocalListBrokenFilesDatabase.cs | 15 +- .../Main/Database/LocalListChangesDatabase.cs | 148 ++++----- .../Main/Database/LocalListDatabase.cs | 105 +++--- .../Main/Database/LocalPurgeDatabase.cs | 23 +- .../Main/Database/LocalRecreateDatabase.cs | 145 +++----- .../Main/Database/LocalRepairDatabase.cs | 42 +-- .../Main/Database/LocalRestoreDatabase.cs | 249 ++++++-------- .../Main/Database/LocalTestDatabase.cs | 77 ++--- .../Library/Main/Database/RemoteVolume.cs | 64 ++-- .../Main/Database/RemoteVolumeEntry.cs | 43 ++- Duplicati/Library/Main/Utility.cs | 39 ++- Duplicati/Library/Utility/Utility.cs | 13 +- 16 files changed, 606 insertions(+), 832 deletions(-) diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index 5f172c5e1..7512f9412 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -20,6 +20,7 @@ // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; +using System.Data; using System.Linq; #nullable enable @@ -27,7 +28,7 @@ using System.Linq; namespace Duplicati.Library.Main.Database; /// -/// Extension method for +/// Extension method for /// public static class ExtensionMethods { @@ -44,7 +45,7 @@ public static class ExtensionMethods /// The parameters to add and set /// The command with the parameters added and set public static T AddAndSetParameters(this T cmd, params object?[]? parameters) - where T : System.Data.IDbCommand + where T : IDbCommand { if (parameters != null) for (var i = 0; i < parameters.Length; i++) @@ -65,7 +66,7 @@ public static class ExtensionMethods /// The number of parameters to add /// The command with the parameters added public static T AddParameters(this T self, int count) - where T : System.Data.IDbCommand + where T : IDbCommand { for (var i = 0; i < count; i++) self.Parameters.Add(self.CreateParameter()); @@ -80,7 +81,7 @@ public static class ExtensionMethods /// The command to add the parameter to /// The command with the parameter added public static T AddParameter(this T self) - where T : System.Data.IDbCommand + where T : IDbCommand { self.Parameters.Add(self.CreateParameter()); return self; @@ -95,7 +96,7 @@ public static class ExtensionMethods /// The optional name of the parameter /// The command with the parameter added public static T AddParameter(this T self, object? value, string? name = null) - where T : System.Data.IDbCommand + where T : IDbCommand { var p = self.CreateParameter(); p.Value = value; @@ -113,10 +114,10 @@ public static class ExtensionMethods /// The values to set the parameters to /// The command with the parameter values set public static T SetParameterValues(this T self, params object?[] values) - where T : System.Data.IDbCommand + where T : IDbCommand { for (var i = 0; i < values.Length; i++) - ((System.Data.IDataParameter)self.Parameters[i]!).Value = values[i]; + ((IDataParameter)self.Parameters[i]!).Value = values[i]; return self; } @@ -129,9 +130,9 @@ public static class ExtensionMethods /// The value to set the parameter to /// The command with the parameter value set public static T SetParameterValue(this T self, int index, object? value) - where T : System.Data.IDbCommand + where T : IDbCommand { - ((System.Data.IDataParameter)self.Parameters[index]!).Value = value; + ((IDataParameter)self.Parameters[index]!).Value = value; return self; } @@ -140,11 +141,11 @@ public static class ExtensionMethods /// The command to get the printable command text for /// The printable command text - public static string GetPrintableCommandText(this System.Data.IDbCommand self) + public static string GetPrintableCommandText(this IDbCommand self) { var txt = self.CommandText; - foreach (var p in self.Parameters.Cast()) + foreach (var p in self.Parameters.Cast()) { var ix = txt.IndexOf('?'); if (ix >= 0) @@ -170,7 +171,7 @@ public static class ExtensionMethods /// The command instance to execute on /// Whether to write a log entry /// The number of rows affected - public static int ExecuteNonQuery(this System.Data.IDbCommand self, bool writeLog) + public static int ExecuteNonQuery(this IDbCommand self, bool writeLog) { return ExecuteNonQuery(self, writeLog, null, null); } @@ -182,7 +183,7 @@ public static class ExtensionMethods /// The command string to execute /// The values to use as parameters. The parameters must already be added. /// The number of rows affected - public static int ExecuteNonQuery(this System.Data.IDbCommand self, string cmd, params object?[]? values) + public static int ExecuteNonQuery(this IDbCommand self, string cmd, params object?[]? values) { return ExecuteNonQuery(self, true, cmd, values); } @@ -195,7 +196,7 @@ public static class ExtensionMethods /// The command string to execute /// The values to use as parameters. The parameters must already be added. /// The number of rows affected - public static int ExecuteNonQuery(this System.Data.IDbCommand self, bool writeLog, string? cmd, params object?[]? values) + public static int ExecuteNonQuery(this IDbCommand self, bool writeLog, string? cmd, params object?[]? values) { if (cmd != null) self.CommandText = cmd; @@ -218,7 +219,7 @@ public static class ExtensionMethods /// The command string to execute /// The values to use as parameters. The parameters must already be added. /// The scalar value of the first row - public static object? ExecuteScalar(this System.Data.IDbCommand self, string cmd, params object[] values) + public static object? ExecuteScalar(this IDbCommand self, string cmd, params object[] values) { return ExecuteScalar(self, true, cmd, values); } @@ -231,7 +232,7 @@ public static class ExtensionMethods /// The command string to execute /// The values to use as parameters. The parameters must already be added. /// The scalar value of the first row - public static object? ExecuteScalar(this System.Data.IDbCommand self, bool writeLog, string cmd, params object[] values) + public static object? ExecuteScalar(this IDbCommand self, bool writeLog, string cmd, params object[] values) { if (cmd != null) self.CommandText = cmd; @@ -254,7 +255,7 @@ public static class ExtensionMethods /// Whether to write a log entry /// The default value to return if no value is found /// The scalar int64 value of the first row as a string - public static long ExecuteScalarInt64(this System.Data.IDbCommand self, bool writeLog, long defaultvalue = -1) + public static long ExecuteScalarInt64(this IDbCommand self, bool writeLog, long defaultvalue = -1) { return ExecuteScalarInt64(self, writeLog, null, defaultvalue); } @@ -265,7 +266,7 @@ public static class ExtensionMethods /// The command instance to execute on /// The default value to return if no value is found /// The scalar int64 value of the first row as a string - public static long ExecuteScalarInt64(this System.Data.IDbCommand self, long defaultvalue = -1) + public static long ExecuteScalarInt64(this IDbCommand self, long defaultvalue = -1) { return ExecuteScalarInt64(self, true, null, defaultvalue); } @@ -278,7 +279,7 @@ public static class ExtensionMethods /// The command string to execute /// The default value to return if no value is found /// The scalar int64 value of the first row as a string - public static long ExecuteScalarInt64(this System.Data.IDbCommand self, bool writeLog, string? cmd, long defaultvalue = -1) + public static long ExecuteScalarInt64(this IDbCommand self, bool writeLog, string? cmd, long defaultvalue = -1) { return ExecuteScalarInt64(self, writeLog, cmd, defaultvalue, null); } @@ -290,7 +291,7 @@ public static class ExtensionMethods /// The command string to execute /// The default value to return if no value is found /// The scalar int64 value of the first row as a string - public static long ExecuteScalarInt64(this System.Data.IDbCommand self, string? cmd, long defaultvalue = -1) + public static long ExecuteScalarInt64(this IDbCommand self, string? cmd, long defaultvalue = -1) { return ExecuteScalarInt64(self, true, cmd, defaultvalue, null); } @@ -303,7 +304,7 @@ public static class ExtensionMethods /// The default value to return if no value is found /// The values to use as parameters. The parameters must already be added. /// The scalar int64 value of the first row as a string - public static long ExecuteScalarInt64(this System.Data.IDbCommand self, string? cmd, long defaultvalue, params object?[]? values) + public static long ExecuteScalarInt64(this IDbCommand self, string? cmd, long defaultvalue, params object?[]? values) { return ExecuteScalarInt64(self, true, cmd, defaultvalue, values); } @@ -317,7 +318,7 @@ public static class ExtensionMethods /// The default value to return if no value is found /// The values to use as parameters. The parameters must already be added. /// The scalar int64 value of the first row as a string - public static long ExecuteScalarInt64(this System.Data.IDbCommand self, bool writeLog, string? cmd, long defaultvalue, params object?[]? values) + public static long ExecuteScalarInt64(this IDbCommand self, bool writeLog, string? cmd, long defaultvalue, params object?[]? values) { if (cmd != null) self.CommandText = cmd; @@ -343,8 +344,8 @@ public static class ExtensionMethods /// The command instance to execute on /// The command string to execute /// The values to use as parameters. The parameters must already be added. - /// A instance - public static System.Data.IDataReader ExecuteReader(this System.Data.IDbCommand self, string cmd, params object[] values) + /// A instance + public static IDataReader ExecuteReader(this IDbCommand self, string cmd, params object[] values) { return ExecuteReader(self, true, cmd, values); } @@ -356,8 +357,8 @@ public static class ExtensionMethods /// Whether to write a log entry /// The command string to execute /// The values to use as parameters. The parameters must already be added. - /// A instance - public static System.Data.IDataReader ExecuteReader(this System.Data.IDbCommand self, bool writeLog, string cmd, params object[] values) + /// A instance + public static IDataReader ExecuteReader(this IDbCommand self, bool writeLog, string cmd, params object[] values) { if (cmd != null) self.CommandText = cmd; @@ -380,7 +381,7 @@ public static class ExtensionMethods /// The command string to execute. /// The values that the command string should be parameterized with. /// - public static IEnumerable ExecuteReaderEnumerable(this System.Data.IDbCommand self, string cmd, params object[] values) + public static IEnumerable ExecuteReaderEnumerable(this IDbCommand self, string cmd, params object[] values) { using var rd = ExecuteReader(self, cmd, values); while (rd.Read()) @@ -393,7 +394,7 @@ public static class ExtensionMethods /// The data reader to convert the value from. /// The index of the value to convert. /// The value at the given index as a string. - public static string? ConvertValueToString(this System.Data.IDataReader reader, int index) + public static string? ConvertValueToString(this IDataReader reader, int index) { var v = reader.GetValue(index); if (v == null || v == DBNull.Value) @@ -409,7 +410,7 @@ public static class ExtensionMethods /// The index of the value to convert. /// The default value to return if the value is null or cannot be converted. /// The value at the given index as a long. - public static long ConvertValueToInt64(this System.Data.IDataReader reader, int index, long defaultvalue = -1) + public static long ConvertValueToInt64(this IDataReader reader, int index, long defaultvalue = -1) { try { @@ -428,12 +429,27 @@ public static class ExtensionMethods /// /// The connection to create the command on. /// The transaction to use for the command. + /// The command string to create the command with. /// A new command with the given transaction. - public static System.Data.IDbCommand CreateCommand(this System.Data.IDbConnection self, System.Data.IDbTransaction transaction) + public static IDbCommand CreateCommand(this IDbConnection self, IDbTransaction? transaction, string? cmdtext = null) { - var con = self.CreateCommand(); - con.Transaction = transaction; - return con; + var cmd = self.CreateCommand(); + cmd.Transaction = transaction; + if (!string.IsNullOrEmpty(cmdtext)) + { + cmd.CommandText = cmdtext; + cmd.AddParameters(cmdtext.Count(c => c == '?')); + } + return cmd; } + + /// + /// Creates a command with the given command string, and adds parameters to fit the input + /// + /// The connection to create the command on + /// The command string to create the command with + /// The command with the parameters added + public static IDbCommand CreateCommand(this IDbConnection self, string cmdtext) + => CreateCommand(self, null, cmdtext); } diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 15310f2fb..b0cbe9037 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -42,25 +42,25 @@ namespace Duplicati.Library.Main.Database /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(LocalDatabase)); - protected readonly System.Data.IDbConnection m_connection; + protected readonly IDbConnection m_connection; protected readonly long m_operationid = -1; private bool m_hasExecutedVacuum; - private readonly System.Data.IDbCommand m_updateremotevolumeCommand; - private readonly System.Data.IDbCommand m_selectremotevolumesCommand; - private readonly System.Data.IDbCommand m_selectremotevolumeCommand; - private readonly System.Data.IDbCommand m_removeremotevolumeCommand; - private readonly System.Data.IDbCommand m_removedeletedremotevolumeCommand; - private readonly System.Data.IDbCommand m_selectremotevolumeIdCommand; - private readonly System.Data.IDbCommand m_createremotevolumeCommand; - private readonly System.Data.IDbCommand m_selectduplicateRemoteVolumesCommand; + private readonly IDbCommand m_updateremotevolumeCommand; + private readonly IDbCommand m_selectremotevolumesCommand; + private readonly IDbCommand m_selectremotevolumeCommand; + private readonly IDbCommand m_removeremotevolumeCommand; + private readonly IDbCommand m_removedeletedremotevolumeCommand; + private readonly IDbCommand m_selectremotevolumeIdCommand; + private readonly IDbCommand m_createremotevolumeCommand; + private readonly IDbCommand m_selectduplicateRemoteVolumesCommand; - private readonly System.Data.IDbCommand m_insertlogCommand; - private readonly System.Data.IDbCommand m_insertremotelogCommand; - private readonly System.Data.IDbCommand m_insertIndexBlockLink; + private readonly IDbCommand m_insertlogCommand; + private readonly IDbCommand m_insertremotelogCommand; + private readonly IDbCommand m_insertIndexBlockLink; - private readonly System.Data.IDbCommand m_findpathprefixCommand; - private readonly System.Data.IDbCommand m_insertpathprefixCommand; + private readonly IDbCommand m_findpathprefixCommand; + private readonly IDbCommand m_insertpathprefixCommand; protected BasicResults m_result; @@ -69,23 +69,23 @@ namespace Duplicati.Library.Main.Database public DateTime OperationTimestamp { get; private set; } - internal System.Data.IDbConnection Connection { get { return m_connection; } } + internal IDbConnection Connection { get { return m_connection; } } public bool IsDisposed { get; private set; } public bool ShouldCloseConnection { get; set; } - protected static System.Data.IDbConnection CreateConnection(string path) + protected static IDbConnection CreateConnection(string path) { - path = System.IO.Path.GetFullPath(path); - if (!System.IO.Directory.Exists(System.IO.Path.GetDirectoryName(path))) - System.IO.Directory.CreateDirectory(System.IO.Path.GetDirectoryName(path)); + path = Path.GetFullPath(path); + if (!Directory.Exists(Path.GetDirectoryName(path))) + Directory.CreateDirectory(Path.GetDirectoryName(path)); - var c = Duplicati.Library.SQLiteHelper.SQLiteLoader.LoadConnection(path); + var c = SQLiteHelper.SQLiteLoader.LoadConnection(path); try { - Library.SQLiteHelper.DatabaseUpgrader.UpgradeDatabase(c, path, typeof(LocalDatabase)); + SQLiteHelper.DatabaseUpgrader.UpgradeDatabase(c, path, typeof(LocalDatabase)); } catch { @@ -127,23 +127,23 @@ namespace Duplicati.Library.Main.Database public LocalDatabase(LocalDatabase db) : this(db.m_connection) { - this.OperationTimestamp = db.OperationTimestamp; - this.m_connection = db.m_connection; - this.m_operationid = db.m_operationid; - this.m_result = db.m_result; + OperationTimestamp = db.OperationTimestamp; + m_connection = db.m_connection; + m_operationid = db.m_operationid; + m_result = db.m_result; } /// /// Creates a new database instance and starts a new operation /// /// The name of the operation. If null, continues last operation - public LocalDatabase(System.Data.IDbConnection connection, string operation) + public LocalDatabase(IDbConnection connection, string operation) : this(connection) { - this.OperationTimestamp = DateTime.UtcNow; + OperationTimestamp = DateTime.UtcNow; m_connection = connection; - if (m_connection.State != System.Data.ConnectionState.Open) + if (m_connection.State != ConnectionState.Open) m_connection.Open(); if (operation != null) @@ -167,57 +167,21 @@ namespace Duplicati.Library.Main.Database } } - private LocalDatabase(System.Data.IDbConnection connection) + private LocalDatabase(IDbConnection connection) { - m_updateremotevolumeCommand = connection.CreateCommand(); - m_selectremotevolumesCommand = connection.CreateCommand(); - m_selectduplicateRemoteVolumesCommand = connection.CreateCommand(); - m_selectremotevolumeCommand = connection.CreateCommand(); - m_insertlogCommand = connection.CreateCommand(); - m_insertremotelogCommand = connection.CreateCommand(); - m_removeremotevolumeCommand = connection.CreateCommand(); - m_removedeletedremotevolumeCommand = connection.CreateCommand(); - m_selectremotevolumeIdCommand = connection.CreateCommand(); - m_createremotevolumeCommand = connection.CreateCommand(); - m_insertIndexBlockLink = connection.CreateCommand(); - m_findpathprefixCommand = connection.CreateCommand(); - m_insertpathprefixCommand = connection.CreateCommand(); - - m_insertlogCommand.CommandText = @"INSERT INTO ""LogData"" (""OperationID"", ""Timestamp"", ""Type"", ""Message"", ""Exception"") VALUES (?, ?, ?, ?, ?)"; - m_insertlogCommand.AddParameters(5); - - m_insertremotelogCommand.CommandText = @"INSERT INTO ""RemoteOperation"" (""OperationID"", ""Timestamp"", ""Operation"", ""Path"", ""Data"") VALUES (?, ?, ?, ?, ?)"; - m_insertremotelogCommand.AddParameters(5); - - m_updateremotevolumeCommand.CommandText = @"UPDATE ""Remotevolume"" SET ""OperationID"" = ?, ""State"" = ?, ""Hash"" = ?, ""Size"" = ? WHERE ""Name"" = ?"; - m_updateremotevolumeCommand.AddParameters(5); - - m_selectremotevolumesCommand.CommandText = @"SELECT ""ID"", ""Name"", ""Type"", ""Size"", ""Hash"", ""State"", ""DeleteGraceTime"" FROM ""Remotevolume"""; - - m_selectremotevolumeCommand.CommandText = m_selectremotevolumesCommand.CommandText + @" WHERE ""Name"" = ?"; - m_selectduplicateRemoteVolumesCommand.CommandText = FormatInvariant($@"SELECT DISTINCT ""Name"", ""State"" FROM ""Remotevolume"" WHERE ""Name"" IN (SELECT ""Name"" FROM ""Remotevolume"" WHERE ""State"" IN ('{RemoteVolumeState.Deleted.ToString()}', '{RemoteVolumeState.Deleting.ToString()}')) AND NOT ""State"" IN ('{RemoteVolumeState.Deleted.ToString()}', '{RemoteVolumeState.Deleting.ToString()}')"); - - m_selectremotevolumeCommand.AddParameter(); - - m_removeremotevolumeCommand.CommandText = @"DELETE FROM ""Remotevolume"" WHERE ""Name"" = ? AND (""DeleteGraceTime"" < ? OR ""State"" != ?)"; - m_removeremotevolumeCommand.AddParameters(3); - - m_removedeletedremotevolumeCommand.CommandText = FormatInvariant($@"DELETE FROM ""Remotevolume"" WHERE ""State"" == '{RemoteVolumeState.Deleted.ToString()}' AND (""DeleteGraceTime"" < ? OR LENGTH(""DeleteGraceTime"") > 12) "); // >12 is to handle removal of old records that were in ticks - m_removedeletedremotevolumeCommand.AddParameter(); - - m_selectremotevolumeIdCommand.CommandText = @"SELECT ""ID"" FROM ""Remotevolume"" WHERE ""Name"" = ?"; - - m_createremotevolumeCommand.CommandText = @"INSERT INTO ""Remotevolume"" (""OperationID"", ""Name"", ""Type"", ""State"", ""Size"", ""VerificationCount"", ""DeleteGraceTime"") VALUES (?, ?, ?, ?, ?, ?, ?); SELECT last_insert_rowid();"; - m_createremotevolumeCommand.AddParameters(7); - - m_insertIndexBlockLink.CommandText = @"INSERT INTO ""IndexBlockLink"" (""IndexVolumeID"", ""BlockVolumeID"") VALUES (?, ?)"; - m_insertIndexBlockLink.AddParameters(2); - - m_findpathprefixCommand.CommandText = @"SELECT ""ID"" FROM ""PathPrefix"" WHERE ""Prefix"" = ?"; - m_findpathprefixCommand.AddParameter(); - - m_insertpathprefixCommand.CommandText = @"INSERT INTO ""PathPrefix"" (""Prefix"") VALUES (?); SELECT last_insert_rowid(); "; - m_insertpathprefixCommand.AddParameter(); + m_insertlogCommand = connection.CreateCommand(@"INSERT INTO ""LogData"" (""OperationID"", ""Timestamp"", ""Type"", ""Message"", ""Exception"") VALUES (?, ?, ?, ?, ?)"); + m_insertremotelogCommand = connection.CreateCommand(@"INSERT INTO ""RemoteOperation"" (""OperationID"", ""Timestamp"", ""Operation"", ""Path"", ""Data"") VALUES (?, ?, ?, ?, ?)"); + m_updateremotevolumeCommand = connection.CreateCommand(@"UPDATE ""Remotevolume"" SET ""OperationID"" = ?, ""State"" = ?, ""Hash"" = ?, ""Size"" = ? WHERE ""Name"" = ?"); + m_selectremotevolumesCommand = connection.CreateCommand(@"SELECT ""ID"", ""Name"", ""Type"", ""Size"", ""Hash"", ""State"", ""DeleteGraceTime"" FROM ""Remotevolume"""); + m_selectremotevolumeCommand = connection.CreateCommand(m_selectremotevolumesCommand.CommandText + @" WHERE ""Name"" = ?"); + m_selectduplicateRemoteVolumesCommand = connection.CreateCommand(FormatInvariant($@"SELECT DISTINCT ""Name"", ""State"" FROM ""Remotevolume"" WHERE ""Name"" IN (SELECT ""Name"" FROM ""Remotevolume"" WHERE ""State"" IN ('{RemoteVolumeState.Deleted.ToString()}', '{RemoteVolumeState.Deleting.ToString()}')) AND NOT ""State"" IN ('{RemoteVolumeState.Deleted.ToString()}', '{RemoteVolumeState.Deleting.ToString()}')")); + m_removeremotevolumeCommand = connection.CreateCommand(@"DELETE FROM ""Remotevolume"" WHERE ""Name"" = ? AND (""DeleteGraceTime"" < ? OR ""State"" != ?)"); + m_removedeletedremotevolumeCommand = connection.CreateCommand(FormatInvariant($@"DELETE FROM ""Remotevolume"" WHERE ""State"" == '{RemoteVolumeState.Deleted.ToString()}' AND (""DeleteGraceTime"" < ? OR LENGTH(""DeleteGraceTime"") > 12) ")); // >12 is to handle removal of old records that were in ticks + m_selectremotevolumeIdCommand = connection.CreateCommand(@"SELECT ""ID"" FROM ""Remotevolume"" WHERE ""Name"" = ?"); + m_createremotevolumeCommand = connection.CreateCommand(@"INSERT INTO ""Remotevolume"" (""OperationID"", ""Name"", ""Type"", ""State"", ""Size"", ""VerificationCount"", ""DeleteGraceTime"") VALUES (?, ?, ?, ?, ?, ?, ?); SELECT last_insert_rowid();"); + m_insertIndexBlockLink = connection.CreateCommand(@"INSERT INTO ""IndexBlockLink"" (""IndexVolumeID"", ""BlockVolumeID"") VALUES (?, ?)"); + m_findpathprefixCommand = connection.CreateCommand(@"SELECT ""ID"" FROM ""PathPrefix"" WHERE ""Prefix"" = ?"); + m_insertpathprefixCommand = connection.CreateCommand(@"INSERT INTO ""PathPrefix"" (""Prefix"") VALUES (?); SELECT last_insert_rowid(); "); } internal void SetResult(BasicResults result) @@ -233,17 +197,17 @@ namespace Duplicati.Library.Main.Database return Library.Utility.Utility.EPOCH.AddSeconds(seconds); } - public void UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string hash, System.Data.IDbTransaction transaction = null) + public void UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string hash, IDbTransaction transaction = null) { UpdateRemoteVolume(name, state, size, hash, false, transaction); } - public void UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string hash, bool suppressCleanup, System.Data.IDbTransaction transaction = null) + public void UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string hash, bool suppressCleanup, IDbTransaction transaction = null) { UpdateRemoteVolume(name, state, size, hash, suppressCleanup, new TimeSpan(0), transaction); } - public void UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string hash, bool suppressCleanup, TimeSpan deleteGraceTime, System.Data.IDbTransaction transaction = null) + public void UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string hash, bool suppressCleanup, TimeSpan deleteGraceTime, IDbTransaction transaction = null) { m_updateremotevolumeCommand.Transaction = transaction; m_updateremotevolumeCommand.SetParameterValue(0, m_operationid); @@ -292,8 +256,8 @@ namespace Duplicati.Library.Main.Database public Tuple GetFilelistWhereClause(DateTime time, long[] versions, IEnumerable> filesetslist = null, bool singleTimeMatch = false) { - var filesets = (filesetslist ?? this.FilesetTimes).ToArray(); - StringBuilder query = new StringBuilder(); + var filesets = (filesetslist ?? FilesetTimes).ToArray(); + var query = new StringBuilder(); var args = new List(); if (time.Ticks > 0 || (versions != null && versions.Length > 0)) { @@ -311,8 +275,7 @@ namespace Duplicati.Library.Main.Database if (versions != null && versions.Length > 0) { - StringBuilder qs = new StringBuilder(); - + var qs = new StringBuilder(); foreach (var v in versions) { if (v >= 0 && v < filesets.Length) @@ -342,13 +305,13 @@ namespace Duplicati.Library.Main.Database return new Tuple(query.ToString(), args.ToArray()); } - public long GetRemoteVolumeID(string file, System.Data.IDbTransaction transaction = null) + public long GetRemoteVolumeID(string file, IDbTransaction transaction = null) { m_selectremotevolumeIdCommand.Transaction = transaction; return m_selectremotevolumeIdCommand.ExecuteScalarInt64(null, -1, file); } - public IEnumerable> GetRemoteVolumeIDs(IEnumerable files, System.Data.IDbTransaction transaction = null) + public IEnumerable> GetRemoteVolumeIDs(IEnumerable files, IDbTransaction transaction = null) { using (var cmd = m_connection.CreateCommand(transaction)) { @@ -362,7 +325,7 @@ namespace Duplicati.Library.Main.Database } } - public RemoteVolumeEntry GetRemoteVolume(string file, System.Data.IDbTransaction transaction = null) + public RemoteVolumeEntry GetRemoteVolume(string file, IDbTransaction transaction = null) { m_selectremotevolumeCommand.Transaction = transaction; m_selectremotevolumeCommand.SetParameterValue(0, file); @@ -392,7 +355,7 @@ namespace Duplicati.Library.Main.Database } } - public IEnumerable GetRemoteVolumes(System.Data.IDbTransaction transaction = null) + public IEnumerable GetRemoteVolumes(IDbTransaction transaction = null) { m_selectremotevolumesCommand.Transaction = transaction; using (var rd = m_selectremotevolumesCommand.ExecuteReader()) @@ -418,7 +381,7 @@ namespace Duplicati.Library.Main.Database /// The operation performed /// The path involved /// Any data relating to the operation - public void LogRemoteOperation(string operation, string path, string data, System.Data.IDbTransaction transaction) + public void LogRemoteOperation(string operation, string path, string data, IDbTransaction transaction) { m_insertremotelogCommand.Transaction = transaction; m_insertremotelogCommand.SetParameterValue(0, m_operationid); @@ -435,7 +398,7 @@ namespace Duplicati.Library.Main.Database /// The message type /// The message /// An optional exception - public void LogMessage(string type, string message, Exception exception, System.Data.IDbTransaction transaction) + public void LogMessage(string type, string message, Exception exception, IDbTransaction transaction) { m_insertlogCommand.Transaction = transaction; m_insertlogCommand.SetParameterValue(0, m_operationid); @@ -446,12 +409,11 @@ namespace Duplicati.Library.Main.Database m_insertlogCommand.ExecuteNonQuery(); } - public void UnlinkRemoteVolume(string name, RemoteVolumeState state, System.Data.IDbTransaction transaction = null) + public void UnlinkRemoteVolume(string name, RemoteVolumeState state, IDbTransaction transaction = null) { using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(tr.Parent)) { - cmd.Transaction = tr.Parent; var c = cmd.ExecuteNonQuery(@"DELETE FROM ""RemoteVolume"" WHERE ""Name"" = ? AND ""State"" = ? ", name, state.ToString()); if (c != 1) throw new Exception($"Unexpected number of remote volumes deleted: {c}, expected {1}"); @@ -470,10 +432,8 @@ namespace Duplicati.Library.Main.Database if (names == null || !names.Any()) return; using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) - using (var deletecmd = m_connection.CreateCommand()) + using (var deletecmd = m_connection.CreateCommand(tr.Parent)) { - deletecmd.Transaction = tr.Parent; - string temptransguid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); var volidstable = "DelVolSetIds-" + temptransguid; var blocksetidstable = "DelBlockSetIds-" + temptransguid; @@ -555,7 +515,7 @@ AND Fileset.ID NOT IN // Clean up temp tables for subqueries. We truncate content and then try to delete. // Drop in try-block, as it fails in nested transactions (SQLite problem) - // System.Data.SQLite.SQLiteException (0x80004005): database table is locked + // SQLite.SQLiteException (0x80004005): database table is locked deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{blocksetidstable}"" ")); deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{volidstable}"" ")); try @@ -592,17 +552,17 @@ AND Fileset.ID NOT IN return RegisterRemoteVolume(name, type, state, size, new TimeSpan(0), null); } - public long RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, System.Data.IDbTransaction transaction) + public long RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, IDbTransaction transaction) { return RegisterRemoteVolume(name, type, state, new TimeSpan(0), transaction); } - public long RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, TimeSpan deleteGraceTime, System.Data.IDbTransaction transaction) + public long RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, TimeSpan deleteGraceTime, IDbTransaction transaction) { return RegisterRemoteVolume(name, type, state, -1, deleteGraceTime, transaction); } - public long RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, long size, TimeSpan deleteGraceTime, System.Data.IDbTransaction transaction) + public long RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, long size, TimeSpan deleteGraceTime, IDbTransaction transaction) { using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) { @@ -679,31 +639,27 @@ AND Fileset.ID NOT IN public bool IsFilesetFullBackup(DateTime filesetTime) { using (var cmd = m_connection.CreateCommand()) + using (var rd = cmd.ExecuteReader(FormatInvariant($@"SELECT ""IsFullBackup"" FROM ""Fileset"" WHERE ""Timestamp"" = {Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(filesetTime)}"))) { - using (var rd = cmd.ExecuteReader(FormatInvariant($@"SELECT ""IsFullBackup"" FROM ""Fileset"" WHERE ""Timestamp"" = {Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(filesetTime)}"))) - { - if (!rd.Read()) - { - return false; - } - var isFullBackup = rd.GetInt32(0); - return isFullBackup == BackupType.FULL_BACKUP; - } + if (!rd.Read()) + return false; + var isFullBackup = rd.GetInt32(0); + return isFullBackup == BackupType.FULL_BACKUP; } } // TODO: Remove this - public System.Data.IDbTransaction BeginTransaction() + public IDbTransaction BeginTransaction() { return m_connection.BeginTransaction(); } protected class TemporaryTransactionWrapper : IDisposable { - private readonly System.Data.IDbTransaction m_parent; + private readonly IDbTransaction m_parent; private readonly bool m_isTemporary; - public TemporaryTransactionWrapper(System.Data.IDbConnection connection, System.Data.IDbTransaction transaction) + public TemporaryTransactionWrapper(IDbConnection connection, IDbTransaction transaction) { if (transaction != null) { @@ -729,10 +685,10 @@ AND Fileset.ID NOT IN m_parent.Dispose(); } - public System.Data.IDbTransaction Parent { get { return m_parent; } } + public IDbTransaction Parent { get { return m_parent; } } } - private IEnumerable> GetDbOptionList(System.Data.IDbTransaction transaction = null) + private IEnumerable> GetDbOptionList(IDbTransaction transaction = null) { using (var cmd = m_connection.CreateCommand(transaction)) using (var rd = cmd.ExecuteReader(@"SELECT ""Key"", ""Value"" FROM ""Configuration"" ")) @@ -740,7 +696,7 @@ AND Fileset.ID NOT IN yield return new KeyValuePair(rd.GetValue(0).ToString(), rd.GetValue(1).ToString()); } - public IDictionary GetDbOptions(System.Data.IDbTransaction transaction = null) + public IDictionary GetDbOptions(IDbTransaction transaction = null) { return GetDbOptionList(transaction).ToDictionary(x => x.Key, x => x.Value); } @@ -794,12 +750,11 @@ AND Fileset.ID NOT IN /// /// The options to set /// An optional transaction - public void SetDbOptions(IDictionary options, System.Data.IDbTransaction transaction = null) + public void SetDbOptions(IDictionary options, IDbTransaction transaction = null) { using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(tr.Parent)) { - cmd.Transaction = tr.Parent; cmd.ExecuteNonQuery(@"DELETE FROM ""Configuration"" "); foreach (var kp in options) cmd.ExecuteNonQuery(@"INSERT INTO ""Configuration"" (""Key"", ""Value"") VALUES (?, ?) ", kp.Key, kp.Value); @@ -814,7 +769,7 @@ AND Fileset.ID NOT IN return cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""Block"" WHERE ""Size"" > ?", -1, fhblocksize); } - public void VerifyConsistency(long blocksize, long hashsize, bool verifyfilelists, System.Data.IDbTransaction transaction) + public void VerifyConsistency(long blocksize, long hashsize, bool verifyfilelists, IDbTransaction transaction) { using (var cmd = m_connection.CreateCommand(transaction)) { @@ -952,12 +907,12 @@ ON public Block(string hash, long size) { - this.Hash = hash; - this.Size = size; + Hash = hash; + Size = size; } } - public IEnumerable GetBlocks(long volumeid, System.Data.IDbTransaction transaction = null) + public IEnumerable GetBlocks(long volumeid, IDbTransaction transaction = null) { using (var cmd = m_connection.CreateCommand(transaction)) using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""Hash"", ""Size"" FROM ""Block"" WHERE ""VolumeID"" = ?", volumeid)) @@ -969,13 +924,13 @@ ON { private class BlocklistHashEnumerator : IEnumerator { - private readonly System.Data.IDataReader m_reader; + private readonly IDataReader m_reader; private readonly BlocklistHashEnumerable m_parent; private string m_path = null; private bool m_first = true; private string m_current = null; - public BlocklistHashEnumerator(BlocklistHashEnumerable parent, System.Data.IDataReader reader) + public BlocklistHashEnumerator(BlocklistHashEnumerable parent, IDataReader reader) { m_reader = reader; m_parent = parent; @@ -987,7 +942,7 @@ ON { } - object System.Collections.IEnumerator.Current { get { return this.Current; } } + object System.Collections.IEnumerator.Current { get { return Current; } } public bool MoveNext() { @@ -1032,12 +987,12 @@ ON } } - private readonly System.Data.IDataReader m_reader; + private readonly IDataReader m_reader; - public BlocklistHashEnumerable(System.Data.IDataReader reader) + public BlocklistHashEnumerable(IDataReader reader) { m_reader = reader; - this.MoreData = true; + MoreData = true; } public bool MoreData { get; protected set; } @@ -1049,7 +1004,7 @@ ON System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { - return this.GetEnumerator(); + return GetEnumerator(); } } @@ -1181,11 +1136,10 @@ ORDER BY "; - public void WriteFileset(Volumes.FilesetVolumeWriter filesetvolume, long filesetId, System.Data.IDbTransaction transaction) + public void WriteFileset(Volumes.FilesetVolumeWriter filesetvolume, long filesetId, IDbTransaction transaction) { - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(transaction)) { - cmd.Transaction = transaction; cmd.CommandText = LIST_FOLDERS_AND_SYMLINKS; cmd.AddParameter(FOLDER_BLOCKSET_ID); cmd.AddParameter(SYMLINK_BLOCKSET_ID); @@ -1256,22 +1210,20 @@ ORDER BY } } - public void LinkFilesetToVolume(long filesetid, long volumeid, System.Data.IDbTransaction transaction) + public void LinkFilesetToVolume(long filesetid, long volumeid, IDbTransaction transaction) { - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(transaction)) { - cmd.Transaction = transaction; var c = cmd.ExecuteNonQuery(@"UPDATE ""Fileset"" SET ""VolumeID"" = ? WHERE ""ID"" = ?", volumeid, filesetid); if (c != 1) throw new Exception($"Failed to link filesetid {filesetid} to volumeid {volumeid}"); } } - public void PushTimestampChangesToPreviousVersion(long filesetId, System.Data.IDbTransaction transaction) + public void PushTimestampChangesToPreviousVersion(long filesetId, IDbTransaction transaction) { - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(transaction)) { - cmd.Transaction = transaction; var query = @" UPDATE FilesetEntry AS oldVersion SET Lastmodified = tempVersion.Lastmodified @@ -1290,37 +1242,34 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != ? ORDER BY Timest public class FilteredFilenameTable : IDisposable { public string Tablename { get; private set; } - private readonly System.Data.IDbConnection m_connection; + private readonly IDbConnection m_connection; - public FilteredFilenameTable(System.Data.IDbConnection connection, Library.Utility.IFilter filter, System.Data.IDbTransaction transaction) + public FilteredFilenameTable(IDbConnection connection, IFilter filter, IDbTransaction transaction) { m_connection = connection; Tablename = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - var type = Library.Utility.FilterType.Regexp; + var type = FilterType.Regexp; if (filter is FilterExpression expression) type = expression.Type; // Bugfix: SQLite does not handle case-insensitive LIKE with non-ascii characters - if (type != Library.Utility.FilterType.Regexp && !Library.Utility.Utility.IsFSCaseSensitive && filter.ToString().Any(x => x > 127)) - type = Library.Utility.FilterType.Regexp; + if (type != FilterType.Regexp && !Library.Utility.Utility.IsFSCaseSensitive && filter.ToString().Any(x => x > 127)) + type = FilterType.Regexp; if (filter.Empty) { - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(transaction)) { - cmd.Transaction = transaction; cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{Tablename}"" AS SELECT DISTINCT ""Path"" FROM ""File"" ")); return; } } - if (type == Library.Utility.FilterType.Regexp || type == Library.Utility.FilterType.Group) + if (type == FilterType.Regexp || type == FilterType.Group) { - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(transaction)) { // TODO: Optimize this to not rely on the "File" view, and not instantiate the paths in full - - cmd.Transaction = transaction; cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{Tablename}"" (""Path"" TEXT NOT NULL)")); using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) { @@ -1332,7 +1281,7 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != ? ORDER BY Timest while (rd.Read()) { var p = rd.GetValue(0).ToString(); - if (Library.Utility.FilterExpression.Matches(filter, p)) + if (FilterExpression.Matches(filter, p)) { cmd.SetParameterValue(0, p); cmd.ExecuteNonQuery(); @@ -1348,7 +1297,7 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != ? ORDER BY Timest { var sb = new StringBuilder(); var args = new List(); - foreach (var f in ((Library.Utility.FilterExpression)filter).GetSimpleList()) + foreach (var f in ((FilterExpression)filter).GetSimpleList()) { if (type == FilterType.Wildcard) { @@ -1364,10 +1313,9 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != ? ORDER BY Timest sb.Length = sb.Length - " OR ".Length; - using (var cmd = m_connection.CreateCommand()) using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) + using (var cmd = m_connection.CreateCommand(tr.Parent)) { - cmd.Transaction = tr.Parent; cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{Tablename}"" (""Path"" TEXT NOT NULL)")); cmd.ExecuteNonQuery(FormatInvariant($@"INSERT INTO ""{Tablename}"" SELECT DISTINCT ""Path"" FROM ""File"" WHERE {sb}"), args.ToArray()); tr.Commit(); @@ -1388,13 +1336,11 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != ? ORDER BY Timest } } - public void RenameRemoteFile(string oldname, string newname, System.Data.IDbTransaction transaction) + public void RenameRemoteFile(string oldname, string newname, IDbTransaction transaction) { using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(tr.Parent)) { - cmd.Transaction = tr.Parent; - //Rename the old entry, to preserve ID links var c = cmd.ExecuteNonQuery(@"UPDATE ""Remotevolume"" SET ""Name"" = ? WHERE ""Name"" = ?", newname, oldname); if (c != 1) @@ -1417,19 +1363,18 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != ? ORDER BY Timest /// The ID of the fileset volume to update /// The timestamp of the operation to create /// An optional external transaction - public virtual long CreateFileset(long volumeid, DateTime timestamp, System.Data.IDbTransaction transaction = null) + public virtual long CreateFileset(long volumeid, DateTime timestamp, IDbTransaction transaction = null) { - using (var cmd = m_connection.CreateCommand()) using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) + using (var cmd = m_connection.CreateCommand(tr.Parent)) { - cmd.Transaction = tr.Parent; var id = cmd.ExecuteScalarInt64(@"INSERT INTO ""Fileset"" (""OperationID"", ""Timestamp"", ""VolumeID"", ""IsFullBackup"") VALUES (?, ?, ?, ?); SELECT last_insert_rowid();", -1, m_operationid, Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(timestamp), volumeid, BackupType.PARTIAL_BACKUP); tr.Commit(); return id; } } - public void AddIndexBlockLink(long indexVolumeID, long blockVolumeID, System.Data.IDbTransaction transaction) + public void AddIndexBlockLink(long indexVolumeID, long blockVolumeID, IDbTransaction transaction) { m_insertIndexBlockLink.Transaction = transaction; m_insertIndexBlockLink.SetParameterValue(0, indexVolumeID); @@ -1445,7 +1390,7 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != ? ORDER BY Timest /// The size of the hash /// An optional external transaction /// An enumerable of tuples containing the blocklist hash, the blocklist data and the length of the data - public IEnumerable> GetBlocklists(long volumeid, long blocksize, int hashsize, System.Data.IDbTransaction transaction = null) + public IEnumerable> GetBlocklists(long volumeid, long blocksize, int hashsize, IDbTransaction transaction = null) { using (var cmd = m_connection.CreateCommand(transaction)) { @@ -1491,13 +1436,9 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != ? ORDER BY Timest public void UpdateFullBackupStateInFileset(long fileSetId, bool isFullBackup, IDbTransaction transaction = null) { using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) + using (var cmd = m_connection.CreateCommand(tr.Parent)) { - using (var cmd = m_connection.CreateCommand()) - { - cmd.Transaction = tr.Parent; - cmd.ExecuteNonQuery(@"UPDATE ""Fileset"" SET ""IsFullBackup"" = ? WHERE ""ID"" = ?;", isFullBackup, fileSetId); - } - + cmd.ExecuteNonQuery(@"UPDATE ""Fileset"" SET ""IsFullBackup"" = ? WHERE ""ID"" = ?;", isFullBackup, fileSetId); tr.Commit(); } } @@ -1524,16 +1465,14 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != ? ORDER BY Timest public IEnumerable> GetIncompleteFilesets(IDbTransaction transaction) { using (var cmd = m_connection.CreateCommand(transaction)) - { - using (var rd = cmd.ExecuteReader(FormatInvariant(@$"SELECT DISTINCT ""Fileset"".""ID"", ""Fileset"".""Timestamp"" FROM ""Fileset"", ""RemoteVolume"" WHERE ""RemoteVolume"".""ID"" = ""Fileset"".""VolumeID"" AND ""Fileset"".""ID"" IN (SELECT ""FilesetID"" FROM ""FilesetEntry"") AND (""RemoteVolume"".""State"" = '{RemoteVolumeState.Uploading}' OR ""RemoteVolume"".""State"" = '{RemoteVolumeState.Temporary}')"))) - while (rd.Read()) - { - yield return new KeyValuePair( - rd.GetInt64(0), - ParseFromEpochSeconds(rd.GetInt64(1)).ToLocalTime() - ); - } - } + using (var rd = cmd.ExecuteReader(FormatInvariant(@$"SELECT DISTINCT ""Fileset"".""ID"", ""Fileset"".""Timestamp"" FROM ""Fileset"", ""RemoteVolume"" WHERE ""RemoteVolume"".""ID"" = ""Fileset"".""VolumeID"" AND ""Fileset"".""ID"" IN (SELECT ""FilesetID"" FROM ""FilesetEntry"") AND (""RemoteVolume"".""State"" = '{RemoteVolumeState.Uploading}' OR ""RemoteVolume"".""State"" = '{RemoteVolumeState.Temporary}')"))) + while (rd.Read()) + { + yield return new KeyValuePair( + rd.GetInt64(0), + ParseFromEpochSeconds(rd.GetInt64(1)).ToLocalTime() + ); + } } /// @@ -1557,7 +1496,7 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != ? ORDER BY Timest ParseFromEpochSeconds(rd.GetInt64(6)).ToLocalTime() ); else - return default(RemoteVolumeEntry); + return default; } public void PurgeLogData(DateTime threshold) @@ -1590,11 +1529,11 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != ? ORDER BY Timest if (IsDisposed) return; - DisposeAllFields(this, false); + DisposeAllFields(this, false); if (ShouldCloseConnection && m_connection != null) { - if (m_connection.State == System.Data.ConnectionState.Open && !m_hasExecutedVacuum) + if (m_connection.State == ConnectionState.Open && !m_hasExecutedVacuum) { using (var transaction = m_connection.BeginTransaction()) using (var command = m_connection.CreateCommand(transaction)) @@ -1606,7 +1545,7 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != ? ORDER BY Timest { transaction.Commit(); } - catch (System.Data.SQLite.SQLiteException ex) + catch (SQLite.SQLiteException ex) { Logging.Log.WriteVerboseMessage(LOGTAG, "FailedToCommitTransaction", ex, "Failed to commit transaction after pragma optimize, usually caused by the a no-op transaction"); } @@ -1697,7 +1636,7 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != ? ORDER BY Timest /// The path prefix ID. /// The path to get the prefix for. /// The transaction to use for insertion, or null for no transaction - public long GetOrCreatePathPrefix(string prefix, System.Data.IDbTransaction transaction) + public long GetOrCreatePathPrefix(string prefix, IDbTransaction transaction) { // Ring-buffer style lookup for (var i = 0; i < m_pathPrefixLookup.Length; i++) @@ -1727,8 +1666,8 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != ? ORDER BY Timest /// The path separators on this system /// private static readonly char[] _pathseparators = new char[] { - System.IO.Path.DirectorySeparatorChar, - System.IO.Path.AltDirectorySeparatorChar, + Path.DirectorySeparatorChar, + Path.AltDirectorySeparatorChar, }; /// diff --git a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs index 0e8910e55..3bbcd8f68 100644 --- a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs @@ -39,7 +39,7 @@ namespace Duplicati.Library.Main.Database /// private const string TEMPORARY = "TEMPORARY"; - private System.Data.IDbCommand m_registerDuplicateBlockCommand; + private IDbCommand m_registerDuplicateBlockCommand; public LocalDeleteDatabase(string path, string operation) : base(path, operation, true) @@ -55,9 +55,7 @@ namespace Duplicati.Library.Main.Database private void InitializeCommands() { - m_registerDuplicateBlockCommand = m_connection.CreateCommand(); - m_registerDuplicateBlockCommand.CommandText = @"INSERT OR IGNORE INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"") SELECT ""ID"", ? FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ? "; - m_registerDuplicateBlockCommand.AddParameters(3); + m_registerDuplicateBlockCommand = m_connection.CreateCommand(@"INSERT OR IGNORE INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"") SELECT ""ID"", ? FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ? "); } /// @@ -66,22 +64,20 @@ namespace Duplicati.Library.Main.Database /// The fileset entries to delete /// The transaction to execute the commands in /// A list of filesets to delete - public IEnumerable> DropFilesetsFromTable(DateTime[] toDelete, System.Data.IDbTransaction transaction) + public IEnumerable> DropFilesetsFromTable(DateTime[] toDelete, IDbTransaction transaction) { - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(transaction)) { - cmd.Transaction = transaction; - var deleted = 0; //Process array in slices to prevent exceeding SQLITE_MAX_VARIABLE_NUMBER (default 999) const int SLICE_SIZE = 128; - for (int sliceStart = 0; sliceStart < toDelete.Length; sliceStart += SLICE_SIZE) + for (var sliceStart = 0; sliceStart < toDelete.Length; sliceStart += SLICE_SIZE) { - int sliceEnd = Math.Min(toDelete.Length, sliceStart + SLICE_SIZE) - 1; - int sliceLen = sliceEnd - sliceStart + 1; + var sliceEnd = Math.Min(toDelete.Length, sliceStart + SLICE_SIZE) - 1; + var sliceLen = sliceEnd - sliceStart + 1; - string q = string.Join(",", Enumerable.Repeat("?", sliceLen)); + var q = string.Join(",", Enumerable.Repeat("?", sliceLen)); //First we remove unwanted entries deleted += cmd.ExecuteNonQuery(@"DELETE FROM ""Fileset"" WHERE ""Timestamp"" IN (" + q + @") ", toDelete.Skip(sliceStart).Take(sliceLen).Select(Library.Utility.Utility.NormalizeDateTimeToEpochSeconds).Cast().ToArray()); @@ -110,12 +106,8 @@ namespace Duplicati.Library.Main.Database throw new Exception($"Unexpected number of remote volumes marked as deleted. Found {deleted} filesets, but {updated} volumes"); using (var rd = cmd.ExecuteReader(@"SELECT ""Name"", ""Size"" FROM ""RemoteVolume"" WHERE ""Type"" = ? AND ""State"" = ? ", RemoteVolumeType.Files.ToString(), RemoteVolumeState.Deleting.ToString())) - { while (rd.Read()) - { yield return new KeyValuePair(rd.GetString(0), rd.ConvertValueToInt64(1)); - } - } } } @@ -128,8 +120,8 @@ namespace Duplicati.Library.Main.Database { get { - List filesets = new List(); - using (IDbCommand cmd = this.m_connection.CreateCommand()) + var filesets = new List(); + using (var cmd = m_connection.CreateCommand()) { // We can also use the ROW_NUMBER() window function to generate the backup versions, // but this requires at least SQLite 3.25, which is not available in some common @@ -157,10 +149,10 @@ namespace Duplicati.Library.Main.Database public VolumeUsage(string name, long datasize, long wastedsize, long compressedsize) { - this.Name = name; - this.DataSize = datasize; - this.WastedSize = wastedsize; - this.CompressedSize = compressedsize; + Name = name; + DataSize = datasize; + WastedSize = wastedsize; + CompressedSize = compressedsize; } } @@ -170,7 +162,7 @@ namespace Duplicati.Library.Main.Database /// The sizes are the uncompressed values. /// /// A list of tuples with name, datasize, wastedbytes. - private IEnumerable GetWastedSpaceReport(System.Data.IDbTransaction transaction) + private IEnumerable GetWastedSpaceReport(IDbTransaction transaction) { var tmptablename = "UsageReport-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); @@ -187,9 +179,8 @@ namespace Duplicati.Library.Main.Database var collected = @"SELECT ""VolumeID"" AS ""VolumeID"", SUM(""ActiveSize"") AS ""ActiveSize"", SUM(""InactiveSize"") AS ""InactiveSize"", MAX(""Sorttime"") AS ""Sorttime"" FROM (" + combined + @") GROUP BY ""VolumeID"" "; var createtable = FormatInvariant($"{@$"CREATE {TEMPORARY} TABLE ""{tmptablename}"" AS "}{collected}"); - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(transaction)) { - cmd.Transaction = transaction; try { cmd.ExecuteNonQuery(createtable, RemoteVolumeType.Blocks.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()); @@ -303,7 +294,7 @@ namespace Duplicati.Library.Main.Database } } - public ICompactReport GetCompactReport(long volsize, long wastethreshold, long smallfilesize, long maxsmallfilecount, System.Data.IDbTransaction transaction) + public ICompactReport GetCompactReport(long volsize, long wastethreshold, long smallfilesize, long maxsmallfilecount, IDbTransaction transaction) { return new CompactReport(volsize, wastethreshold, smallfilesize, maxsmallfilecount, GetWastedSpaceReport(transaction).ToList()); } @@ -311,24 +302,19 @@ namespace Duplicati.Library.Main.Database public interface IBlockQuery : IDisposable { - bool UseBlock(string hash, long size, System.Data.IDbTransaction transaction); + bool UseBlock(string hash, long size, IDbTransaction transaction); } private class BlockQuery : IBlockQuery { - private System.Data.IDbCommand m_command; + private IDbCommand m_command; - public BlockQuery(System.Data.IDbConnection con, System.Data.IDbTransaction transaction) + public BlockQuery(IDbConnection con, IDbTransaction transaction) { - m_command = con.CreateCommand(); - m_command.Transaction = transaction; - - m_command.Parameters.Clear(); - m_command.CommandText = @"SELECT ""VolumeID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ? "; - m_command.AddParameters(2); + m_command = con.CreateCommand(transaction, @"SELECT ""VolumeID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ? "); } - public bool UseBlock(string hash, long size, System.Data.IDbTransaction transaction) + public bool UseBlock(string hash, long size, IDbTransaction transaction) { m_command.Transaction = transaction; m_command.SetParameterValue(0, hash); @@ -348,7 +334,7 @@ namespace Duplicati.Library.Main.Database /// /// Builds a lookup table to enable faster response to block queries /// - public IBlockQuery CreateBlockQueryHelper(System.Data.IDbTransaction transaction) + public IBlockQuery CreateBlockQueryHelper(IDbTransaction transaction) { return new BlockQuery(m_connection, transaction); } @@ -360,7 +346,7 @@ namespace Duplicati.Library.Main.Database /// The size of the block /// The new volume ID /// The transaction to execute the command in - public void RegisterDuplicatedBlock(string hash, long size, long volumeID, System.Data.IDbTransaction tr) + public void RegisterDuplicatedBlock(string hash, long size, long volumeID, IDbTransaction tr) { m_registerDuplicateBlockCommand.Transaction = tr; m_registerDuplicateBlockCommand.SetParameterValue(0, volumeID); @@ -376,7 +362,7 @@ namespace Duplicati.Library.Main.Database /// The file to remove /// The volume IDs that will be removed /// The transaction to execute the command in - public void PrepareForDelete(string filename, IEnumerable volumeIdsToBeRemoved, System.Data.IDbTransaction transaction) + public void PrepareForDelete(string filename, IEnumerable volumeIdsToBeRemoved, IDbTransaction transaction) { var deletedVolume = GetRemoteVolume(filename, transaction); if (deletedVolume.Type != RemoteVolumeType.Blocks) @@ -420,7 +406,7 @@ namespace Duplicati.Library.Main.Database /// Block volumes slated for deletion. /// The transaction to execute the command in /// The deletable volumes. - public IEnumerable ReOrderDeleteableVolumes(IEnumerable deleteableVolumes, System.Data.IDbTransaction transaction) + public IEnumerable ReOrderDeleteableVolumes(IEnumerable deleteableVolumes, IDbTransaction transaction) { using (var cmd = m_connection.CreateCommand(transaction)) { diff --git a/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs b/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs index c9ed24200..7a3dd5448 100644 --- a/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs @@ -33,7 +33,7 @@ namespace Duplicati.Library.Main.Database ShouldCloseConnection = true; } - private class ListResultFileset : Duplicati.Library.Interface.IListResultFileset + private class ListResultFileset : Interface.IListResultFileset { public long Version { get; set; } public int IsFullBackup { get; set; } @@ -42,24 +42,24 @@ namespace Duplicati.Library.Main.Database public long FileSizes { get; set; } } - private class ListResultFile : Duplicati.Library.Interface.IListResultFile + private class ListResultFile : Interface.IListResultFile { public string Path { get; set; } public IEnumerable Sizes { get; set; } } - private class ListResultRemoteLog : Duplicati.Library.Interface.IListResultRemoteLog + private class ListResultRemoteLog : Interface.IListResultRemoteLog { public DateTime Timestamp { get; set; } public string Message { get; set; } } - private class ListResultRemoteVolume : Duplicati.Library.Interface.IListResultRemoteVolume + private class ListResultRemoteVolume : Interface.IListResultRemoteVolume { public string Name { get; set; } } - public IEnumerable GetFilesets(IEnumerable items) + public IEnumerable GetFilesets(IEnumerable items) { var filesets = FilesetTimes.ToArray(); var dict = new Dictionary(); @@ -88,7 +88,7 @@ SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""Remo } } - public IEnumerable GetFiles(IEnumerable items) + public IEnumerable GetFiles(IEnumerable items) { var sql = FormatInvariant($@"SELECT DISTINCT ""Path"" FROM ( SELECT ""Path"" FROM ""File"" WHERE ""BlocksetID"" IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN (SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" IN (SELECT ""ID"" from ""RemoteVolume"" WHERE ""Name"" IN ({string.Join(",", items.Select(x => "?"))})))) @@ -112,7 +112,7 @@ SELECT ""Path"" FROM ""File"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""Fileset } - public IEnumerable GetLogLines(IEnumerable items) + public IEnumerable GetLogLines(IEnumerable items) { var sql = FormatInvariant($@" SELECT ""TimeStamp"", ""Message"" || ' ' || CASE WHEN ""Exception"" IS NULL THEN '' ELSE ""Exception"" END FROM ""LogData"" WHERE {string.Join(" OR ", items.Select(x => @"""Message"" LIKE ?"))} @@ -132,7 +132,7 @@ SELECT ""Timestamp"", ""Data"" FROM ""RemoteOperation"" WHERE ""Path"" IN ({stri }; } - public IEnumerable GetVolumes(IEnumerable items) + public IEnumerable GetVolumes(IEnumerable items) { var sql = FormatInvariant($@"SELECT DISTINCT ""Name"" FROM ( SELECT ""Name"" FROM ""Remotevolume"" WHERE ""ID"" IN ( SELECT ""VolumeID"" FROM ""Block"" WHERE ""ID"" IN ( SELECT ""BlockID"" FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""FileLookup"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" IN ( SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN ({string.Join(",", items.Select(x => "?"))}))))))) diff --git a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs index 75dd81aae..e85232092 100644 --- a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs @@ -21,6 +21,7 @@ using System; using System.Collections.Generic; +using System.Data; using System.Linq; namespace Duplicati.Library.Main.Database @@ -76,7 +77,7 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN ShouldCloseConnection = false; } - public IEnumerable> GetBrokenFilesets(DateTime time, long[] versions, System.Data.IDbTransaction transaction) + public IEnumerable> GetBrokenFilesets(DateTime time, long[] versions, IDbTransaction transaction) { var query = BROKEN_FILE_SETS; var clause = GetFilelistWhereClause(time, versions); @@ -91,7 +92,7 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN yield return new Tuple(ParseFromEpochSeconds(rd.ConvertValueToInt64(0, 0)), rd.ConvertValueToInt64(1, -1), rd.ConvertValueToInt64(2, 0)); } - public IEnumerable> GetBrokenFilenames(long filesetid, System.Data.IDbTransaction transaction) + public IEnumerable> GetBrokenFilenames(long filesetid, IDbTransaction transaction) { using (var cmd = Connection.CreateCommand(transaction)) foreach (var rd in cmd.ExecuteReaderEnumerable(BROKEN_FILE_NAMES, filesetid)) @@ -99,13 +100,13 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN yield return new Tuple(rd.ConvertValueToString(0), rd.ConvertValueToInt64(1)); } - public void InsertBrokenFileIDsIntoTable(long filesetid, string tablename, string IDfieldname, System.Data.IDbTransaction transaction) + public void InsertBrokenFileIDsIntoTable(long filesetid, string tablename, string IDfieldname, IDbTransaction transaction) { using (var cmd = Connection.CreateCommand(transaction)) cmd.ExecuteNonQuery(INSERT_BROKEN_IDS(tablename, IDfieldname), filesetid); } - public void RemoveMissingBlocks(IEnumerable names, System.Data.IDbTransaction transaction) + public void RemoveMissingBlocks(IEnumerable names, IDbTransaction transaction) { if (names == null || !names.Any()) return; if (transaction == null) @@ -113,7 +114,7 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN using (var deletecmd = m_connection.CreateCommand(transaction)) { - string temptransguid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + var temptransguid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); var volidstable = "DelVolSetIds-" + temptransguid; // Create and fill a temp table with the volids to delete. We avoid using too many parameters that way. @@ -137,7 +138,7 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN // Clean up temp tables for subqueries. We truncate content and then try to delete. // Drop in try-block, as it fails in nested transactions (SQLite problem) - // System.Data.SQLite.SQLiteException (0x80004005): database table is locked + // SQLite.SQLiteException (0x80004005): database table is locked deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{volidstable}"" ")); try { @@ -148,7 +149,7 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN } } - public long GetFilesetFileCount(long filesetid, System.Data.IDbTransaction transaction) + public long GetFilesetFileCount(long filesetid, IDbTransaction transaction) { using (var cmd = m_connection.CreateCommand(transaction)) return cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetID"" = ?", 0, filesetid); diff --git a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs index f58c856dc..fda6f663d 100644 --- a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs @@ -20,6 +20,7 @@ // DEALINGS IN THE SOFTWARE. using System; +using System.Data; using System.Collections.Generic; using Duplicati.Library.Utility; @@ -35,13 +36,13 @@ namespace Duplicati.Library.Main.Database public interface IStorageHelper : IDisposable { - void AddElement(string path, string filehash, string metahash, long size, Library.Interface.ListChangesElementType type, bool asNew); + void AddElement(string path, string filehash, string metahash, long size, Interface.ListChangesElementType type, bool asNew); - void AddFromDb(long filesetId, bool asNew, Library.Utility.IFilter filter); + void AddFromDb(long filesetId, bool asNew, IFilter filter); IChangeCountReport CreateChangeCountReport(); IChangeSizeReport CreateChangeSizeReport(); - IEnumerable> CreateChangedFileReport(); + IEnumerable> CreateChangedFileReport(); } public interface IChangeCountReport @@ -94,16 +95,16 @@ namespace Duplicati.Library.Main.Database private class StorageHelper : IStorageHelper { - private readonly System.Data.IDbConnection m_connection; - private System.Data.IDbTransaction m_transaction; + private readonly IDbConnection m_connection; + private IDbTransaction m_transaction; - private System.Data.IDbCommand m_insertPreviousElementCommand; - private System.Data.IDbCommand m_insertCurrentElementCommand; + private IDbCommand m_insertPreviousElementCommand; + private IDbCommand m_insertCurrentElementCommand; private string m_previousTable; private string m_currentTable; - public StorageHelper(System.Data.IDbConnection con) + public StorageHelper(IDbConnection con) { m_connection = con; m_previousTable = "Previous-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); @@ -111,38 +112,28 @@ namespace Duplicati.Library.Main.Database m_transaction = m_connection.BeginTransaction(); - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(m_transaction)) { - cmd.Transaction = m_transaction; - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_previousTable}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) ")); cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_currentTable}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) ")); } - m_insertPreviousElementCommand = m_connection.CreateCommand(); - m_insertPreviousElementCommand.Transaction = m_transaction; - m_insertPreviousElementCommand.CommandText = FormatInvariant($@"INSERT INTO ""{m_previousTable}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (?,?,?,?,?)"); - m_insertPreviousElementCommand.AddParameters(5); - - m_insertCurrentElementCommand = m_connection.CreateCommand(); - m_insertCurrentElementCommand.Transaction = m_transaction; - m_insertCurrentElementCommand.CommandText = FormatInvariant($@"INSERT INTO ""{m_currentTable}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (?,?,?,?,?)"); - m_insertCurrentElementCommand.AddParameters(5); + m_insertPreviousElementCommand = m_connection.CreateCommand(m_transaction, FormatInvariant($@"INSERT INTO ""{m_previousTable}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (?,?,?,?,?)")); + m_insertCurrentElementCommand = m_connection.CreateCommand(FormatInvariant($@"INSERT INTO ""{m_currentTable}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (?,?,?,?,?)")); } - public void AddFromDb(long filesetId, bool asNew, Library.Utility.IFilter filter) + public void AddFromDb(long filesetId, bool asNew, IFilter filter) { var tablename = asNew ? m_currentTable : m_previousTable; - var folders = FormatInvariant($@"SELECT ""File"".""Path"" AS ""Path"", NULL AS ""FileHash"", ""Blockset"".""Fullhash"" AS ""MetaHash"", -1 AS ""Size"", {(int)Library.Interface.ListChangesElementType.Folder} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" = -100 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" "); - var symlinks = FormatInvariant($@"SELECT ""File"".""Path"" AS ""Path"", NULL AS ""FileHash"", ""Blockset"".""Fullhash"" AS ""MetaHash"", -1 AS ""Size"", {(int)Library.Interface.ListChangesElementType.Symlink} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" = -200 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" "); - var files = FormatInvariant($@"SELECT ""File"".""Path"" AS ""Path"", ""FB"".""FullHash"" AS ""FileHash"", ""MB"".""Fullhash"" AS ""MetaHash"", ""FB"".""Length"" AS ""Size"", {(int)Library.Interface.ListChangesElementType.File} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" MB, ""Blockset"" FB WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" >= 0 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""MB"".""ID"" AND ""File"".""BlocksetID"" = ""FB"".""ID"" "); + var folders = FormatInvariant($@"SELECT ""File"".""Path"" AS ""Path"", NULL AS ""FileHash"", ""Blockset"".""Fullhash"" AS ""MetaHash"", -1 AS ""Size"", {(int)Interface.ListChangesElementType.Folder} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" = -100 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" "); + var symlinks = FormatInvariant($@"SELECT ""File"".""Path"" AS ""Path"", NULL AS ""FileHash"", ""Blockset"".""Fullhash"" AS ""MetaHash"", -1 AS ""Size"", {(int)Interface.ListChangesElementType.Symlink} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" = -200 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" "); + var files = FormatInvariant($@"SELECT ""File"".""Path"" AS ""Path"", ""FB"".""FullHash"" AS ""FileHash"", ""MB"".""Fullhash"" AS ""MetaHash"", ""FB"".""Length"" AS ""Size"", {(int)Interface.ListChangesElementType.File} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" MB, ""Blockset"" FB WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" >= 0 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""MB"".""ID"" AND ""File"".""BlocksetID"" = ""FB"".""ID"" "); var combined = "(" + folders + " UNION " + symlinks + " UNION " + files + ")"; - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(m_transaction)) { - cmd.Transaction = m_transaction; if (filter == null || filter.Empty) { // Simple case, select everything @@ -182,33 +173,27 @@ namespace Duplicati.Library.Main.Database else { // Do row-wise iteration - object[] values = new object[5]; - using (var cmd2 = m_connection.CreateCommand()) - { - cmd2.CommandText = FormatInvariant($@"INSERT INTO ""{tablename}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (?,?,?,?,?)"); - cmd2.AddParameters(5); - cmd2.Transaction = m_transaction; - - using (var rd = cmd.ExecuteReader(FormatInvariant($@"SELECT ""A"".""Path"", ""A"".""FileHash"", ""A"".""MetaHash"", ""A"".""Size"", ""A"".""Type"" FROM {combined} A WHERE ""A"".""FilesetID"" = ?"), filesetId)) - while (rd.Read()) + var values = new object[5]; + using (var cmd2 = m_connection.CreateCommand(m_transaction, FormatInvariant($@"INSERT INTO ""{tablename}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (?,?,?,?,?)"))) + using (var rd = cmd.ExecuteReader(FormatInvariant($@"SELECT ""A"".""Path"", ""A"".""FileHash"", ""A"".""MetaHash"", ""A"".""Size"", ""A"".""Type"" FROM {combined} A WHERE ""A"".""FilesetID"" = ?"), filesetId)) + while (rd.Read()) + { + rd.GetValues(values); + if (values[0] != null && values[0] != DBNull.Value && FilterExpression.Matches(filter, values[0].ToString())) { - rd.GetValues(values); - if (values[0] != null && values[0] != DBNull.Value && Library.Utility.FilterExpression.Matches(filter, values[0].ToString())) - { - cmd2.SetParameterValue(0, values[0]); - cmd2.SetParameterValue(1, values[1]); - cmd2.SetParameterValue(2, values[2]); - cmd2.SetParameterValue(3, values[3]); - cmd2.SetParameterValue(4, values[4]); - cmd2.ExecuteNonQuery(); - } + cmd2.SetParameterValue(0, values[0]); + cmd2.SetParameterValue(1, values[1]); + cmd2.SetParameterValue(2, values[2]); + cmd2.SetParameterValue(3, values[3]); + cmd2.SetParameterValue(4, values[4]); + cmd2.ExecuteNonQuery(); } - } + } } } } - public void AddElement(string path, string filehash, string metahash, long size, Library.Interface.ListChangesElementType type, bool asNew) + public void AddElement(string path, string filehash, string metahash, long size, Interface.ListChangesElementType type, bool asNew) { var cmd = asNew ? m_insertCurrentElementCommand : m_insertPreviousElementCommand; cmd.SetParameterValue(0, path); @@ -219,7 +204,7 @@ namespace Duplicati.Library.Main.Database cmd.ExecuteNonQuery(); } - private static IEnumerable ReaderToStringList(System.Data.IDataReader rd) + private static IEnumerable ReaderToStringList(IDataReader rd) { using (rd) while (rd.Read()) @@ -248,10 +233,8 @@ namespace Duplicati.Library.Main.Database var deleted = sqls.Item2; //var modified = sqls.Item3; - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(m_transaction)) { - cmd.Transaction = m_transaction; - var result = new ChangeSizeReport(); result.PreviousSize = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT SUM(""Size"") FROM ""{m_previousTable}"" "), 0); @@ -271,57 +254,54 @@ namespace Duplicati.Library.Main.Database var deleted = @"SELECT COUNT(*) FROM (" + sqls.Item2 + ")"; var modified = @"SELECT COUNT(*) FROM (" + sqls.Item3 + ")"; - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(m_transaction)) { - cmd.Transaction = m_transaction; - var result = new ChangeCountReport(); - result.AddedFolders = cmd.ExecuteScalarInt64(added, 0, (int)Library.Interface.ListChangesElementType.Folder); - result.AddedSymlinks = cmd.ExecuteScalarInt64(added, 0, (int)Library.Interface.ListChangesElementType.Symlink); - result.AddedFiles = cmd.ExecuteScalarInt64(added, 0, (int)Library.Interface.ListChangesElementType.File); + result.AddedFolders = cmd.ExecuteScalarInt64(added, 0, (int)Interface.ListChangesElementType.Folder); + result.AddedSymlinks = cmd.ExecuteScalarInt64(added, 0, (int)Interface.ListChangesElementType.Symlink); + result.AddedFiles = cmd.ExecuteScalarInt64(added, 0, (int)Interface.ListChangesElementType.File); - result.DeletedFolders = cmd.ExecuteScalarInt64(deleted, 0, (int)Library.Interface.ListChangesElementType.Folder); - result.DeletedSymlinks = cmd.ExecuteScalarInt64(deleted, 0, (int)Library.Interface.ListChangesElementType.Symlink); - result.DeletedFiles = cmd.ExecuteScalarInt64(deleted, 0, (int)Library.Interface.ListChangesElementType.File); + result.DeletedFolders = cmd.ExecuteScalarInt64(deleted, 0, (int)Interface.ListChangesElementType.Folder); + result.DeletedSymlinks = cmd.ExecuteScalarInt64(deleted, 0, (int)Interface.ListChangesElementType.Symlink); + result.DeletedFiles = cmd.ExecuteScalarInt64(deleted, 0, (int)Interface.ListChangesElementType.File); - result.ModifiedFolders = cmd.ExecuteScalarInt64(modified, 0, (int)Library.Interface.ListChangesElementType.Folder); - result.ModifiedSymlinks = cmd.ExecuteScalarInt64(modified, 0, (int)Library.Interface.ListChangesElementType.Symlink); - result.ModifiedFiles = cmd.ExecuteScalarInt64(modified, 0, (int)Library.Interface.ListChangesElementType.File); + result.ModifiedFolders = cmd.ExecuteScalarInt64(modified, 0, (int)Interface.ListChangesElementType.Folder); + result.ModifiedSymlinks = cmd.ExecuteScalarInt64(modified, 0, (int)Interface.ListChangesElementType.Symlink); + result.ModifiedFiles = cmd.ExecuteScalarInt64(modified, 0, (int)Interface.ListChangesElementType.File); return result; } } - public IEnumerable> CreateChangedFileReport() + public IEnumerable> CreateChangedFileReport() { var sqls = GetSqls(false); var added = sqls.Item1; var deleted = sqls.Item2; var modified = sqls.Item3; - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(m_transaction)) { - cmd.Transaction = m_transaction; - foreach (var s in ReaderToStringList(cmd.ExecuteReader(added, (int)Library.Interface.ListChangesElementType.Folder))) - yield return new Tuple(Library.Interface.ListChangesChangeType.Added, Library.Interface.ListChangesElementType.Folder, s); - foreach (var s in ReaderToStringList(cmd.ExecuteReader(added, (int)Library.Interface.ListChangesElementType.Symlink))) - yield return new Tuple(Library.Interface.ListChangesChangeType.Added, Library.Interface.ListChangesElementType.Symlink, s); - foreach (var s in ReaderToStringList(cmd.ExecuteReader(added, (int)Library.Interface.ListChangesElementType.File))) - yield return new Tuple(Library.Interface.ListChangesChangeType.Added, Library.Interface.ListChangesElementType.File, s); + foreach (var s in ReaderToStringList(cmd.ExecuteReader(added, (int)Interface.ListChangesElementType.Folder))) + yield return new Tuple(Interface.ListChangesChangeType.Added, Interface.ListChangesElementType.Folder, s); + foreach (var s in ReaderToStringList(cmd.ExecuteReader(added, (int)Interface.ListChangesElementType.Symlink))) + yield return new Tuple(Interface.ListChangesChangeType.Added, Interface.ListChangesElementType.Symlink, s); + foreach (var s in ReaderToStringList(cmd.ExecuteReader(added, (int)Interface.ListChangesElementType.File))) + yield return new Tuple(Interface.ListChangesChangeType.Added, Interface.ListChangesElementType.File, s); - foreach (var s in ReaderToStringList(cmd.ExecuteReader(deleted, (int)Library.Interface.ListChangesElementType.Folder))) - yield return new Tuple(Library.Interface.ListChangesChangeType.Deleted, Library.Interface.ListChangesElementType.Folder, s); - foreach (var s in ReaderToStringList(cmd.ExecuteReader(deleted, (int)Library.Interface.ListChangesElementType.Symlink))) - yield return new Tuple(Library.Interface.ListChangesChangeType.Deleted, Library.Interface.ListChangesElementType.Symlink, s); - foreach (var s in ReaderToStringList(cmd.ExecuteReader(deleted, (int)Library.Interface.ListChangesElementType.File))) - yield return new Tuple(Library.Interface.ListChangesChangeType.Deleted, Library.Interface.ListChangesElementType.File, s); + foreach (var s in ReaderToStringList(cmd.ExecuteReader(deleted, (int)Interface.ListChangesElementType.Folder))) + yield return new Tuple(Interface.ListChangesChangeType.Deleted, Interface.ListChangesElementType.Folder, s); + foreach (var s in ReaderToStringList(cmd.ExecuteReader(deleted, (int)Interface.ListChangesElementType.Symlink))) + yield return new Tuple(Interface.ListChangesChangeType.Deleted, Interface.ListChangesElementType.Symlink, s); + foreach (var s in ReaderToStringList(cmd.ExecuteReader(deleted, (int)Interface.ListChangesElementType.File))) + yield return new Tuple(Interface.ListChangesChangeType.Deleted, Interface.ListChangesElementType.File, s); - foreach (var s in ReaderToStringList(cmd.ExecuteReader(modified, (int)Library.Interface.ListChangesElementType.Folder))) - yield return new Tuple(Library.Interface.ListChangesChangeType.Modified, Library.Interface.ListChangesElementType.Folder, s); - foreach (var s in ReaderToStringList(cmd.ExecuteReader(modified, (int)Library.Interface.ListChangesElementType.Symlink))) - yield return new Tuple(Library.Interface.ListChangesChangeType.Modified, Library.Interface.ListChangesElementType.Symlink, s); - foreach (var s in ReaderToStringList(cmd.ExecuteReader(modified, (int)Library.Interface.ListChangesElementType.File))) - yield return new Tuple(Library.Interface.ListChangesChangeType.Modified, Library.Interface.ListChangesElementType.File, s); + foreach (var s in ReaderToStringList(cmd.ExecuteReader(modified, (int)Interface.ListChangesElementType.Folder))) + yield return new Tuple(Interface.ListChangesChangeType.Modified, Interface.ListChangesElementType.Folder, s); + foreach (var s in ReaderToStringList(cmd.ExecuteReader(modified, (int)Interface.ListChangesElementType.Symlink))) + yield return new Tuple(Interface.ListChangesChangeType.Modified, Interface.ListChangesElementType.Symlink, s); + foreach (var s in ReaderToStringList(cmd.ExecuteReader(modified, (int)Interface.ListChangesElementType.File))) + yield return new Tuple(Interface.ListChangesChangeType.Modified, Interface.ListChangesElementType.File, s); } } diff --git a/Duplicati/Library/Main/Database/LocalListDatabase.cs b/Duplicati/Library/Main/Database/LocalListDatabase.cs index 36b8b4958..b13208ef8 100644 --- a/Duplicati/Library/Main/Database/LocalListDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListDatabase.cs @@ -20,9 +20,11 @@ // DEALINGS IN THE SOFTWARE. using System; +using System.Data; using System.Linq; using System.Collections.Generic; using Duplicati.Library.Common.IO; +using Duplicati.Library.Utility; namespace Duplicati.Library.Main.Database { @@ -53,15 +55,15 @@ namespace Duplicati.Library.Main.Database { IEnumerable Sets { get; } IEnumerable QuickSets { get; } - IEnumerable SelectFiles(Library.Utility.IFilter filter); - IEnumerable GetLargestPrefix(Library.Utility.IFilter filter); - IEnumerable SelectFolderContents(Library.Utility.IFilter filter); + IEnumerable SelectFiles(IFilter filter); + IEnumerable GetLargestPrefix(IFilter filter); + IEnumerable SelectFolderContents(IFilter filter); void TakeFirst(); } private class FileSets : IFileSets { - private readonly System.Data.IDbConnection m_connection; + private readonly IDbConnection m_connection; private string m_tablename; private readonly KeyValuePair[] m_filesets; @@ -71,7 +73,7 @@ namespace Duplicati.Library.Main.Database m_filesets = owner.FilesetTimes.ToArray(); m_tablename = "Filesets-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); var tmp = owner.GetFilelistWhereClause(time, versions, m_filesets); - string query = tmp.Item1; + var query = tmp.Item1; var args = tmp.Item2; using (var cmd = m_connection.CreateCommand()) @@ -91,35 +93,35 @@ namespace Duplicati.Library.Main.Database public Fileset(long version, int isFullBackup, DateTime time, long filecount, long filesizes) { - this.Version = version; - this.IsFullBackup = isFullBackup; - this.Time = time; - this.FileCount = filecount; - this.FileSizes = filesizes; + Version = version; + IsFullBackup = isFullBackup; + Time = time; + FileCount = filecount; + FileSizes = filesizes; } } private class Fileversion : IFileversion { - private readonly System.Data.IDataReader m_reader; + private readonly IDataReader m_reader; public string Path { get; private set; } public bool More { get; private set; } - public Fileversion(System.Data.IDataReader reader) + public Fileversion(IDataReader reader) { m_reader = reader; - this.Path = reader.GetValue(0).ToString(); - this.More = true; + Path = reader.GetValue(0).ToString(); + More = true; } public IEnumerable Sizes { get { - while (this.More && this.Path == m_reader.GetValue(0).ToString()) + while (More && Path == m_reader.GetValue(0).ToString()) { yield return m_reader.ConvertValueToInt64(1, -1); - this.More = m_reader.Read(); + More = m_reader.Read(); } } } @@ -131,12 +133,12 @@ namespace Duplicati.Library.Main.Database public IEnumerable Sizes { get { return new long[0]; } } } - public IEnumerable GetLargestPrefix(Library.Utility.IFilter filter) + public IEnumerable GetLargestPrefix(IFilter filter) { return GetLargestPrefix(filter, null); } - private IEnumerable GetLargestPrefix(Library.Utility.IFilter filter, string prefixrule) + private IEnumerable GetLargestPrefix(IFilter filter, string prefixrule) { using (var tmpnames = new FilteredFilenameTable(m_connection, filter, null)) using (var cmd = m_connection.CreateCommand()) @@ -151,7 +153,7 @@ namespace Duplicati.Library.Main.Database // Then we recursively find the largest prefix cmd.CommandText = FormatInvariant($@"SELECT ""Path"" FROM ""{tmpnames.Tablename}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1"); var v0 = cmd.ExecuteScalar(); - string maxpath = ""; + var maxpath = ""; if (v0 != null) maxpath = v0.ToString(); @@ -159,7 +161,7 @@ namespace Duplicati.Library.Main.Database cmd.CommandText = FormatInvariant($@"SELECT COUNT(*) FROM ""{tmpnames.Tablename}"""); var filecount = cmd.ExecuteScalarInt64(0); - long foundfiles = -1; + var foundfiles = -1L; //TODO: Handle FS case-sensitive? cmd.CommandText = FormatInvariant($@"SELECT COUNT(*) FROM ""{tmpnames.Tablename}"" WHERE SUBSTR(""Path"", 1, ?) = ?"); @@ -198,14 +200,11 @@ namespace Duplicati.Library.Main.Database return roots.Concat(rootsUNC).Select(x => GetLargestPrefix(filter, x).First()).Distinct().ToArray(); } - return - new IFileversion[] { - new FileversionFixed { Path = maxpath == "" ? "" : Util.AppendDirSeparator(maxpath, dirsep) } - }; + return [new FileversionFixed { Path = maxpath == "" ? "" : Util.AppendDirSeparator(maxpath, dirsep) }]; } } - private IEnumerable SelectFolderEntries(System.Data.IDbCommand cmd, string prefix, string table) + private IEnumerable SelectFolderEntries(IDbCommand cmd, string prefix, string table) { if (!string.IsNullOrEmpty(prefix)) prefix = Util.AppendDirSeparator(prefix, Util.GuessDirSeparator(prefix)); @@ -228,7 +227,7 @@ namespace Duplicati.Library.Main.Database } } - public IEnumerable SelectFolderContents(Library.Utility.IFilter filter) + public IEnumerable SelectFolderContents(IFilter filter) { var tbname = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); try @@ -236,17 +235,17 @@ namespace Duplicati.Library.Main.Database string pathprefix; if (filter == null || filter.Empty) pathprefix = ""; - else if (filter as Library.Utility.FilterExpression == null || ((Library.Utility.FilterExpression)filter).Type != Duplicati.Library.Utility.FilterType.Simple || ((Library.Utility.FilterExpression)filter).GetSimpleList().Length != 1) + else if (filter as FilterExpression == null || ((FilterExpression)filter).Type != FilterType.Simple || ((FilterExpression)filter).GetSimpleList().Length != 1) throw new ArgumentException("Filter for list-folder-contents must be a path prefix with no wildcards", nameof(filter)); else - pathprefix = ((Library.Utility.FilterExpression)filter).GetSimpleList().First(); + pathprefix = ((FilterExpression)filter).GetSimpleList().First(); var dirsep = Util.GuessDirSeparator(pathprefix); if (pathprefix.Length > 0 || dirsep == "/") pathprefix = Util.AppendDirSeparator(pathprefix, dirsep); - using (var tmpnames = new FilteredFilenameTable(m_connection, new Library.Utility.FilterExpression(new string[] { pathprefix + "*" }, true), null)) + using (var tmpnames = new FilteredFilenameTable(m_connection, new FilterExpression(new string[] { pathprefix + "*" }, true), null)) using (var cmd = m_connection.CreateCommand()) { //First we trim the filelist to exclude filenames not found in any of the filesets @@ -321,7 +320,7 @@ namespace Duplicati.Library.Main.Database } } - public IEnumerable SelectFiles(Library.Utility.IFilter filter) + public IEnumerable SelectFiles(IFilter filter) { using (var tmpnames = new FilteredFilenameTable(m_connection, filter, null)) using (var cmd = m_connection.CreateCommand()) @@ -360,23 +359,17 @@ namespace Duplicati.Library.Main.Database { var dict = new Dictionary(); for (var i = 0; i < m_filesets.Length; i++) - { dict[m_filesets[i].Key] = i; - } using (var cmd = m_connection.CreateCommand()) - { - using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""ID"", ""IsFullBackup"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC ")) + using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""ID"", ""IsFullBackup"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC ")) + while (rd.Read()) { - while (rd.Read()) - { - var id = rd.GetInt64(0); - var backupType = rd.GetInt32(1); - var e = dict[id]; - yield return new Fileset(e, backupType, m_filesets[e].Value, -1L, -1L); - } + var id = rd.GetInt64(0); + var backupType = rd.GetInt32(1); + var e = dict[id]; + yield return new Fileset(e, backupType, m_filesets[e].Value, -1L, -1L); } - } } } @@ -386,31 +379,25 @@ namespace Duplicati.Library.Main.Database { var dict = new Dictionary(); for (var i = 0; i < m_filesets.Length; i++) - { dict[m_filesets[i].Key] = i; - } var summation = FormatInvariant( $@"SELECT ""A"".""FilesetID"" AS ""FilesetID"", COUNT(*) AS ""FileCount"", SUM(""C"".""Length"") AS ""FileSizes"" FROM ""FilesetEntry"" A, ""File"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" AND ""A"".""FilesetID"" IN (SELECT DISTINCT ""FilesetID"" FROM ""{m_tablename}"") GROUP BY ""A"".""FilesetID"" "); using (var cmd = m_connection.CreateCommand()) - { - using (var rd = cmd.ExecuteReader(FormatInvariant( - $@"SELECT DISTINCT ""A"".""FilesetID"", ""A"".""IsFullBackup"", ""B"".""FileCount"", ""B"".""FileSizes"" FROM ""{m_tablename}"" A LEFT OUTER JOIN ( {summation} ) B ON ""A"".""FilesetID"" = ""B"".""FilesetID"" ORDER BY ""A"".""Timestamp"" DESC ")) - ) + using (var rd = cmd.ExecuteReader(FormatInvariant( + $@"SELECT DISTINCT ""A"".""FilesetID"", ""A"".""IsFullBackup"", ""B"".""FileCount"", ""B"".""FileSizes"" FROM ""{m_tablename}"" A LEFT OUTER JOIN ( {summation} ) B ON ""A"".""FilesetID"" = ""B"".""FilesetID"" ORDER BY ""A"".""Timestamp"" DESC ")) + ) + while (rd.Read()) { - while (rd.Read()) - { - var id = rd.GetInt64(0); - var isFullBackup = rd.GetInt32(1); - var e = dict[id]; - var filecount = rd.ConvertValueToInt64(2, -1L); - var filesizes = rd.ConvertValueToInt64(3, -1L); + var id = rd.GetInt64(0); + var isFullBackup = rd.GetInt32(1); + var e = dict[id]; + var filecount = rd.ConvertValueToInt64(2, -1L); + var filesizes = rd.ConvertValueToInt64(3, -1L); - yield return new Fileset(e, isFullBackup, m_filesets[e].Value, filecount, filesizes); - } + yield return new Fileset(e, isFullBackup, m_filesets[e].Value, filecount, filesizes); } - } } } @@ -421,9 +408,7 @@ namespace Duplicati.Library.Main.Database try { using (var cmd = m_connection.CreateCommand()) - { cmd.ExecuteNonQuery(FormatInvariant(@$"DROP TABLE IF EXISTS ""{m_tablename}"" ")); - } } catch { } finally { m_tablename = null; } diff --git a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs index 0528a69e3..c539164f8 100644 --- a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs @@ -20,6 +20,7 @@ // DEALINGS IN THE SOFTWARE. using System; +using System.Data; using System.Collections.Generic; using Duplicati.Library.Utility; @@ -38,12 +39,12 @@ namespace Duplicati.Library.Main.Database { } - public ITemporaryFileset CreateTemporaryFileset(long parentid, System.Data.IDbTransaction transaction) + public ITemporaryFileset CreateTemporaryFileset(long parentid, IDbTransaction transaction) { return new TemporaryFileset(parentid, this, m_connection, transaction); } - public string GetRemoteVolumeNameForFileset(long id, System.Data.IDbTransaction transaction) + public string GetRemoteVolumeNameForFileset(long id, IDbTransaction transaction) { using (var cmd = m_connection.CreateCommand(transaction)) using (var rd = cmd.ExecuteReader(@"SELECT ""B"".""Name"" FROM ""Fileset"" A, ""RemoteVolume"" B WHERE ""A"".""VolumeID"" = ""B"".""ID"" AND ""A"".""ID"" = ? ", id)) @@ -53,7 +54,7 @@ namespace Duplicati.Library.Main.Database return rd.ConvertValueToString(0); } - internal long CountOrphanFiles(System.Data.IDbTransaction transaction) + internal long CountOrphanFiles(IDbTransaction transaction) { using (var cmd = m_connection.CreateCommand(transaction)) using (var rd = cmd.ExecuteReader(@"SELECT COUNT(*) FROM ""FileLookup"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""FileID"" FROM ""FilesetEntry"")")) @@ -70,15 +71,15 @@ namespace Duplicati.Library.Main.Database long RemovedFileSize { get; } void ApplyFilter(Library.Utility.IFilter filter); - void ApplyFilter(Action filtercommand); + void ApplyFilter(Action filtercommand); Tuple ConvertToPermanentFileset(string name, DateTime timestamp, bool isFullBackup); IEnumerable> ListAllDeletedFiles(); } private class TemporaryFileset : ITemporaryFileset { - private readonly System.Data.IDbConnection m_connection; - private readonly System.Data.IDbTransaction m_transaction; + private readonly IDbConnection m_connection; + private readonly IDbTransaction m_transaction; private readonly string m_tablename; private readonly LocalPurgeDatabase m_parentdb; @@ -86,9 +87,9 @@ namespace Duplicati.Library.Main.Database public long RemovedFileCount { get; private set; } public long RemovedFileSize { get; private set; } - public TemporaryFileset(long parentid, LocalPurgeDatabase parentdb, System.Data.IDbConnection connection, System.Data.IDbTransaction transaction) + public TemporaryFileset(long parentid, LocalPurgeDatabase parentdb, IDbConnection connection, IDbTransaction transaction) { - this.ParentID = parentid; + ParentID = parentid; m_parentdb = parentdb; m_connection = connection; m_transaction = transaction; @@ -98,7 +99,7 @@ namespace Duplicati.Library.Main.Database cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tablename}"" (""FileID"" INTEGER PRIMARY KEY) ")); } - public void ApplyFilter(Action filtercommand) + public void ApplyFilter(Action filtercommand) { using (var cmd = m_connection.CreateCommand(m_transaction)) filtercommand(cmd, ParentID, m_tablename); @@ -134,7 +135,7 @@ namespace Duplicati.Library.Main.Database else { // Do row-wise iteration - object[] values = new object[2]; + var values = new object[2]; using (var cmd = m_connection.CreateCommand(m_transaction)) using (var cmd2 = m_connection.CreateCommand(m_transaction)) { @@ -165,7 +166,7 @@ namespace Duplicati.Library.Main.Database RemovedFileSize = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT SUM(""C"".""Length"") FROM ""{m_tablename}"" A, ""FileLookup"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" "), 0); var filesetcount = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetID"" = {ParentID}"), 0); if (filesetcount == RemovedFileCount) - throw new Duplicati.Library.Interface.UserInformationException($"Refusing to purge {RemovedFileCount} files from fileset with ID {ParentID}, as that would remove the entire fileset.\nTo delete a fileset, use the \"delete\" command.", "PurgeWouldRemoveEntireFileset"); + throw new Interface.UserInformationException($"Refusing to purge {RemovedFileCount} files from fileset with ID {ParentID}, as that would remove the entire fileset.\nTo delete a fileset, use the \"delete\" command.", "PurgeWouldRemoveEntireFileset"); } } diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index ffbc4438e..05499fedf 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -19,6 +19,7 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; +using System.Data; using System.Collections.Generic; namespace Duplicati.Library.Main.Database @@ -45,21 +46,21 @@ namespace Duplicati.Library.Main.Database } } - private readonly System.Data.IDbCommand m_insertFileCommand; - private readonly System.Data.IDbCommand m_insertFilesetEntryCommand; - private readonly System.Data.IDbCommand m_insertMetadatasetCommand; - private readonly System.Data.IDbCommand m_insertBlocksetCommand; - private readonly System.Data.IDbCommand m_insertBlocklistHashCommand; - private readonly System.Data.IDbCommand m_updateBlockVolumeCommand; - private readonly System.Data.IDbCommand m_insertTempBlockListHash; - private readonly System.Data.IDbCommand m_insertSmallBlockset; - private readonly System.Data.IDbCommand m_findBlocksetCommand; - private readonly System.Data.IDbCommand m_findMetadatasetCommand; - private readonly System.Data.IDbCommand m_findFilesetCommand; - private readonly System.Data.IDbCommand m_findTempBlockListHashCommand; - private readonly System.Data.IDbCommand m_findHashBlockCommand; - private readonly System.Data.IDbCommand m_insertBlockCommand; - private readonly System.Data.IDbCommand m_insertDuplicateBlockCommand; + private readonly IDbCommand m_insertFileCommand; + private readonly IDbCommand m_insertFilesetEntryCommand; + private readonly IDbCommand m_insertMetadatasetCommand; + private readonly IDbCommand m_insertBlocksetCommand; + private readonly IDbCommand m_insertBlocklistHashCommand; + private readonly IDbCommand m_updateBlockVolumeCommand; + private readonly IDbCommand m_insertTempBlockListHash; + private readonly IDbCommand m_insertSmallBlockset; + private readonly IDbCommand m_findBlocksetCommand; + private readonly IDbCommand m_findMetadatasetCommand; + private readonly IDbCommand m_findFilesetCommand; + private readonly IDbCommand m_findTempBlockListHashCommand; + private readonly IDbCommand m_findHashBlockCommand; + private readonly IDbCommand m_insertBlockCommand; + private readonly IDbCommand m_insertDuplicateBlockCommand; private string m_tempblocklist; private string m_tempsmalllist; @@ -131,74 +132,27 @@ namespace Duplicati.Library.Main.Database cmd.ExecuteNonQuery(FormatInvariant($@"CREATE UNIQUE INDEX ""Index_Block_{m_tempsmalllist}"" ON ""{m_tempsmalllist}"" (""BlockHash"", ""BlockSize"");")); } - m_insertFileCommand = m_connection.CreateCommand(); - m_insertFilesetEntryCommand = m_connection.CreateCommand(); - m_insertMetadatasetCommand = m_connection.CreateCommand(); - m_insertBlocksetCommand = m_connection.CreateCommand(); - m_insertBlocklistHashCommand = m_connection.CreateCommand(); - m_updateBlockVolumeCommand = m_connection.CreateCommand(); - m_insertTempBlockListHash = m_connection.CreateCommand(); - m_insertSmallBlockset = m_connection.CreateCommand(); - m_findBlocksetCommand = m_connection.CreateCommand(); - m_findMetadatasetCommand = m_connection.CreateCommand(); - m_findFilesetCommand = m_connection.CreateCommand(); - m_findTempBlockListHashCommand = m_connection.CreateCommand(); - m_findHashBlockCommand = m_connection.CreateCommand(); - m_insertBlockCommand = m_connection.CreateCommand(); - m_insertDuplicateBlockCommand = m_connection.CreateCommand(); - - m_insertFileCommand.CommandText = @"INSERT INTO ""FileLookup"" (""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"") VALUES (?,?,?,?); SELECT last_insert_rowid();"; - m_insertFileCommand.AddParameters(4); - - m_insertFilesetEntryCommand.CommandText = @"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") VALUES (?,?,?)"; - m_insertFilesetEntryCommand.AddParameters(3); - - m_insertMetadatasetCommand.CommandText = @"INSERT INTO ""Metadataset"" (""BlocksetID"") VALUES (?); SELECT last_insert_rowid();"; - m_insertMetadatasetCommand.AddParameters(1); - - m_insertBlocksetCommand.CommandText = @"INSERT INTO ""Blockset"" (""Length"", ""FullHash"") VALUES (?,?); SELECT last_insert_rowid();"; - m_insertBlocksetCommand.AddParameters(2); - - m_insertBlocklistHashCommand.CommandText = @"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (?,?,?)"; - m_insertBlocklistHashCommand.AddParameters(3); - - m_updateBlockVolumeCommand.CommandText = @"UPDATE ""Block"" SET ""VolumeID"" = ? WHERE ""Hash"" = ? AND ""Size"" = ?"; - m_updateBlockVolumeCommand.AddParameters(3); - - m_insertTempBlockListHash.CommandText = FormatInvariant($@"INSERT INTO ""{m_tempblocklist}"" (""BlocklistHash"", ""BlockHash"", ""Index"") VALUES (?,?,?) "); - m_insertTempBlockListHash.AddParameters(3); - - m_insertSmallBlockset.CommandText = FormatInvariant($@"INSERT OR IGNORE INTO ""{m_tempsmalllist}"" (""FileHash"", ""BlockHash"", ""BlockSize"") VALUES (?,?,?) "); - m_insertSmallBlockset.AddParameters(3); - - m_findBlocksetCommand.CommandText = @"SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" = ? AND ""FullHash"" = ? "; - m_findBlocksetCommand.AddParameters(2); - - m_findMetadatasetCommand.CommandText = @"SELECT ""Metadataset"".""ID"" FROM ""Metadataset"",""Blockset"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Blockset"".""FullHash"" = ? AND ""Blockset"".""Length"" = ? "; - m_findMetadatasetCommand.AddParameters(2); - - m_findFilesetCommand.CommandText = @"SELECT ""ID"" FROM ""FileLookup"" WHERE ""PrefixID"" = ? AND ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ? "; - m_findFilesetCommand.AddParameters(4); - - m_findTempBlockListHashCommand.CommandText = FormatInvariant($@"SELECT DISTINCT ""BlockListHash"" FROM ""{m_tempblocklist}"" WHERE ""BlockListHash"" = ? "); - m_findTempBlockListHashCommand.AddParameters(1); - - m_findHashBlockCommand.CommandText = @"SELECT ""VolumeID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ? "; - m_findHashBlockCommand.AddParameters(2); - - m_insertBlockCommand.CommandText = @"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") VALUES (?,?,?)"; - m_insertBlockCommand.AddParameters(3); - - m_insertDuplicateBlockCommand.CommandText = @"INSERT OR IGNORE INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"") VALUES ((SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ?), ?)"; - m_insertDuplicateBlockCommand.AddParameters(3); + m_insertFileCommand = m_connection.CreateCommand(@"INSERT INTO ""FileLookup"" (""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"") VALUES (?,?,?,?); SELECT last_insert_rowid();"); + m_insertFilesetEntryCommand = m_connection.CreateCommand(@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") VALUES (?,?,?)"); + m_insertMetadatasetCommand = m_connection.CreateCommand(@"INSERT INTO ""Metadataset"" (""BlocksetID"") VALUES (?); SELECT last_insert_rowid();"); + m_insertBlocksetCommand = m_connection.CreateCommand(@"INSERT INTO ""Blockset"" (""Length"", ""FullHash"") VALUES (?,?); SELECT last_insert_rowid();"); + m_insertBlocklistHashCommand = m_connection.CreateCommand(@"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (?,?,?)"); + m_updateBlockVolumeCommand = m_connection.CreateCommand(@"UPDATE ""Block"" SET ""VolumeID"" = ? WHERE ""Hash"" = ? AND ""Size"" = ?"); + m_insertTempBlockListHash = m_connection.CreateCommand(FormatInvariant($@"INSERT INTO ""{m_tempblocklist}"" (""BlocklistHash"", ""BlockHash"", ""Index"") VALUES (?,?,?) ")); + m_insertSmallBlockset = m_connection.CreateCommand(FormatInvariant($@"INSERT OR IGNORE INTO ""{m_tempsmalllist}"" (""FileHash"", ""BlockHash"", ""BlockSize"") VALUES (?,?,?) ")); + m_findBlocksetCommand = m_connection.CreateCommand(@"SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" = ? AND ""FullHash"" = ? "); + m_findMetadatasetCommand = m_connection.CreateCommand(@"SELECT ""Metadataset"".""ID"" FROM ""Metadataset"",""Blockset"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Blockset"".""FullHash"" = ? AND ""Blockset"".""Length"" = ? "); + m_findFilesetCommand = m_connection.CreateCommand(@"SELECT ""ID"" FROM ""FileLookup"" WHERE ""PrefixID"" = ? AND ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ? "); + m_findTempBlockListHashCommand = m_connection.CreateCommand(FormatInvariant($@"SELECT DISTINCT ""BlockListHash"" FROM ""{m_tempblocklist}"" WHERE ""BlockListHash"" = ? ")); + m_findHashBlockCommand = m_connection.CreateCommand(@"SELECT ""VolumeID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ? "); + m_insertBlockCommand = m_connection.CreateCommand(@"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") VALUES (?,?,?)"); + m_insertDuplicateBlockCommand = m_connection.CreateCommand(@"INSERT OR IGNORE INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"") VALUES ((SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ?), ?)"); } - public void FindMissingBlocklistHashes(long hashsize, long blocksize, System.Data.IDbTransaction transaction) + public void FindMissingBlocklistHashes(long hashsize, long blocksize, IDbTransaction transaction) { - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(transaction)) { - cmd.Transaction = transaction; - //Update all small blocklists and matching blocks var selectSmallBlocks = FormatInvariant($@"SELECT ""BlockHash"", ""BlockSize"" FROM ""{m_tempsmalllist}"""); @@ -276,13 +230,11 @@ WHERE ""FullHash"" != ""Hash"" AND ""Length"" != ""Size"" "); /// temp small list structure: filehash, blockhash, blocksize: as the small files are defined /// by the fact that they are contained in a single block, blockhash is the same as the filehash, /// and blocksize can vary from 0 to the configured block size for the backup - public void AddBlockAndBlockSetEntryFromTemp(long hashsize, long blocksize, System.Data.IDbTransaction transaction, bool hashOnly = false) + public void AddBlockAndBlockSetEntryFromTemp(long hashsize, long blocksize, IDbTransaction transaction, bool hashOnly = false) { - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(transaction)) { - cmd.Transaction = transaction; - var insertBlocksCommand = FormatInvariant($@"INSERT INTO BLOCK (Hash, Size, VolumeID) SELECT DISTINCT BlockHash AS Hash, BlockSize AS Size, -1 AS VolumeID FROM ( @@ -351,31 +303,29 @@ BE.BlocksetID IS NULL "); } } - public void AddDirectoryEntry(long filesetid, long pathprefixid, string path, DateTime time, long metadataid, System.Data.IDbTransaction transaction) + public void AddDirectoryEntry(long filesetid, long pathprefixid, string path, DateTime time, long metadataid, IDbTransaction transaction) { AddEntry(filesetid, pathprefixid, path, time, FOLDER_BLOCKSET_ID, metadataid, transaction); } - public void AddSymlinkEntry(long filesetid, long pathprefixid, string path, DateTime time, long metadataid, System.Data.IDbTransaction transaction) + public void AddSymlinkEntry(long filesetid, long pathprefixid, string path, DateTime time, long metadataid, IDbTransaction transaction) { AddEntry(filesetid, pathprefixid, path, time, SYMLINK_BLOCKSET_ID, metadataid, transaction); } - public void AddFileEntry(long filesetid, long pathprefixid, string path, DateTime time, long blocksetid, long metadataid, System.Data.IDbTransaction transaction) + public void AddFileEntry(long filesetid, long pathprefixid, string path, DateTime time, long blocksetid, long metadataid, IDbTransaction transaction) { AddEntry(filesetid, pathprefixid, path, time, blocksetid, metadataid, transaction); } - private void AddEntry(long filesetid, long pathprefixid, string path, DateTime time, long blocksetid, long metadataid, System.Data.IDbTransaction transaction) + private void AddEntry(long filesetid, long pathprefixid, string path, DateTime time, long blocksetid, long metadataid, IDbTransaction transaction) { - var fileid = -1L; - m_findFilesetCommand.Transaction = transaction; m_findFilesetCommand.SetParameterValue(0, pathprefixid); m_findFilesetCommand.SetParameterValue(1, path); m_findFilesetCommand.SetParameterValue(2, blocksetid); m_findFilesetCommand.SetParameterValue(3, metadataid); - fileid = m_findFilesetCommand.ExecuteScalarInt64(-1); + var fileid = m_findFilesetCommand.ExecuteScalarInt64(-1); if (fileid < 0) { @@ -394,7 +344,7 @@ BE.BlocksetID IS NULL "); m_insertFilesetEntryCommand.ExecuteNonQuery(); } - public long AddMetadataset(string metahash, long metahashsize, IEnumerable metablocklisthashes, long expectedmetablocklisthashes, System.Data.IDbTransaction transaction) + public long AddMetadataset(string metahash, long metahashsize, IEnumerable metablocklisthashes, long expectedmetablocklisthashes, IDbTransaction transaction) { var metadataid = -1L; if (metahash == null) @@ -416,7 +366,7 @@ BE.BlocksetID IS NULL "); return metadataid; } - public long AddBlockset(string fullhash, long size, IEnumerable blocklisthashes, long expectedblocklisthashes, System.Data.IDbTransaction transaction) + public long AddBlockset(string fullhash, long size, IEnumerable blocklisthashes, long expectedblocklisthashes, IDbTransaction transaction) { m_findBlocksetCommand.Transaction = transaction; m_findBlocksetCommand.SetParameterValue(0, size); @@ -458,7 +408,7 @@ BE.BlocksetID IS NULL "); return blocksetid; } - public bool UpdateBlock(string hash, long size, long volumeID, System.Data.IDbTransaction transaction, ref bool anyChange) + public bool UpdateBlock(string hash, long size, long volumeID, IDbTransaction transaction, ref bool anyChange) { m_findHashBlockCommand.Transaction = transaction; m_findHashBlockCommand.SetParameterValue(0, hash); @@ -506,7 +456,7 @@ BE.BlocksetID IS NULL "); } } - public void AddSmallBlocksetLink(string filehash, string blockhash, long blocksize, System.Data.IDbTransaction transaction) + public void AddSmallBlocksetLink(string filehash, string blockhash, long blocksize, IDbTransaction transaction) { m_insertSmallBlockset.Transaction = transaction; m_insertSmallBlockset.SetParameterValue(0, filehash); @@ -515,7 +465,7 @@ BE.BlocksetID IS NULL "); m_insertSmallBlockset.ExecuteNonQuery(); } - public bool AddTempBlockListHash(string hash, IEnumerable blocklisthashes, System.Data.IDbTransaction transaction) + public bool AddTempBlockListHash(string hash, IEnumerable blocklisthashes, IDbTransaction transaction) { m_findTempBlockListHashCommand.Transaction = transaction; m_findTempBlockListHashCommand.SetParameterValue(0, hash); @@ -609,7 +559,6 @@ BE.BlocksetID IS NULL "); { while (rd.Read()) { - var volumeID = rd.GetInt64(3); // Guard against multiple downloads of the same file @@ -625,8 +574,6 @@ BE.BlocksetID IS NULL "); } } } - - } } @@ -704,7 +651,7 @@ DELETE FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND "" /// Move blocks that are not referenced by any files to DeletedBlock table. /// /// Needs to be called after the last FindMissingBlocklistHashes, otherwise the tables are not up to date. - public void CleanupDeletedBlocks(System.Data.IDbTransaction transaction) + public void CleanupDeletedBlocks(IDbTransaction transaction) { // Find out which blocks are deleted and move them into DeletedBlock, so that compact notices these blocks are empty // Deleted blocks do not appear in the BlocksetEntry and not in the BlocklistHash table diff --git a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs index bc9aee475..1ef96055d 100644 --- a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs @@ -20,9 +20,8 @@ // DEALINGS IN THE SOFTWARE. using System; -using System.Linq; +using System.Data; using System.Collections.Generic; -using System.Security.Cryptography; using Duplicati.Library.Utility; namespace Duplicati.Library.Main.Database @@ -58,12 +57,12 @@ namespace Duplicati.Library.Main.Database long Offset { get; } } - public interface IBlockWithSources : LocalBackupDatabase.IBlock + public interface IBlockWithSources : IBlock { IEnumerable Sources { get; } } - private class BlockWithSources : LocalBackupDatabase.Block, IBlockWithSources + private class BlockWithSources : Block, IBlockWithSources { private class BlockSource : IBlockSource { @@ -78,10 +77,10 @@ namespace Duplicati.Library.Main.Database } } - private readonly System.Data.IDataReader m_rd; + private readonly IDataReader m_rd; public bool Done { get; private set; } - public BlockWithSources(System.Data.IDataReader rd) + public BlockWithSources(IDataReader rd) : base(rd.GetString(0), rd.GetInt64(1)) { m_rd = rd; @@ -141,21 +140,20 @@ namespace Duplicati.Library.Main.Database private class MissingBlockList : IMissingBlockList { - private readonly System.Data.IDbConnection m_connection; + private readonly IDbConnection m_connection; private readonly TemporaryTransactionWrapper m_transaction; - private System.Data.IDbCommand m_insertCommand; + private IDbCommand m_insertCommand; private string m_tablename; private readonly string m_volumename; - public MissingBlockList(string volumename, System.Data.IDbConnection connection, System.Data.IDbTransaction transaction) + public MissingBlockList(string volumename, IDbConnection connection, IDbTransaction transaction) { m_connection = connection; m_transaction = new TemporaryTransactionWrapper(m_connection, transaction); m_volumename = volumename; var tablename = "MissingBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(m_transaction.Parent)) { - cmd.Transaction = m_transaction.Parent; cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" (""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Restored"" INTEGER NOT NULL) ")); m_tablename = tablename; @@ -166,10 +164,7 @@ namespace Duplicati.Library.Main.Database cmd.ExecuteNonQuery(FormatInvariant($@"CREATE UNIQUE INDEX ""{tablename}-Ix"" ON ""{tablename}"" (""Hash"", ""Size"", ""Restored"")")); } - m_insertCommand = m_connection.CreateCommand(); - m_insertCommand.Transaction = m_transaction.Parent; - m_insertCommand.CommandText = FormatInvariant($@"UPDATE ""{tablename}"" SET ""Restored"" = ? WHERE ""Hash"" = ? AND ""Size"" = ? AND ""Restored"" = ? "); - m_insertCommand.AddParameters(4); + m_insertCommand = m_connection.CreateCommand(m_transaction.Parent, FormatInvariant($@"UPDATE ""{tablename}"" SET ""Restored"" = ? WHERE ""Hash"" = ? AND ""Size"" = ? AND ""Restored"" = ? ")); } public bool SetBlockRestored(string hash, long size) @@ -238,7 +233,7 @@ namespace Duplicati.Library.Main.Database } } - public IMissingBlockList CreateBlockList(string volumename, System.Data.IDbTransaction transaction = null) + public IMissingBlockList CreateBlockList(string volumename, IDbTransaction transaction = null) { return new MissingBlockList(volumename, m_connection, transaction); } @@ -248,8 +243,6 @@ namespace Duplicati.Library.Main.Database using (var tr = m_connection.BeginTransaction()) using (var cmd = m_connection.CreateCommand(tr)) { - cmd.Transaction = tr; - var sql_count = @"SELECT COUNT(*) FROM (" + @" SELECT DISTINCT c1 FROM (" + @@ -281,9 +274,8 @@ namespace Duplicati.Library.Main.Database SELECT MIN(""ID"") AS ""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Entries"" FROM ""{tablename}"" GROUP BY ""Path"", ""BlocksetID"", ""MetadataID"") WHERE ""Entries"" > 1 ORDER BY ""ID"""); - using (var c2 = m_connection.CreateCommand()) + using (var c2 = m_connection.CreateCommand(tr)) { - c2.Transaction = tr; c2.CommandText = FormatInvariant($@"UPDATE ""FilesetEntry"" SET ""FileID"" = ? WHERE ""FileID"" IN (SELECT ""ID"" FROM ""{tablename}"" WHERE ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ?)"); c2.CommandText += FormatInvariant($@"; DELETE FROM ""{tablename}"" WHERE ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ? AND ""ID"" != ?"); foreach (var rd in cmd.ExecuteReaderEnumerable(sql)) @@ -298,7 +290,7 @@ namespace Duplicati.Library.Main.Database cmd.CommandText = sql_count; x = cmd.ExecuteScalarInt64(0); if (x > 1) - throw new Duplicati.Library.Interface.UserInformationException("Repair failed, there are still duplicate metadatahashes!", "DuplicateHashesRepairFailed"); + throw new Interface.UserInformationException("Repair failed, there are still duplicate metadatahashes!", "DuplicateHashesRepairFailed"); Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateMetadataHashesFixed", "Duplicate metadatahashes repaired succesfully"); tr.Commit(); @@ -333,7 +325,7 @@ namespace Duplicati.Library.Main.Database cmd.CommandText = sql_count; x = cmd.ExecuteScalarInt64(0); if (x > 1) - throw new Duplicati.Library.Interface.UserInformationException("Repair failed, there are still duplicate file entries!", "DuplicateFilesRepairFailed"); + throw new Interface.UserInformationException("Repair failed, there are still duplicate file entries!", "DuplicateFilesRepairFailed"); Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateFileEntriesFixed", "Duplicate file entries repaired succesfully"); tr.Commit(); @@ -437,7 +429,7 @@ namespace Duplicati.Library.Main.Database itemswithnoblocklisthash = cmd.ExecuteScalarInt64(countsql, 0); if (itemswithnoblocklisthash != 0) - throw new Duplicati.Library.Interface.UserInformationException($"Failed to repair, after repair {itemswithnoblocklisthash} blocklisthashes were missing", "MissingBlocklistHashesRepairFailed"); + throw new Interface.UserInformationException($"Failed to repair, after repair {itemswithnoblocklisthash} blocklisthashes were missing", "MissingBlocklistHashesRepairFailed"); Logging.Log.WriteInformationMessage(LOGTAG, "MissingBlocklisthashesRepaired", "Missing blocklisthashes repaired succesfully"); tr.Commit(); @@ -482,7 +474,7 @@ namespace Duplicati.Library.Main.Database var real_count = cmd.ExecuteScalarInt64(@"SELECT Count(*) FROM ""BlocklistHash""", 0); if (real_count != unique_count) - throw new Duplicati.Library.Interface.UserInformationException($"Failed to repair, result should have been {unique_count} blocklist hashes, but result was {real_count} blocklist hashes", "DuplicateBlocklistHashesRepairFailed"); + throw new Interface.UserInformationException($"Failed to repair, result should have been {unique_count} blocklist hashes, but result was {real_count} blocklist hashes", "DuplicateBlocklistHashesRepairFailed"); try { @@ -490,7 +482,7 @@ namespace Duplicati.Library.Main.Database } catch (Exception ex) { - throw new Duplicati.Library.Interface.UserInformationException("Repaired blocklisthashes, but the database was broken afterwards, rolled back changes", "DuplicateBlocklistHashesRepairFailed", ex); + throw new Interface.UserInformationException("Repaired blocklisthashes, but the database was broken afterwards, rolled back changes", "DuplicateBlocklistHashesRepairFailed", ex); } Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateBlocklistHashesRepaired", "Duplicate blocklisthashes repaired succesfully"); diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index 23ebb2be8..74223dbc6 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -82,9 +82,9 @@ namespace Duplicati.Library.Main.Database /// public void CreateProgressTracker(bool createFilesNewlyDoneTracker) { - m_fileprogtable = "FileProgress-" + this.m_temptabsetguid; - m_totalprogtable = "TotalProgress-" + this.m_temptabsetguid; - m_filesnewlydonetable = createFilesNewlyDoneTracker ? "FilesNewlyDone-" + this.m_temptabsetguid : null; + m_fileprogtable = "FileProgress-" + m_temptabsetguid; + m_totalprogtable = "TotalProgress-" + m_temptabsetguid; + m_filesnewlydonetable = createFilesNewlyDoneTracker ? "FilesNewlyDone-" + m_temptabsetguid : null; using (var cmd = m_connection.CreateCommand()) { @@ -199,7 +199,7 @@ END "); } - public Tuple PrepareRestoreFilelist(DateTime restoretime, long[] versions, Library.Utility.IFilter filter) + public Tuple PrepareRestoreFilelist(DateTime restoretime, long[] versions, IFilter filter) { m_tempfiletable = "Fileset-" + m_temptabsetguid; m_tempblocktable = "Blocks-" + m_temptabsetguid; @@ -214,7 +214,7 @@ END "); m_restoreTime = ParseFromEpochSeconds(cmd.ExecuteScalarInt64(@"SELECT ""Timestamp"" FROM ""Fileset"" WHERE ""ID"" = ?", 0, filesetId)); - var ix = this.FilesetTimes.Select((value, index) => new { value.Key, index }) + var ix = FilesetTimes.Select((value, index) => new { value.Key, index }) .Where(n => n.Key == filesetId) .Select(pair => pair.index + 1) .FirstOrDefault() - 1; @@ -291,28 +291,20 @@ END "); cmd.AddParameter(filesetId); object[] values = new object[4]; - using (var cmd2 = m_connection.CreateCommand()) - { - cmd2.CommandText = FormatInvariant($@"INSERT INTO ""{m_tempfiletable}"" (""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") VALUES (?,?,?,?,0)"); - cmd2.AddParameter(); - cmd2.AddParameter(); - cmd2.AddParameter(); - cmd2.AddParameter(); - - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) + using (var cmd2 = m_connection.CreateCommand(FormatInvariant($@"INSERT INTO ""{m_tempfiletable}"" (""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") VALUES (?,?,?,?,0)"))) + using (var rd = cmd.ExecuteReader()) + while (rd.Read()) + { + rd.GetValues(values); + if (values[1] != null && values[1] != DBNull.Value && Library.Utility.FilterExpression.Matches(filter, values[1].ToString())) { - rd.GetValues(values); - if (values[1] != null && values[1] != DBNull.Value && Library.Utility.FilterExpression.Matches(filter, values[1].ToString())) - { - cmd2.SetParameterValue(0, values[0]); - cmd2.SetParameterValue(1, values[1]); - cmd2.SetParameterValue(2, values[2]); - cmd2.SetParameterValue(3, values[3]); - cmd2.ExecuteNonQuery(); - } + cmd2.SetParameterValue(0, values[0]); + cmd2.SetParameterValue(1, values[1]); + cmd2.SetParameterValue(2, values[2]); + cmd2.SetParameterValue(3, values[3]); + cmd2.ExecuteNonQuery(); } - } + } } //creating indexes after insertion is much faster @@ -362,7 +354,7 @@ END "); { cmd.CommandText = FormatInvariant($@"SELECT ""Path"" FROM ""{m_tempfiletable}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1"); var v0 = cmd.ExecuteScalar(); - string maxpath = ""; + var maxpath = ""; if (v0 != null && v0 != DBNull.Value) maxpath = v0.ToString(); @@ -370,7 +362,7 @@ END "); cmd.CommandText = FormatInvariant($@"SELECT COUNT(*) FROM ""{m_tempfiletable}"""); var filecount = cmd.ExecuteScalarInt64(-1); - long foundfiles = -1; + var foundfiles = -1L; //TODO: Handle FS case-sensitive? cmd.CommandText = FormatInvariant($@"SELECT COUNT(*) FROM ""{m_tempfiletable}"" WHERE SUBSTR(""Path"", 1, ?) = ?"); @@ -475,7 +467,7 @@ END "); cmd.CommandText = FormatInvariant($@"INSERT INTO ""{m_tempblocktable}"" (""FileID"", ""Index"", ""Hash"", ""Size"", ""Restored"", ""Metadata"", ""VolumeId"", ""BlockId"") SELECT DISTINCT ""{m_tempfiletable}"".""ID"", ""BlocksetEntry"".""Index"", ""Block"".""Hash"", ""Block"".""Size"", 0, 0, ""Block"".""VolumeID"", ""Block"".""ID"" FROM ""{m_tempfiletable}"", ""BlocksetEntry"", ""Block"" WHERE ""{m_tempfiletable}"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" "); var p1 = cmd.ExecuteNonQuery(); - int p2 = 0; + var p2 = 0; if (!skipMetadata) { cmd.CommandText = FormatInvariant($@"INSERT INTO ""{m_tempblocktable}"" (""FileID"", ""Index"", ""Hash"", ""Size"", ""Restored"", ""Metadata"", ""VolumeId"", ""BlockId"") SELECT DISTINCT ""{m_tempfiletable}"".""ID"", ""BlocksetEntry"".""Index"", ""Block"".""Hash"", ""Block"".""Size"", 0, 1, ""Block"".""VolumeID"", ""Block"".""ID"" FROM ""{m_tempfiletable}"", ""BlocksetEntry"", ""Block"", ""Metadataset"" WHERE ""{m_tempfiletable}"".""MetadataID"" = ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" "); @@ -566,9 +558,9 @@ END "); private class ExistingFile : IExistingFile { - private readonly System.Data.IDataReader m_reader; + private readonly IDataReader m_reader; - public ExistingFile(System.Data.IDataReader rd) { m_reader = rd; HasMore = true; } + public ExistingFile(IDataReader rd) { m_reader = rd; HasMore = true; } public string TargetPath { get { return m_reader.ConvertValueToString(0); } } public string TargetHash { get { return m_reader.ConvertValueToString(1); } } @@ -579,9 +571,9 @@ END "); private class ExistingFileBlock : IExistingFileBlock { - private readonly System.Data.IDataReader m_reader; + private readonly IDataReader m_reader; - public ExistingFileBlock(System.Data.IDataReader rd) { m_reader = rd; } + public ExistingFileBlock(IDataReader rd) { m_reader = rd; } public string Hash { get { return m_reader.ConvertValueToString(4); } } public long Index { get { return m_reader.ConvertValueToInt64(5); } } @@ -592,8 +584,8 @@ END "); { get { - string p = this.TargetPath; - while (HasMore && p == this.TargetPath) + string p = TargetPath; + while (HasMore && p == TargetPath) { yield return new ExistingFileBlock(m_reader); HasMore = m_reader.Read(); @@ -601,7 +593,7 @@ END "); } } - public static IEnumerable GetExistingFilesWithBlocks(System.Data.IDbConnection connection, string tablename) + public static IEnumerable GetExistingFilesWithBlocks(IDbConnection connection, string tablename) { using (var cmd = connection.CreateCommand()) { @@ -636,16 +628,16 @@ END "); { private class BlockSource : IBlockSource { - private readonly System.Data.IDataReader m_reader; - public BlockSource(System.Data.IDataReader rd) { m_reader = rd; } + private readonly IDataReader m_reader; + public BlockSource(IDataReader rd) { m_reader = rd; } public string Path { get { return m_reader.ConvertValueToString(6); } } public long Offset { get { return m_reader.ConvertValueToInt64(7); } } public bool IsMetadata { get { return false; } } } - private readonly System.Data.IDataReader m_reader; - public BlockDescriptor(System.Data.IDataReader rd) { m_reader = rd; HasMore = true; } + private readonly IDataReader m_reader; + public BlockDescriptor(IDataReader rd) { m_reader = rd; HasMore = true; } private string TargetPath { get { return m_reader.ConvertValueToString(0); } } @@ -661,11 +653,11 @@ END "); { get { - var p = this.TargetPath; - var h = this.Hash; - var s = this.Size; + var p = TargetPath; + var h = Hash; + var s = Size; - while (HasMore && p == this.TargetPath && h == this.Hash && s == this.Size) + while (HasMore && p == TargetPath && h == Hash && s == Size) { yield return new BlockSource(m_reader); HasMore = m_reader.Read(); @@ -674,8 +666,8 @@ END "); } } - private readonly System.Data.IDataReader m_reader; - public LocalBlockSource(System.Data.IDataReader rd) { m_reader = rd; HasMore = true; } + private readonly IDataReader m_reader; + public LocalBlockSource(IDataReader rd) { m_reader = rd; HasMore = true; } public string TargetPath { get { return m_reader.ConvertValueToString(0); } } public long TargetFileID { get { return m_reader.ConvertValueToInt64(1); } } @@ -686,8 +678,8 @@ END "); { get { - var p = this.TargetPath; - while (HasMore && p == this.TargetPath) + var p = TargetPath; + while (HasMore && p == TargetPath) { var c = new BlockDescriptor(m_reader); var h = c.Hash; @@ -696,13 +688,13 @@ END "); yield return c; HasMore = c.HasMore; - while (HasMore && c.Hash == h && c.Size == s && this.TargetPath == p) + while (HasMore && c.Hash == h && c.Size == s && TargetPath == p) HasMore = m_reader.Read(); } } } - public static IEnumerable GetFilesAndSourceBlocks(System.Data.IDbConnection connection, string filetablename, string blocktablename, long blocksize, bool skipMetadata) + public static IEnumerable GetFilesAndSourceBlocks(IDbConnection connection, string filetablename, string blocktablename, long blocksize, bool skipMetadata) { using (var cmd = connection.CreateCommand()) { @@ -786,9 +778,9 @@ ORDER BY ""BB"".""MaxIndex"" "); private readonly string m_blocktablename; private readonly long m_blocksize; - private readonly System.Data.IDbConnection m_connection; + private readonly IDbConnection m_connection; - public FilesAndMetadata(System.Data.IDbConnection connection, string filetablename, string blocktablename, long blocksize, BlockVolumeReader curvolume) + public FilesAndMetadata(IDbConnection connection, string filetablename, string blocktablename, long blocksize, BlockVolumeReader curvolume) { m_filetablename = filetablename; m_blocktablename = blocktablename; @@ -830,16 +822,16 @@ ORDER BY ""BB"".""MaxIndex"" "); { private class PatchBlock : IPatchBlock { - private readonly System.Data.IDataReader m_reader; - public PatchBlock(System.Data.IDataReader rd) { m_reader = rd; } + private readonly IDataReader m_reader; + public PatchBlock(IDataReader rd) { m_reader = rd; } public long Offset { get { return m_reader.ConvertValueToInt64(2); } } public long Size { get { return m_reader.ConvertValueToInt64(3); } } public string Key { get { return m_reader.ConvertValueToString(4); } } } - private readonly System.Data.IDataReader m_reader; - public VolumePatch(System.Data.IDataReader rd) { m_reader = rd; HasMore = true; } + private readonly IDataReader m_reader; + public VolumePatch(IDataReader rd) { m_reader = rd; HasMore = true; } public string Path { get { return m_reader.ConvertValueToString(0); } } public long FileID { get { return m_reader.ConvertValueToInt64(1); } } @@ -849,8 +841,8 @@ ORDER BY ""BB"".""MaxIndex"" "); { get { - string p = this.Path; - while (HasMore && p == this.Path) + string p = Path; + while (HasMore && p == Path) { yield return new PatchBlock(m_reader); HasMore = m_reader.Read(); @@ -879,7 +871,7 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index"""); while (more) { var f = new VolumePatch(rd); - string current = f.Path; + var current = f.Path; yield return f; more = f.HasMore; @@ -939,7 +931,7 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index"""); { if (!m_connection_pool.TryTake(out var connection)) { - connection = (System.Data.IDbConnection)Activator.CreateInstance(Duplicati.Library.SQLiteHelper.SQLiteLoader.SQLiteConnectionType); + connection = (IDbConnection)Activator.CreateInstance(Duplicati.Library.SQLiteHelper.SQLiteLoader.SQLiteConnectionType); connection.ConnectionString = m_connection.ConnectionString + ";Cache=Shared;"; connection.Open(); @@ -959,9 +951,9 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index"""); public FileToRestore(long id, string path, string hash, long length) { - this.Path = path; - this.Hash = hash; - this.Length = length; + Path = path; + Hash = hash; + Length = length; } } @@ -1174,54 +1166,44 @@ ORDER BY ""A"".""TargetPath"", ""BB"".""Index"""); /// private class DirectBlockMarker : IBlockMarker { - private System.Data.IDbCommand m_insertblockCommand; - private System.Data.IDbCommand m_resetfileCommand; - private System.Data.IDbCommand m_updateAsRestoredCommand; - private System.Data.IDbCommand m_updateFileAsDataVerifiedCommand; - private System.Data.IDbCommand m_statUpdateCommand; + private IDbCommand m_insertblockCommand; + private IDbCommand m_resetfileCommand; + private IDbCommand m_updateAsRestoredCommand; + private IDbCommand m_updateFileAsDataVerifiedCommand; + private IDbCommand m_statUpdateCommand; + private IDbTransaction m_transaction; private bool m_hasUpdates = false; private readonly string m_blocktablename; private readonly string m_filetablename; - public DirectBlockMarker(System.Data.IDbConnection connection, string blocktablename, string filetablename, string statstablename) + public DirectBlockMarker(IDbConnection connection, string blocktablename, string filetablename, string statstablename) { - m_insertblockCommand = connection.CreateCommand(); - m_resetfileCommand = connection.CreateCommand(); - m_updateAsRestoredCommand = connection.CreateCommand(); - m_updateFileAsDataVerifiedCommand = connection.CreateCommand(); - m_statUpdateCommand = connection.CreateCommand(); - - m_insertblockCommand.Transaction = connection.BeginTransaction(); - m_resetfileCommand.Transaction = m_insertblockCommand.Transaction; - m_updateAsRestoredCommand.Transaction = m_insertblockCommand.Transaction; - m_updateFileAsDataVerifiedCommand.Transaction = m_insertblockCommand.Transaction; - m_statUpdateCommand.Transaction = m_insertblockCommand.Transaction; - + m_transaction = connection.BeginTransaction(); m_blocktablename = blocktablename; m_filetablename = filetablename; - m_insertblockCommand.CommandText = FormatInvariant($@"UPDATE ""{m_blocktablename}"" SET ""Restored"" = 1 -WHERE ""FileID"" = ? AND ""Index"" = ? AND ""Hash"" = ? AND ""Size"" = ? AND ""Metadata"" = ? AND ""Restored"" = 0 "); + m_insertblockCommand = connection.CreateCommand(m_transaction, FormatInvariant($@"UPDATE ""{m_blocktablename}"" SET ""Restored"" = 1 +WHERE ""FileID"" = ? AND ""Index"" = ? AND ""Hash"" = ? AND ""Size"" = ? AND ""Metadata"" = ? AND ""Restored"" = 0 ")); m_insertblockCommand.AddParameters(5); - m_resetfileCommand.CommandText = FormatInvariant($@"UPDATE ""{m_blocktablename}"" SET ""Restored"" = 0 WHERE ""FileID"" = ? "); + m_resetfileCommand = connection.CreateCommand(m_transaction, FormatInvariant($@"UPDATE ""{m_blocktablename}"" SET ""Restored"" = 0 WHERE ""FileID"" = ? ")); m_resetfileCommand.AddParameters(1); - m_updateAsRestoredCommand.CommandText = FormatInvariant($@"UPDATE ""{m_blocktablename}"" SET ""Restored"" = 1 WHERE ""FileID"" = ? AND ""Metadata"" <= ? "); + m_updateAsRestoredCommand = connection.CreateCommand(m_transaction, FormatInvariant($@"UPDATE ""{m_blocktablename}"" SET ""Restored"" = 1 WHERE ""FileID"" = ? AND ""Metadata"" <= ? ")); m_updateAsRestoredCommand.AddParameters(2); - m_updateFileAsDataVerifiedCommand.CommandText = FormatInvariant($@"UPDATE ""{m_filetablename}"" SET ""DataVerified"" = 1 WHERE ""ID"" = ?"); + m_updateFileAsDataVerifiedCommand = connection.CreateCommand(m_transaction, FormatInvariant($@"UPDATE ""{m_filetablename}"" SET ""DataVerified"" = 1 WHERE ""ID"" = ?")); m_updateFileAsDataVerifiedCommand.AddParameters(1); if (statstablename != null) { // Fields in Stats: TotalFiles, TotalBlocks, TotalSize // FilesFullyRestored, FilesPartiallyRestored, BlocksRestored, SizeRestored - m_statUpdateCommand.CommandText = FormatInvariant($@"SELECT SUM(""FilesFullyRestored""), SUM(""SizeRestored"") FROM ""{statstablename}"" "); + m_statUpdateCommand = connection.CreateCommand(m_transaction, FormatInvariant($@"SELECT SUM(""FilesFullyRestored""), SUM(""SizeRestored"") FROM ""{statstablename}"" ")); } else // very slow fallback if stats tables were not created - m_statUpdateCommand.CommandText = FormatInvariant($@"SELECT COUNT(DISTINCT ""FileID""), SUM(""Size"") FROM ""{m_blocktablename}"" WHERE ""Restored"" = 1 "); + m_statUpdateCommand = connection.CreateCommand(m_transaction, FormatInvariant($@"SELECT COUNT(DISTINCT ""FileID""), SUM(""Size"") FROM ""{m_blocktablename}"" WHERE ""Restored"" = 1 ")); } @@ -1290,40 +1272,22 @@ WHERE ""FileID"" = ? AND ""Index"" = ? AND ""Hash"" = ? AND ""Size"" = ? AND ""M public void Commit() { - var tr = m_insertblockCommand.Transaction; m_insertblockCommand.Dispose(); m_insertblockCommand = null; using (new Logging.Timer(LOGTAG, "CommitBlockMarker", "CommitBlockMarker")) - tr.Commit(); - tr.Dispose(); + m_transaction.Commit(); + m_transaction.Dispose(); + m_transaction = null; } public void Dispose() { - if (m_insertblockCommand != null) - try { m_insertblockCommand.Dispose(); } - catch { } - finally { m_insertblockCommand = null; } - - if (m_resetfileCommand != null) - try { m_resetfileCommand.Dispose(); } - catch { } - finally { m_resetfileCommand = null; } - - if (m_updateAsRestoredCommand != null) - try { m_updateAsRestoredCommand.Dispose(); } - catch { } - finally { m_updateAsRestoredCommand = null; } - - if (m_updateFileAsDataVerifiedCommand != null) - try { m_updateFileAsDataVerifiedCommand.Dispose(); } - catch { } - finally { m_updateFileAsDataVerifiedCommand = null; } - - if (m_statUpdateCommand != null) - try { m_statUpdateCommand.Dispose(); } - catch { } - finally { m_statUpdateCommand = null; } + m_insertblockCommand?.Dispose(); + m_resetfileCommand?.Dispose(); + m_updateAsRestoredCommand?.Dispose(); + m_updateFileAsDataVerifiedCommand?.Dispose(); + m_statUpdateCommand?.Dispose(); + m_transaction?.Dispose(); } } @@ -1372,18 +1336,18 @@ WHERE ""FileID"" = ? AND ""Index"" = ? AND ""Hash"" = ? AND ""Size"" = ? AND ""M { private class BlockEntry : IBlockEntry { - private readonly System.Data.IDataReader m_rd; + private readonly IDataReader m_rd; private readonly long m_blocksize; - public BlockEntry(System.Data.IDataReader rd, long blocksize) { m_rd = rd; m_blocksize = blocksize; } + public BlockEntry(IDataReader rd, long blocksize) { m_rd = rd; m_blocksize = blocksize; } public long Offset { get { return m_rd.ConvertValueToInt64(3) * m_blocksize; } } public long Index { get { return m_rd.ConvertValueToInt64(3); } } public long Size { get { return m_rd.ConvertValueToInt64(5); } } public string Hash { get { return m_rd.ConvertValueToString(4); } } } - private readonly System.Data.IDataReader m_rd; + private readonly IDataReader m_rd; private readonly long m_blocksize; - public FastSource(System.Data.IDataReader rd, long blocksize) { m_rd = rd; m_blocksize = blocksize; MoreData = true; } + public FastSource(IDataReader rd, long blocksize) { m_rd = rd; m_blocksize = blocksize; MoreData = true; } public bool MoreData { get; private set; } public string TargetPath { get { return m_rd.ConvertValueToString(0); } } public long TargetFileID { get { return m_rd.ConvertValueToInt64(2); } } @@ -1393,12 +1357,12 @@ WHERE ""FileID"" = ? AND ""Index"" = ? AND ""Hash"" = ? AND ""Size"" = ? AND ""M { get { - var tid = this.TargetFileID; + var tid = TargetFileID; do { yield return new BlockEntry(m_rd, m_blocksize); - } while ((MoreData = m_rd.Read()) && tid == this.TargetFileID); + } while ((MoreData = m_rd.Read()) && tid == TargetFileID); } } @@ -1407,37 +1371,32 @@ WHERE ""FileID"" = ? AND ""Index"" = ? AND ""Hash"" = ? AND ""Size"" = ? AND ""M public IEnumerable GetFilesAndSourceBlocksFast(long blocksize) { using (var transaction = m_connection.BeginTransaction()) + using (var cmdReader = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(transaction)) { - using (var cmdReader = m_connection.CreateCommand()) + cmd.CommandText = FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""LocalSourceExists"" = 1 WHERE Path = ?"); + cmd.AddParameters(1); + + var fileset = FormatInvariant($@"SELECT DISTINCT ""{m_tempfiletable}"".""Path"" FROM ""{m_tempfiletable}"""); + using (var rd = cmdReader.ExecuteReader(fileset)) { - using (var cmd = m_connection.CreateCommand()) + while (rd.Read()) { - cmd.CommandText = FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""LocalSourceExists"" = 1 WHERE Path = ?"); - cmd.AddParameters(1); - cmd.Transaction = transaction; - - var fileset = FormatInvariant($@"SELECT DISTINCT ""{m_tempfiletable}"".""Path"" FROM ""{m_tempfiletable}"""); - using (var rd = cmdReader.ExecuteReader(fileset)) + var sourcepath = rd.GetValue(0).ToString(); + if (SystemIO.IO_OS.FileExists(sourcepath)) { - while (rd.Read()) - { - var sourcepath = rd.GetValue(0).ToString(); - if (SystemIO.IO_OS.FileExists(sourcepath)) - { - cmd.SetParameterValue(0, sourcepath); - cmd.ExecuteNonQuery(); - } - else - { - Logging.Log.WriteVerboseMessage(LOGTAG, "LocalSourceMissing", "Local source file not found: {0}", sourcepath); - } - } + cmd.SetParameterValue(0, sourcepath); + cmd.ExecuteNonQuery(); + } + else + { + Logging.Log.WriteVerboseMessage(LOGTAG, "LocalSourceMissing", "Local source file not found: {0}", sourcepath); } - - //This localSourceExists index will make the query engine to start by searching FileSet table. As the result is ordered by FileSet.ID, we will get the cursor "instantly" - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""{m_tempfiletable}_LocalSourceExists"" ON ""{m_tempfiletable}"" (""LocalSourceExists"")")); } } + + //This localSourceExists index will make the query engine to start by searching FileSet table. As the result is ordered by FileSet.ID, we will get the cursor "instantly" + cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""{m_tempfiletable}_LocalSourceExists"" ON ""{m_tempfiletable}"" (""LocalSourceExists"")")); transaction.Commit(); } @@ -1505,7 +1464,7 @@ ORDER BY ""{m_tempfiletable}"".""ID"", ""{m_tempblocktable}"".""Index"" "); { if (rd.Read()) { - var more = false; + bool more; do { var n = new FastSource(rd, blocksize); diff --git a/Duplicati/Library/Main/Database/LocalTestDatabase.cs b/Duplicati/Library/Main/Database/LocalTestDatabase.cs index 16aadde88..e25ee0248 100644 --- a/Duplicati/Library/Main/Database/LocalTestDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalTestDatabase.cs @@ -20,6 +20,7 @@ // DEALINGS IN THE SOFTWARE. using System; +using System.Data; using System.Linq; using System.Collections.Generic; @@ -51,13 +52,13 @@ namespace Duplicati.Library.Main.Database public string Hash { get; private set; } public long VerificationCount { get; private set; } - public RemoteVolume(System.Data.IDataReader rd) + public RemoteVolume(IDataReader rd) { - this.ID = rd.GetInt64(0); - this.Name = rd.GetString(1); - this.Size = rd.ConvertValueToInt64(2); - this.Hash = rd.ConvertValueToString(3); - this.VerificationCount = rd.ConvertValueToInt64(4); + ID = rd.GetInt64(0); + Name = rd.GetString(1); + Size = rd.ConvertValueToInt64(2); + Hash = rd.ConvertValueToString(3); + VerificationCount = rd.ConvertValueToInt64(4); } } @@ -171,30 +172,26 @@ namespace Duplicati.Library.Main.Database private abstract class Basiclist : IDisposable { - protected readonly System.Data.IDbConnection m_connection; + protected readonly IDbConnection m_connection; protected readonly string m_volumename; protected string m_tablename; - protected System.Data.IDbTransaction m_transaction; - protected System.Data.IDbCommand m_insertCommand; + protected IDbTransaction m_transaction; + protected IDbCommand m_insertCommand; - protected Basiclist(System.Data.IDbConnection connection, string volumename, string tablePrefix, string tableFormat, string insertCommand, int insertArguments) + protected Basiclist(IDbConnection connection, string volumename, string tablePrefix, string tableFormat, string insertCommand) { m_connection = connection; m_volumename = volumename; m_transaction = m_connection.BeginTransaction(); var tablename = tablePrefix + "-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(m_transaction)) { - cmd.Transaction = m_transaction; cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" {tableFormat}")); m_tablename = tablename; } - m_insertCommand = m_connection.CreateCommand(); - m_insertCommand.Transaction = m_transaction; - m_insertCommand.CommandText = FormatInvariant($@"INSERT INTO ""{m_tablename}"" {insertCommand}"); - m_insertCommand.AddParameters(insertArguments); + m_insertCommand = m_connection.CreateCommand(m_transaction, FormatInvariant($@"INSERT INTO ""{m_tablename}"" {insertCommand}")); } public virtual void Dispose() @@ -202,31 +199,21 @@ namespace Duplicati.Library.Main.Database if (m_tablename != null) try { - using (var cmd = m_connection.CreateCommand()) - { - cmd.Transaction = m_transaction; + using (var cmd = m_connection.CreateCommand(m_transaction)) cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tablename}""")); - } } catch { } finally { m_tablename = null; } - if (m_insertCommand != null) - try { m_insertCommand.Dispose(); } - catch { } - finally { m_insertCommand = null; } - - if (m_transaction != null) - try { m_transaction.Rollback(); } - catch { } - finally { m_transaction = null; } + m_insertCommand?.Dispose(); + m_transaction?.Rollback(); } } public interface IFilelist : IDisposable { void Add(string path, long size, string hash, long metasize, string metahash, IEnumerable blocklistHashes, FilelistEntryType type, DateTime time); - IEnumerable> Compare(); + IEnumerable> Compare(); } private class Filelist : Basiclist, IFilelist @@ -234,10 +221,8 @@ namespace Duplicati.Library.Main.Database private const string TABLE_PREFIX = "Filelist"; private const string TABLE_FORMAT = @"(""Path"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Hash"" TEXT NULL, ""Metasize"" INTEGER NOT NULL, ""Metahash"" TEXT NOT NULL)"; private const string INSERT_COMMAND = @"(""Path"", ""Size"", ""Hash"", ""Metasize"", ""Metahash"") VALUES (?,?,?,?,?)"; - private const int INSERT_ARGUMENTS = 5; - - public Filelist(System.Data.IDbConnection connection, string volumename) - : base(connection, volumename, Filelist.TABLE_PREFIX, Filelist.TABLE_FORMAT, Filelist.INSERT_COMMAND, Filelist.INSERT_ARGUMENTS) + public Filelist(IDbConnection connection, string volumename) + : base(connection, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) { } @@ -261,10 +246,8 @@ namespace Duplicati.Library.Main.Database var modified = FormatInvariant($@"SELECT ? AS ""Type"", ""E"".""Path"" AS ""Path"" FROM ""{m_tablename}"" E, ""{cmpName}"" D WHERE ""D"".""Path"" = ""E"".""Path"" AND (""D"".""Size"" != ""E"".""Size"" OR ""D"".""Hash"" != ""E"".""Hash"" OR ""D"".""Metasize"" != ""E"".""Metasize"" OR ""D"".""Metahash"" != ""E"".""Metahash"") "); var drop = FormatInvariant($@"DROP TABLE IF EXISTS ""{cmpName}"" "); - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(m_transaction)) { - cmd.Transaction = m_transaction; - try { cmd.ExecuteNonQuery(create, m_volumename); @@ -293,10 +276,9 @@ namespace Duplicati.Library.Main.Database private const string TABLE_PREFIX = "Indexlist"; private const string TABLE_FORMAT = @"(""Name"" TEXT NOT NULL, ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)"; private const string INSERT_COMMAND = @"(""Name"", ""Hash"", ""Size"") VALUES (?,?,?)"; - private const int INSERT_ARGUMENTS = 3; - public Indexlist(System.Data.IDbConnection connection, string volumename) - : base(connection, volumename, Indexlist.TABLE_PREFIX, Indexlist.TABLE_FORMAT, Indexlist.INSERT_COMMAND, Indexlist.INSERT_ARGUMENTS) + public Indexlist(IDbConnection connection, string volumename) + : base(connection, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) { } @@ -317,10 +299,8 @@ namespace Duplicati.Library.Main.Database var modified = FormatInvariant($@"SELECT ? AS ""Type"", ""E"".""Name"" AS ""Name"" FROM ""{m_tablename}"" E, ""{cmpName}"" D WHERE ""D"".""Name"" = ""E"".""Name"" AND (""D"".""Hash"" != ""E"".""Hash"" OR ""D"".""Size"" != ""E"".""Size"") "); var drop = FormatInvariant($@"DROP TABLE IF EXISTS ""{cmpName}"" "); - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(m_transaction)) { - cmd.Transaction = m_transaction; - try { cmd.ExecuteNonQuery(create, m_volumename); @@ -349,10 +329,9 @@ namespace Duplicati.Library.Main.Database private const string TABLE_PREFIX = "Blocklist"; private const string TABLE_FORMAT = @"(""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)"; private const string INSERT_COMMAND = @"(""Hash"", ""Size"") VALUES (?,?)"; - private const int INSERT_ARGUMENTS = 2; - public Blocklist(System.Data.IDbConnection connection, string volumename) - : base(connection, volumename, Blocklist.TABLE_PREFIX, Blocklist.TABLE_FORMAT, Blocklist.INSERT_COMMAND, Blocklist.INSERT_ARGUMENTS) + public Blocklist(IDbConnection connection, string volumename) + : base(connection, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) { } public void AddBlock(string hash, long size) @@ -362,7 +341,7 @@ namespace Duplicati.Library.Main.Database m_insertCommand.ExecuteNonQuery(); } - public IEnumerable> Compare() + public IEnumerable> Compare() { var cmpName = "CmpTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); var curBlocks = @"SELECT ""Block"".""Hash"" AS ""Hash"", ""Block"".""Size"" AS ""Size"" FROM ""Remotevolume"", ""Block"" WHERE ""Remotevolume"".""Name"" = ? AND ""Remotevolume"".""ID"" = ""Block"".""VolumeID"""; @@ -374,10 +353,8 @@ namespace Duplicati.Library.Main.Database var modified = FormatInvariant($@"SELECT ? AS ""Type"", ""E"".""Hash"" AS ""Hash"" FROM ""{m_tablename}"" E, ""{cmpName}"" D WHERE ""D"".""Hash"" = ""E"".""Hash"" AND ""D"".""Size"" != ""E"".""Size"" "); var drop = FormatInvariant($@"DROP TABLE IF EXISTS ""{cmpName}"" "); - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(m_transaction)) { - cmd.Transaction = m_transaction; - try { cmd.ExecuteNonQuery(create, m_volumename, m_volumename, m_volumename); diff --git a/Duplicati/Library/Main/Database/RemoteVolume.cs b/Duplicati/Library/Main/Database/RemoteVolume.cs index e2bc5e2e8..92f3bc77b 100644 --- a/Duplicati/Library/Main/Database/RemoteVolume.cs +++ b/Duplicati/Library/Main/Database/RemoteVolume.cs @@ -1,51 +1,47 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. namespace Duplicati.Library.Main.Database { public class RemoteVolume : IRemoteVolume { - public RemoteVolume(Library.Interface.IFileEntry file, string hash = null) + public RemoteVolume(Interface.IFileEntry file, string hash = null) { - this.Name = file.Name; - this.Size = file.Size; - this.Hash = hash; - this.File = file; + Name = file.Name; + Size = file.Size; + Hash = hash; + File = file; } public RemoteVolume(string name, string hash, long size) { - this.Name = name; - this.Hash = hash; - this.Size = size; - this.File = null; + Name = name; + Hash = hash; + Size = size; + File = null; } public string Name { get; private set; } public string Hash { get; private set; } public long Size { get; private set; } - public Library.Interface.IFileEntry File { get; private set; } + public Interface.IFileEntry File { get; private set; } } } diff --git a/Duplicati/Library/Main/Database/RemoteVolumeEntry.cs b/Duplicati/Library/Main/Database/RemoteVolumeEntry.cs index 6256b01ab..fb17607e9 100644 --- a/Duplicati/Library/Main/Database/RemoteVolumeEntry.cs +++ b/Duplicati/Library/Main/Database/RemoteVolumeEntry.cs @@ -1,27 +1,24 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; namespace Duplicati.Library.Main.Database { diff --git a/Duplicati/Library/Main/Utility.cs b/Duplicati/Library/Main/Utility.cs index ae80bce81..844d28b56 100644 --- a/Duplicati/Library/Main/Utility.cs +++ b/Duplicati/Library/Main/Utility.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -167,7 +167,6 @@ namespace Duplicati.Library.Main newDict.Add("passphrase", Library.Utility.Utility.ByteArrayAsHexString(Library.Utility.Utility.RepeatedHashWithSalt(options.Passphrase, salt, 1200))); } - var needsUpdate = false; foreach (var k in newDict) if (!opts.ContainsKey(k.Key)) diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index 9410e6bd6..92a6ecc24 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -777,7 +777,7 @@ namespace Duplicati.Library.Utility return (T)(object)flags; } - + /// /// Parses an option with int value, returning the default value if the option is not found or cannot be parsed /// @@ -785,9 +785,9 @@ namespace Duplicati.Library.Utility /// The value to look for in the settings /// default value /// - public static int ParseIntOption(IReadOnlyDictionary options, string value, int @default) - { - return options.TryGetValue(value, out var opt) && int.TryParse(opt ?? string.Empty, out var result) ? result : @default; + public static int ParseIntOption(IReadOnlyDictionary options, string value, int @default) + { + return options.TryGetValue(value, out var opt) && int.TryParse(opt ?? string.Empty, out var result) ? result : @default; } /// @@ -1546,7 +1546,7 @@ namespace Duplicati.Library.Utility collection.Import(pfxPath, password, X509KeyStorageFlags.Exportable | X509KeyStorageFlags.PersistKeySet); return collection; } - + /// /// Probes the system for the presence of a loopback address on IPv4 /// @@ -1564,7 +1564,6 @@ namespace Duplicati.Library.Utility /// public static string IpVersionCompatibleLoopback => HasIPv4Loopback ? IPAddress.Loopback.ToString() : $"[{IPAddress.IPv6Loopback.ToString()}]"; - /// /// Flattens an exception and its inner exceptions @@ -1650,7 +1649,7 @@ namespace Duplicati.Library.Utility /// /// The formattable string /// The formatted string - public static string FormatInvariant(FormattableString formattable) + public static string FormatInvariant(this FormattableString formattable) => formattable.ToString(CultureInfo.InvariantCulture); } } From a15dd4f2e08d69b27c18be7c47638a5582e8a1c6 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 14 Mar 2025 15:11:38 +0100 Subject: [PATCH 355/362] Safeguards for release compile This adds additional checks for the compile phase to avoid missing files in the released builds. The build is now done into isolated folders, and then copied into the shared target folder to avoid issues with the compiler replacing or removing files. --- ReleaseBuilder/Build/Command.Compile.Post.cs | 33 ++++++++++++++++++++ ReleaseBuilder/Build/Command.Compile.cs | 13 +++++++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/ReleaseBuilder/Build/Command.Compile.Post.cs b/ReleaseBuilder/Build/Command.Compile.Post.cs index d33b195b0..0be144823 100644 --- a/ReleaseBuilder/Build/Command.Compile.Post.cs +++ b/ReleaseBuilder/Build/Command.Compile.Post.cs @@ -64,6 +64,39 @@ public static partial class Command } } + /// + /// Verify that some files that are expected to be present in the target directory are there + /// + /// The build directory to verify + /// The target to verify for + /// An awaitable task + public static Task VerifyTargetDirectory(string buildDir, PackageTarget target) + { + var rootFiles = Directory.EnumerateFiles(buildDir, "*", SearchOption.TopDirectoryOnly) + .Where(x => x.EndsWith(".dll", StringComparison.OrdinalIgnoreCase) || x.EndsWith(".exe", StringComparison.OrdinalIgnoreCase)) + .Select(x => Path.GetFileName(x)) + .ToHashSet(Duplicati.Library.Utility.Utility.ClientFilenameStringComparer); + + // Random sample of files we expect + var probeFiles = new string[] { + "System.CommandLine.dll", + "System.CommandLine.NamingConventionBinder.dll", + "AWSSDK.S3.dll", + "CoCoL.dll", + "Duplicati.Library.Interface.dll", + "Google.Apis.Auth.dll", + "Google.Apis.Core.dll", + "SQLiteHelper.dll", + "SQLite.Interop.dll", + }; + + foreach (var f in probeFiles) + if (!rootFiles.Contains(f)) + throw new Exception($"Expected file {f} for {target.BuildTargetString}, but was not found in build directory {buildDir}"); + + return Task.CompletedTask; + } + /// /// Set of files that are unwanted despite the OS /// diff --git a/ReleaseBuilder/Build/Command.Compile.cs b/ReleaseBuilder/Build/Command.Compile.cs index 9851b45db..90ee18ce9 100644 --- a/ReleaseBuilder/Build/Command.Compile.cs +++ b/ReleaseBuilder/Build/Command.Compile.cs @@ -78,6 +78,11 @@ public static partial class Command // TODO: Self contained builds are bloating the build size // Alternative is to require the .NET runtime to be installed + // Build into isolated folder + var buildfolder = Path.Combine(buildDir, target.BuildTargetString + "-out"); + if (Directory.Exists(buildfolder)) + Directory.Delete(buildfolder, true); + // Fix any RIDs that differ from .NET SDK var archstring = target.Arch switch { @@ -88,7 +93,7 @@ public static partial class Command var command = new string[] { "dotnet", "publish", proj, "-c", "Release", - "-o", tmpfolder, + "-o", buildfolder, "-r", archstring, $"/p:AssemblyVersion={releaseInfo.Version}", $"/p:Version={releaseInfo.Version}-{releaseInfo.Channel}-{releaseInfo.Timestamp:yyyyMMdd}", @@ -104,11 +109,17 @@ public static partial class Command Console.WriteLine($"Error building {proj} for {target.BuildTargetString}: {ex.Message}"); throw; } + + // Merge the build into the temporary target folder + EnvHelper.CopyDirectory(buildfolder, tmpfolder, true); + Directory.Delete(buildfolder, true); } // Perform any post-build steps, cleaning and signing as needed await PostCompile.PrepareTargetDirectory(baseDir, tmpfolder, target, rtcfg, keepBuilds); + await PostCompile.VerifyTargetDirectory(tmpfolder, target); + // Move the final build to the output folder Directory.Move(tmpfolder, outputFolder); } From 27032048a3600b86fdb279d7c224f9308dfaf416 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 14 Mar 2025 15:12:03 +0100 Subject: [PATCH 356/362] Added Armv7 agent builds --- ReleaseBuilder/Program.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ReleaseBuilder/Program.cs b/ReleaseBuilder/Program.cs index f410d52c0..290476c2f 100644 --- a/ReleaseBuilder/Program.cs +++ b/ReleaseBuilder/Program.cs @@ -61,6 +61,8 @@ class Program "linux-arm7-cli.zip", "linux-arm7-cli.deb", "linux-arm7-cli.docker", + "linux-arm7-agent.zip", + "linux-arm7-agent.deb", "linux-arm64-gui.zip", "linux-arm64-gui.deb", From 12f53513d0bd7a48782264ffa500dc293cb2591c Mon Sep 17 00:00:00 2001 From: "Marcelo C." Date: Fri, 14 Mar 2025 12:09:01 -0300 Subject: [PATCH 357/362] Re-work of SSL Validation + Root Certificate feature Initially the idea was to allow scanning not only the certificate but scan the whole certificate chain for a explicitly supplied hash to be accepted unconditionally. A review of the validation revealed it was always missing a fallback to the certificate.verify() Also decided that even when a hash is supplied and matched, the validity date supersedes other criteria except the acceptAll. --- .../Utility/SslCertificateValidator.cs | 100 +++++++++--------- 1 file changed, 52 insertions(+), 48 deletions(-) diff --git a/Duplicati/Library/Utility/SslCertificateValidator.cs b/Duplicati/Library/Utility/SslCertificateValidator.cs index e21e36469..41e62bd6b 100644 --- a/Duplicati/Library/Utility/SslCertificateValidator.cs +++ b/Duplicati/Library/Utility/SslCertificateValidator.cs @@ -1,25 +1,26 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; +using System.Linq; using System.Net.Security; using System.Security.Cryptography.X509Certificates; @@ -28,52 +29,55 @@ namespace Duplicati.Library.Utility; public class SslCertificateValidator(bool acceptAll, string[] validHashes) { [Serializable] - public class InvalidCertificateException : Exception + public class InvalidCertificateException(string certificate, SslPolicyErrors error) + : Exception(Strings.SslCertificateValidator.VerifyCertificateException(error, certificate)) { - private readonly string m_certificate; - private readonly SslPolicyErrors m_errors = SslPolicyErrors.None; + private readonly string _mCertificate = certificate; + private readonly SslPolicyErrors _mErrors = error; - public string Certificate => m_certificate; - public SslPolicyErrors SslError => m_errors; - - public InvalidCertificateException(string certificate, SslPolicyErrors error) - : base(Strings.SslCertificateValidator.VerifyCertificateException(error, certificate)) - { - m_certificate = certificate; - m_errors = error; - } + public string Certificate => _mCertificate; + public SslPolicyErrors SslError => _mErrors; } public bool ValidateServerCertificate(object sender, X509Certificate cert, X509Chain chain, SslPolicyErrors sslPolicyErrors) { - - if (sslPolicyErrors == SslPolicyErrors.None) - return true; - if (acceptAll) return true; - - string certHash; - + try { - certHash = Utility.ByteArrayAsHexString(cert.GetCertHash()); - if (certHash != null && validHashes != null) + DateTime now = DateTime.Now; + + using var certificate = cert as X509Certificate2 ?? new X509Certificate2(cert); + + if (!IsDateValid(certificate, now)) + return false; + + if (validHashes != null) { - foreach (var hash in validHashes) - { - if (!string.IsNullOrEmpty(hash) && certHash.Equals(hash, StringComparison.OrdinalIgnoreCase)) - { + // Check main certificate hash + if (IsTrustedHash(Utility.ByteArrayAsHexString(certificate.GetCertHash()))) + return true; + + // Check chain certificate from root for the hash (this allows custom CA certificates hashes to be added) + if (chain?.ChainElements != null) + if (chain.ChainElements.Any(element => IsTrustedHash(Utility.ByteArrayAsHexString(element.Certificate.GetCertHash())) && IsDateValid(element.Certificate, now))) return true; - } - } } + + // If no hash is found, perform the standard validations + return sslPolicyErrors == SslPolicyErrors.None && certificate.Verify(); } catch (Exception ex) { throw new Exception(Strings.SslCertificateValidator.VerifyCertificateHashError(ex, sslPolicyErrors), ex); } - - return false; } + + private bool IsTrustedHash(string hash) => + !string.IsNullOrWhiteSpace(hash) && + validHashes.Any(validHash => !string.IsNullOrEmpty(validHash) && + hash.Equals(validHash, StringComparison.OrdinalIgnoreCase)); + private bool IsDateValid(X509Certificate2 cert, DateTime now) => now <= cert.NotAfter && now >= cert.NotBefore; + } \ No newline at end of file From 359fd507679ced34ae6a0c3b7692f9655a9c1306 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sat, 15 Mar 2025 10:43:07 +0100 Subject: [PATCH 358/362] Bump ngclient to 0.0.97 --- Duplicati/Server/webroot/ngclient/package-lock.json | 8 ++++---- Duplicati/Server/webroot/ngclient/package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Duplicati/Server/webroot/ngclient/package-lock.json b/Duplicati/Server/webroot/ngclient/package-lock.json index 34c630c3e..34f830ffb 100644 --- a/Duplicati/Server/webroot/ngclient/package-lock.json +++ b/Duplicati/Server/webroot/ngclient/package-lock.json @@ -9,13 +9,13 @@ "version": "1.0.0", "license": "MIT", "dependencies": { - "@duplicati/ngclient": "^0.0.85" + "@duplicati/ngclient": "^0.0.97" } }, "node_modules/@duplicati/ngclient": { - "version": "0.0.85", - "resolved": "https://registry.npmjs.org/@duplicati/ngclient/-/ngclient-0.0.85.tgz", - "integrity": "sha512-998IYLYD1KYY2q9l3z1EiqxyoVFCs3aCXKnZJtE3gK/PJcFagIAAbGWhxJ6RvXozn4dm+8PFcu2bcTJdE37qeQ==", + "version": "0.0.97", + "resolved": "https://registry.npmjs.org/@duplicati/ngclient/-/ngclient-0.0.97.tgz", + "integrity": "sha512-4c0RCVfBk3WwqDeK5mYc6NwhEwnM1UQLtfD5n7FoMs0i6NTFkRBkKX35bgxFZK2ujLONDylt/hkR8FxZNiMxAA==", "license": "MIT" } } diff --git a/Duplicati/Server/webroot/ngclient/package.json b/Duplicati/Server/webroot/ngclient/package.json index beed1e594..c64d2fd23 100644 --- a/Duplicati/Server/webroot/ngclient/package.json +++ b/Duplicati/Server/webroot/ngclient/package.json @@ -4,6 +4,6 @@ "description": "Modern Angular Client for Duplicati", "license": "MIT", "dependencies": { - "@duplicati/ngclient": "^0.0.85" + "@duplicati/ngclient": "^0.0.97" } } From d44673c23bd00b304cac1ed285435ca171617655 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sat, 15 Mar 2025 12:06:44 +0100 Subject: [PATCH 359/362] Version bump to v2.1.0.111-2.1.0.111_canary_2025-03-15 You can download this build from: Binaries: https://updates.duplicati.com/canary/?version=2.1.0.111 Signature file: https://updates.duplicati.com/canary/duplicati-2.1.0.111_canary_2025-03-15.signatures.zip --- ReleaseBuilder/build_version.txt | 2 +- changelog.txt | 38 ++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/ReleaseBuilder/build_version.txt b/ReleaseBuilder/build_version.txt index 4e5414f9a..c4d3bba64 100644 --- a/ReleaseBuilder/build_version.txt +++ b/ReleaseBuilder/build_version.txt @@ -1 +1 @@ -2.1.0.110 \ No newline at end of file +2.1.0.111 \ No newline at end of file diff --git a/changelog.txt b/changelog.txt index e2acdeba2..cbb55c69f 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,41 @@ +2025-03-15 - 2.1.0.111_canary_2025-03-15 +========== +This release is a canary release intended to be used for testing. + +# Major changes in this version + +- Restore improvements +This version has a few tweaks to the new restore flow that makes it more robust. + +- Improved validation +To safeguard against internal errors, there is now a more strict validation before performing cleanup. +The new logic checks if the previous operation failed and will only automatically repair the remote if this is the case. +The repair command bypasses this check and can repair the remote regardless. + +If the previous run was fully completed, any leftover remote files are now treated as an error. +Additionally, operations that does not change remote state (list, test, restore, etc) will not repair the remote state, but just verify that it is working for the requested operation. + +- Updates to ngclient +Multiple updates to ngclient: improved auth flow, searchable menus, Windows filesystem support, restore from configuration, and more. + +# Potentially breaking changes (from 2.1.0.109) + - Location of data folder has changed in some cases, refer to previous release for details + - Permissions are now applied to the data folder, refer to previous release for details + - New user interface, `ngclient` is the default; there is a button to switch between the two + +## Detailed list of changes: +- Reduced number of samples taken for testing +- Improved restore flow +- Improved error handling for Azure Blob storage +- Improved help text for remote synchronization tool +- Implemented more rigorous validation and safeguards for failed backups +- Better support for IPv6 and unmapped localhost scenarios +- Added check to avoid block sizes that are the same or less than volume size +- Introduced more async calls for better thread usage in handlers +- Added forced culture for SQL strings and simpler transaction logic +- Added Arm v7 Agent builds +- Improved certificate validation to check expiration and support root or intermediate certificate hashes + 2025-02-28 - 2.1.0.110_canary_2025-02-28 ========== This release is a canary release intended to be used for testing. From e37f20aedd9660985d8cea0ff9f3b9ec2734d349 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sun, 16 Mar 2025 16:52:30 +0100 Subject: [PATCH 360/362] Added retries for HTTP messages This adds a retry count and a retry delay for posting backup reports via HTTP requests. This will make the report delivery more reliable if the network is unstable or server is briefly unreachable. --- .../Modules/Builtin/SendHttpMessage.cs | 54 +++++++++++--- .../Modules/Builtin/SendJabberMessage.cs | 3 - Duplicati/Library/Modules/Builtin/SendMail.cs | 48 +++++------- Duplicati/Library/Modules/Builtin/Strings.cs | 4 + Duplicati/Library/Utility/RetryHelper.cs | 17 +++-- Duplicati/Library/Utility/Timeparser.cs | 74 +++++++++---------- Duplicati/Library/Utility/Utility.cs | 18 ++++- 7 files changed, 131 insertions(+), 87 deletions(-) diff --git a/Duplicati/Library/Modules/Builtin/SendHttpMessage.cs b/Duplicati/Library/Modules/Builtin/SendHttpMessage.cs index 94f97bc56..6bd57fdec 100644 --- a/Duplicati/Library/Modules/Builtin/SendHttpMessage.cs +++ b/Duplicati/Library/Modules/Builtin/SendHttpMessage.cs @@ -28,6 +28,7 @@ using System.Text; using System.Threading.Tasks; using Duplicati.Library.Utility; using Uri = System.Uri; +using System.Threading; namespace Duplicati.Library.Modules.Builtin { @@ -113,6 +114,15 @@ namespace Duplicati.Library.Modules.Builtin /// private const string OPTION_ACCEPT_ANY_CERTIFICATE = "send-http-accept-any-ssl-certificate"; + /// + /// The option used to specify the number of retries for sending the HTTP request + /// + private const string OPTION_SEND_HTTP_RETRIES = "send-http-retries"; + /// + /// The option used to specify the delay between retries for sending the HTTP request + /// + private const string OPTION_SEND_HTTP_RETRY_DELAY = "send-http-retry-delay"; + #endregion #region Option defaults @@ -128,6 +138,14 @@ namespace Duplicati.Library.Modules.Builtin /// Don't use the subject for HTTP /// protected override string DEFAULT_SUBJECT => string.Empty; + /// + /// The default number of retries + /// + private const int DEFAULT_RETRIES = 3; + /// + /// The default delay between retries + /// + private const string DEFAULT_RETRY_DELAY = "1s"; #endregion #region Private variables @@ -154,6 +172,15 @@ namespace Duplicati.Library.Modules.Builtin /// private string[] m_acceptSpecificCertificates; + /// + /// The number of retries to attempt + /// + private int m_retries; + /// + /// The delay between retries + /// + private TimeSpan m_retryDelay; + #endregion @@ -208,6 +235,9 @@ namespace Duplicati.Library.Modules.Builtin new CommandLineArgument(OPTION_ACCEPT_ANY_CERTIFICATE, CommandLineArgument.ArgumentType.Boolean, Strings.SendHttpMessage.AcceptAnyCertificateShort, Strings.SendHttpMessage.AcceptAnyCertificateLong), new CommandLineArgument(OPTION_ACCEPT_SPECIFIED_CERTIFICATE, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.AcceptSpecifiedCertificateShort, Strings.SendHttpMessage.AcceptSpecifiedCertificateLong), + + new CommandLineArgument(OPTION_SEND_HTTP_RETRIES, CommandLineArgument.ArgumentType.Integer, Strings.SendHttpMessage.SendHttpRetriesShort, Strings.SendHttpMessage.SendHttpRetriesLong, DEFAULT_RETRIES.ToString()), + new CommandLineArgument(OPTION_SEND_HTTP_RETRY_DELAY, CommandLineArgument.ArgumentType.Integer, Strings.SendHttpMessage.SendHttpRetryDelayShort, Strings.SendHttpMessage.SendHttpRetryDelayLong, DEFAULT_RETRY_DELAY), }); } } @@ -266,9 +296,12 @@ namespace Duplicati.Library.Modules.Builtin m_messageParameterName = DEFAULT_MESSAGE_PARAMETER_NAME; commandlineOptions.TryGetValue(OPTION_EXTRA_PARAMETERS, out m_extraParameters); - m_acceptAnyCertificate = commandlineOptions.ContainsKey(OPTION_ACCEPT_ANY_CERTIFICATE) && Utility.Utility.ParseBoolOption(commandlineOptions.AsReadOnly(), OPTION_ACCEPT_ANY_CERTIFICATE); + m_acceptAnyCertificate = Utility.Utility.ParseBoolOption(commandlineOptions.AsReadOnly(), OPTION_ACCEPT_ANY_CERTIFICATE); m_acceptSpecificCertificates = commandlineOptions.ContainsKey(OPTION_ACCEPT_SPECIFIED_CERTIFICATE) ? commandlineOptions[OPTION_ACCEPT_SPECIFIED_CERTIFICATE].Split([",", ";"], StringSplitOptions.RemoveEmptyEntries) : null; + m_retries = Utility.Utility.ParseIntOption(commandlineOptions.AsReadOnly(), OPTION_SEND_HTTP_RETRIES, DEFAULT_RETRIES); + m_retryDelay = Utility.Utility.ParseTimespanOption(commandlineOptions.AsReadOnly(), OPTION_SEND_HTTP_RETRY_DELAY, DEFAULT_RETRY_DELAY); + return true; } @@ -301,7 +334,9 @@ namespace Duplicati.Library.Modules.Builtin }; request.Content.Headers.ContentType = contenttype; - try + Exception lastEx = null; + + await RetryHelper.Retry(async () => { var response = await client.SendAsync(request); var responseContent = await response.Content.ReadAsStringAsync(); @@ -315,18 +350,17 @@ namespace Duplicati.Library.Modules.Builtin ); response.EnsureSuccessStatusCode(); - } - catch (Exception ex) + lastEx = null; + }, (ex, retry) => { - Logging.Log.WriteWarningMessage(LOGTAG, "HttpResponseError", ex, "HTTP Response request failed for: {0}", target.Url); - return ex; - } + Logging.Log.WriteWarningMessage(LOGTAG, "HttpResponseError", ex, "HTTP Response request attempt {0} of {1} failed for: {2}", retry, m_retries, target.Url); + lastEx = ex; + }, m_retries, m_retryDelay, CancellationToken.None); - return null; + return lastEx; } private Dictionary m_cachedBodyResults; - private string m_form_body = string.Empty; protected override string ReplaceTemplate(string input, object result, Exception exception, bool subjectline) { @@ -364,7 +398,7 @@ namespace Duplicati.Library.Modules.Builtin foreach (var target in m_report_targets) { if (m_cachedBodyResults.TryGetValue(target.Format, out var result)) - ex ??= SendMessage(client, target, subject, result).ConfigureAwait(false).GetAwaiter().GetResult(); + ex ??= SendMessage(client, target, subject, result).Await(); } if (ex != null) diff --git a/Duplicati/Library/Modules/Builtin/SendJabberMessage.cs b/Duplicati/Library/Modules/Builtin/SendJabberMessage.cs index bbb565017..81c54e07b 100644 --- a/Duplicati/Library/Modules/Builtin/SendJabberMessage.cs +++ b/Duplicati/Library/Modules/Builtin/SendJabberMessage.cs @@ -22,9 +22,6 @@ using System; using System.Collections.Generic; using System.Linq; using Duplicati.Library.Interface; -using Duplicati.Library.Logging; -using System.Net.NetworkInformation; -using Duplicati.Library.Modules.Builtin.ResultSerialization; using Artalk.Xmpp.Client; namespace Duplicati.Library.Modules.Builtin diff --git a/Duplicati/Library/Modules/Builtin/SendMail.cs b/Duplicati/Library/Modules/Builtin/SendMail.cs index beac39fee..9dc2b9e03 100644 --- a/Duplicati/Library/Modules/Builtin/SendMail.cs +++ b/Duplicati/Library/Modules/Builtin/SendMail.cs @@ -1,39 +1,31 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; -using System.Collections; using System.Collections.Generic; using System.IO; using System.Linq; -using System.Net; using System.Text; -using System.Text.RegularExpressions; using Duplicati.Library.Interface; -using Duplicati.Library.Logging; -using Duplicati.Library.Utility; -using System.Net.NetworkInformation; -using System.Net.Sockets; using MailKit.Net.Smtp; using MimeKit; -using Duplicati.Library.Modules.Builtin.ResultSerialization; using DnsClient; namespace Duplicati.Library.Modules.Builtin @@ -277,7 +269,7 @@ namespace Duplicati.Library.Modules.Builtin lastServer = server; try { - using (MemoryStream ms = new MemoryStream()) + using (var ms = new MemoryStream()) { try { diff --git a/Duplicati/Library/Modules/Builtin/Strings.cs b/Duplicati/Library/Modules/Builtin/Strings.cs index 5d4750cc5..bcd66b666 100644 --- a/Duplicati/Library/Modules/Builtin/Strings.cs +++ b/Duplicati/Library/Modules/Builtin/Strings.cs @@ -231,6 +231,10 @@ You can supply multiple options with a comma separator, e.g. ""{0},{1}"". The sp public static string AcceptAnyCertificateShort { get { return LC.L(@"Accept any server certificate"); } } public static string AcceptSpecifiedCertificateLong { get { return LC.L(@"If your server certificate is reported as invalid (e.g. with self-signed certificates), you can supply the certificate hash (SHA1) to approve it anyway. The hash value must be entered in hex format without spaces or colons. You can enter multiple hashes separated by commas."); } } public static string AcceptSpecifiedCertificateShort { get { return LC.L(@"Optionally accept a known SSL certificate"); } } + public static string SendHttpRetriesLong { get { return LC.L(@"Use this option to set the number of retries to attempt if the HTTP request fails."); } } + public static string SendHttpRetriesShort { get { return LC.L(@"Set the number of retries"); } } + public static string SendHttpRetryDelayLong { get { return LC.L(@"Use this option to set the delay between retries."); } } + public static string SendHttpRetryDelayShort { get { return LC.L(@"Set the retry delay"); } } } internal static class ReportHelper diff --git a/Duplicati/Library/Utility/RetryHelper.cs b/Duplicati/Library/Utility/RetryHelper.cs index 38554e15a..1bbcaae3e 100644 --- a/Duplicati/Library/Utility/RetryHelper.cs +++ b/Duplicati/Library/Utility/RetryHelper.cs @@ -19,6 +19,8 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +#nullable enable + using System; using System.Threading; using System.Threading.Tasks; @@ -27,7 +29,7 @@ namespace Duplicati.Library.Utility; public static class RetryHelper { - public static async Task Retry(Func> action, int maxRetries, TimeSpan delay, CancellationToken token) + public static async Task Retry(Func> action, Action? errorCallback, int maxRetries, TimeSpan delay, CancellationToken token) { var attempt = 0; @@ -37,21 +39,26 @@ public static class RetryHelper { return await action(); } - catch + catch (Exception ex) { + attempt++; + errorCallback?.Invoke(ex, attempt); + if (token.IsCancellationRequested) throw; if (attempt >= maxRetries) throw; - - attempt++; } await Task.Delay(delay, token); } } + public static async Task Retry(Func> action, int maxRetries, TimeSpan delay, CancellationToken token) + => await Retry(action, null, maxRetries, delay, token); + public static async Task Retry(Func action, Action? errorCallback, int maxRetries, TimeSpan delay, CancellationToken token) + => await Retry(async () => { await action(); return true; }, errorCallback, maxRetries, delay, token); public static async Task Retry(Func action, int maxRetries, TimeSpan delay, CancellationToken token) - => await Retry(async () => { await action(); return true; }, maxRetries, delay, token); + => await Retry(async () => { await action(); return true; }, null, maxRetries, delay, token); } diff --git a/Duplicati/Library/Utility/Timeparser.cs b/Duplicati/Library/Utility/Timeparser.cs index cc505b468..64f9a23cc 100644 --- a/Duplicati/Library/Utility/Timeparser.cs +++ b/Duplicati/Library/Utility/Timeparser.cs @@ -1,27 +1,25 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; -using System.Collections.Generic; -using System.Text; namespace Duplicati.Library.Utility { @@ -34,7 +32,7 @@ namespace Duplicati.Library.Utility { public static TimeSpan ParseTimeSpan(string datestring) { - DateTime dt = new DateTime(0, DateTimeKind.Local); + var dt = new DateTime(0, DateTimeKind.Local); return ParseTimeInterval(datestring, dt) - dt; } @@ -48,16 +46,15 @@ namespace Duplicati.Library.Utility if (offset.Kind == DateTimeKind.Unspecified) offset = new DateTime(offset.Ticks, DateTimeKind.Local); - int multiplier = negate ? -1 : 1; + var multiplier = negate ? -1 : 1; if (string.IsNullOrEmpty(datestring)) return offset; - if (String.Equals(datestring.Trim(), "now", StringComparison.OrdinalIgnoreCase)) + if (string.Equals(datestring.Trim(), "now", StringComparison.OrdinalIgnoreCase)) return DateTime.Now; - long l; - if (long.TryParse(datestring, System.Globalization.NumberStyles.Integer, null, out l)) + if (long.TryParse(datestring, System.Globalization.NumberStyles.Integer, null, out var l)) return offset.AddSeconds(l * multiplier); if (DateTime.TryParse(datestring, System.Globalization.CultureInfo.CurrentCulture, System.Globalization.DateTimeStyles.AssumeLocal, out var t)) @@ -66,16 +63,15 @@ namespace Duplicati.Library.Utility if (Utility.TryDeserializeDateTime(datestring, out t)) return t; - char[] separators = new char[] { 's', 'm', 'h', 'D', 'W', 'M', 'Y' }; + var separators = new char[] { 's', 'm', 'h', 'D', 'W', 'M', 'Y' }; - int index = 0; - int previndex = 0; + int index; + var previndex = 0; while ((index = datestring.IndexOfAny(separators, previndex)) > 0) { - string partial = datestring.Substring(previndex, index - previndex).Trim(); - int factor; - if (!int.TryParse(partial, System.Globalization.NumberStyles.Integer, null, out factor)) + var partial = datestring.Substring(previndex, index - previndex).Trim(); + if (!int.TryParse(partial, System.Globalization.NumberStyles.Integer, null, out var factor)) throw new Exception(Strings.Timeparser.InvalidIntegerError(partial)); factor *= multiplier; @@ -134,11 +130,10 @@ namespace Duplicati.Library.Utility if (string.IsNullOrEmpty(datestring)) return offset; - if (String.Equals(datestring.Trim(), "now", StringComparison.OrdinalIgnoreCase)) + if (string.Equals(datestring.Trim(), "now", StringComparison.OrdinalIgnoreCase)) return DateTime.UtcNow; - long l; - if (long.TryParse(datestring, System.Globalization.NumberStyles.Integer, null, out l)) + if (long.TryParse(datestring, System.Globalization.NumberStyles.Integer, null, out var l)) return keepTimeOfDay ? timeZoneInfo.DSTAwareAddSeconds(offset, l * multiplier) : timeZoneInfo.DSTAwareAddSeconds(DateTime.UtcNow, l * multiplier); @@ -149,16 +144,15 @@ namespace Duplicati.Library.Utility if (Utility.TryDeserializeDateTime(datestring, out t)) return t; - char[] separators = ['s', 'm', 'h', 'D', 'W', 'M', 'Y']; + var separators = new char[] { 's', 'm', 'h', 'D', 'W', 'M', 'Y' }; int index; - int previndex = 0; + var previndex = 0; while ((index = datestring.IndexOfAny(separators, previndex)) > 0) { - string partial = datestring.Substring(previndex, index - previndex).Trim(); - int factor; - if (!int.TryParse(partial, System.Globalization.NumberStyles.Integer, null, out factor)) + var partial = datestring.Substring(previndex, index - previndex).Trim(); + if (!int.TryParse(partial, System.Globalization.NumberStyles.Integer, null, out var factor)) throw new Exception(Strings.Timeparser.InvalidIntegerError(partial)); factor *= multiplier; diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index 92a6ecc24..9264943d7 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -702,7 +702,7 @@ namespace Duplicati.Library.Utility /// /// The set of options to look for the setting in /// The value to look for in the settings - /// + /// The parsed value, or the default value (false). public static bool ParseBoolOption(IReadOnlyDictionary options, string value) { if (options.TryGetValue(value, out var opt)) @@ -711,6 +711,22 @@ namespace Duplicati.Library.Utility return false; } + /// + /// Parses an integer option from the option set, returning the default value if the option is not found or cannot be parsed + /// + /// The set of options to look for the setting in + /// The value to look for in the settings + /// The default value to return if there are no matches. + /// The parsed or default integer value. + public static TimeSpan ParseTimespanOption(IReadOnlyDictionary options, string value, string @default) + { + var opt = options.GetValueOrDefault(value); + if (string.IsNullOrWhiteSpace(opt)) + opt = @default; + + return Timeparser.ParseTimeSpan(opt); + } + /// /// Parses an enum found in the options dictionary /// From 55096931376504b6a4e03c8801d77e0e023a24ed Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sun, 16 Mar 2025 17:26:12 +0100 Subject: [PATCH 361/362] Added warning state back --- Duplicati/GUI/Duplicati.GUI.TrayIcon/AvaloniaRunner.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/AvaloniaRunner.cs b/Duplicati/GUI/Duplicati.GUI.TrayIcon/AvaloniaRunner.cs index 6473f334d..e10821d74 100644 --- a/Duplicati/GUI/Duplicati.GUI.TrayIcon/AvaloniaRunner.cs +++ b/Duplicati/GUI/Duplicati.GUI.TrayIcon/AvaloniaRunner.cs @@ -357,6 +357,9 @@ namespace Duplicati.GUI.TrayIcon case TrayIcons.IdleError: this.trayIcon.Icon = AvaloniaRunner.LoadIcon("normal-error.png"); break; + case TrayIcons.IdleWarning: + this.trayIcon.Icon = AvaloniaRunner.LoadIcon("normal-warning.png"); + break; case TrayIcons.Paused: case TrayIcons.PausedError: this.trayIcon.Icon = AvaloniaRunner.LoadIcon("normal-pause.png"); @@ -399,7 +402,7 @@ namespace Duplicati.GUI.TrayIcon trayIcon.Clicked += HandleTrayIconClick; } } - + private readonly ClickDebouncer _clickDebouncer = new ClickDebouncer(); private void HandleTrayIconClick(object? sender, EventArgs e) From 902bd2218520c6c0b71da5154162da71db3402b5 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 17 Mar 2025 13:32:39 +0100 Subject: [PATCH 362/362] Making comparison in local time --- Duplicati/Library/Main/Operation/RepairHandler.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 0b72185c8..ea405ef1c 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -149,8 +149,8 @@ namespace Duplicati.Library.Main.Operation var progress = 0; var targetProgess = tp.ExtraVolumes.Count() + tp.MissingVolumes.Count() + tp.VerificationRequiredVolumes.Count() + missingRemoteFilesets.Count + missingLocalFilesets.Count; - var mostRecentLocal = db.FilesetTimes.Select(x => x.Value).Append(DateTime.MinValue).Max(); - var mostRecentRemote = tp.ParsedVolumes.Select(x => x.Time).Append(DateTime.MinValue).Max(); + var mostRecentLocal = db.FilesetTimes.Select(x => x.Value.ToLocalTime()).Append(DateTime.MinValue).Max(); + var mostRecentRemote = tp.ParsedVolumes.Select(x => x.Time.ToLocalTime()).Append(DateTime.MinValue).Max(); if (mostRecentLocal < DateTime.UnixEpoch) throw new UserInformationException("The local database has no fileset times. Consider deleting the local database and run the repair operation again.", "LocalDatabaseHasNoFilesetTimes"); if (mostRecentRemote > mostRecentLocal)