haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #5955 from duplicati/feature/alternate-source-provider

Browse Source 
Alternate source provider
This commit is contained in:
Kenneth Skovhede
2025-02-28 10:07:53 +01:00
committed by GitHub
parent 9e74180b73 11a27042bc
commit e614b08bb3
103 changed files with 4596 additions and 1255 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.vscode/launch.json
Vendored
+8 -1
View File
@@ -114,6 +114,13 @@
"stopAtEntry": false,
"console": "internalConsole"
},
{
"name": "Launch SourceTool executable",
"type": "coreclr",
"request": "launch",
"preLaunchTask": "build",
"program": "${workspaceFolder}/Executables/net8/Duplicati.CommandLine.SourceTool/bin/Debug/net8.0/Duplicati.CommandLine.SourceTool",
"args": []
}
]
}
Duplicati.sln
+29
View File
@@ -166,12 +166,22 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.Crashlog"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.Backend.CIFS", "Duplicati\Library\Backend\CIFS\Duplicati.Library.Backend.CIFS.csproj", "{836E0557-B40C-4DC7-9A2A-5C062F9ACC6B}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.SourceProviders", "Duplicati\Library\SourceProviders\Duplicati.Library.SourceProviders.csproj", "{8A2C7A9F-3EC8-4DE5-A9FA-9E4BCF955EF3}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SourceProviders", "SourceProviders", "{424CEF73-4984-430E-9C8B-E61CC0F22074}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.SourceProvider.Builtin", "Duplicati\Library\SourceProvider\Builtin\Duplicati.Library.SourceProvider.Builtin.csproj", "{6D9742A7-6F32-4571-8276-79716168B525}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tools", "Tools", "{15388C37-9218-4818-972E-738EEA8F1602}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RemoteSynchronization", "Tools\RemoteSynchronization\RemoteSynchronization.csproj", "{D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.CommandLine.SyncTool", "Executables\net8\Duplicati.CommandLine.SyncTool\Duplicati.CommandLine.SyncTool.csproj", "{48D32674-0FE9-4407-B102-4AE46D93595F}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.CommandLine.SourceTool", "Duplicati\CommandLine\SourceTool\Duplicati.CommandLine.SourceTool.csproj", "{2873BCE2-2EA6-419D-BFFA-693FDB34926E}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.CommandLine.SourceTool", "Executables\net8\Duplicati.CommandLine.SourceTool\Duplicati.CommandLine.SourceTool.csproj", "{4003BF90-6681-4155-9D06-639ED23FE3AB}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@@ -478,6 +488,14 @@ Global
{836E0557-B40C-4DC7-9A2A-5C062F9ACC6B}.Debug|Any CPU.Build.0 = Debug|Any CPU
{836E0557-B40C-4DC7-9A2A-5C062F9ACC6B}.Release|Any CPU.ActiveCfg = Release|Any CPU
{836E0557-B40C-4DC7-9A2A-5C062F9ACC6B}.Release|Any CPU.Build.0 = Release|Any CPU
{8A2C7A9F-3EC8-4DE5-A9FA-9E4BCF955EF3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{8A2C7A9F-3EC8-4DE5-A9FA-9E4BCF955EF3}.Debug|Any CPU.Build.0 = Debug|Any CPU
{8A2C7A9F-3EC8-4DE5-A9FA-9E4BCF955EF3}.Release|Any CPU.ActiveCfg = Release|Any CPU
{8A2C7A9F-3EC8-4DE5-A9FA-9E4BCF955EF3}.Release|Any CPU.Build.0 = Release|Any CPU
{6D9742A7-6F32-4571-8276-79716168B525}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{6D9742A7-6F32-4571-8276-79716168B525}.Debug|Any CPU.Build.0 = Debug|Any CPU
{6D9742A7-6F32-4571-8276-79716168B525}.Release|Any CPU.ActiveCfg = Release|Any CPU
{6D9742A7-6F32-4571-8276-79716168B525}.Release|Any CPU.Build.0 = Release|Any CPU
{D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -486,6 +504,14 @@ Global
{48D32674-0FE9-4407-B102-4AE46D93595F}.Debug|Any CPU.Build.0 = Debug|Any CPU
{48D32674-0FE9-4407-B102-4AE46D93595F}.Release|Any CPU.ActiveCfg = Release|Any CPU
{48D32674-0FE9-4407-B102-4AE46D93595F}.Release|Any CPU.Build.0 = Release|Any CPU
{2873BCE2-2EA6-419D-BFFA-693FDB34926E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{2873BCE2-2EA6-419D-BFFA-693FDB34926E}.Debug|Any CPU.Build.0 = Debug|Any CPU
{2873BCE2-2EA6-419D-BFFA-693FDB34926E}.Release|Any CPU.ActiveCfg = Release|Any CPU
{2873BCE2-2EA6-419D-BFFA-693FDB34926E}.Release|Any CPU.Build.0 = Release|Any CPU
{4003BF90-6681-4155-9D06-639ED23FE3AB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{4003BF90-6681-4155-9D06-639ED23FE3AB}.Debug|Any CPU.Build.0 = Debug|Any CPU
{4003BF90-6681-4155-9D06-639ED23FE3AB}.Release|Any CPU.ActiveCfg = Release|Any CPU
{4003BF90-6681-4155-9D06-639ED23FE3AB}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@@ -546,8 +572,11 @@ Global
{CA1A0EE8-DFFF-42AF-B15D-EF538357A2DE} = {E1A9B303-F281-45C5-A4F6-CADD9DE3F3C4}
{8ACA2736-3C69-4FA5-BCF9-5EDF50CAF332} = {566EBBDA-19A4-4056-A615-D901D57D2439}
{836E0557-B40C-4DC7-9A2A-5C062F9ACC6B} = {E1A9B303-F281-45C5-A4F6-CADD9DE3F3C4}
{6D9742A7-6F32-4571-8276-79716168B525} = {424CEF73-4984-430E-9C8B-E61CC0F22074}
{D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D} = {15388C37-9218-4818-972E-738EEA8F1602}
{48D32674-0FE9-4407-B102-4AE46D93595F} = {6B46F6B1-1898-49B8-ADA7-5CAF68EB77E3}
{2873BCE2-2EA6-419D-BFFA-693FDB34926E} = {D19A38DD-68F1-4EF5-BF5F-8966CE0D9A5B}
{4003BF90-6681-4155-9D06-639ED23FE3AB} = {6B46F6B1-1898-49B8-ADA7-5CAF68EB77E3}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {8B40BAFE-D862-4397-9495-8F5EAF5CE80C}
Duplicati/CommandLine/CLI/Commands.cs
+1 -1
View File
@@ -622,7 +622,7 @@ namespace Duplicati.CommandLine
switch (phase)
{
case Duplicati.Library.Main.OperationPhase.Backup_ProcessingFiles:
output.MessageEvent("Scanning local files ...");
output.MessageEvent("Scanning source files ...");
periodicOutput.SetReady();
break;
case Duplicati.Library.Main.OperationPhase.Backup_Finalize:
Duplicati/CommandLine/SourceTool/CommandExtensions.cs
+42
View File
@@ -0,0 +1,42 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System.CommandLine;
using System.CommandLine.Invocation;
namespace Duplicati.CommandLine.SourceTool;
/// <summary>
/// Extensions for <see cref="Command"/>.
/// </summary>
public static class CommandExtensions
{
/// <summary>
/// Adds the missing WithHandler method to <see cref="Command"/>.
/// </summary>
/// <param name="command">The command to add the handler to.</param>
/// <param name="handler">The handler to add.</param>
/// <returns>The command with the handler added.</returns>
public static Command WithHandler(this Command command, ICommandHandler handler)
{
command.Handler = handler;
return command;
}
}
Duplicati/CommandLine/SourceTool/Commands/Download.cs
+110
View File
@@ -0,0 +1,110 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System.CommandLine;
using System.CommandLine.NamingConventionBinder;
namespace Duplicati.CommandLine.SourceTool.Commands;
/// <summary>
/// The download command
/// </summary>
public static class Download
{
/// <summary>
/// Creates the download command
/// </summary>
/// <returns>The download command</returns>
public static Command Create() =>
new Command("download", "Downloads all files from the remote")
{
new Argument<string>("url", "The source URL") {
Arity = ArgumentArity.ExactlyOne
},
new Option<DirectoryInfo>("--destination", description: "The destination folder", getDefaultValue: () => new DirectoryInfo(Directory.GetCurrentDirectory())),
new Option<int>("--max-depth", description: "The maximum depth to visit", getDefaultValue: () => 0),
new Option<long>("--max-size", description: "The maximum filesize to download", getDefaultValue: () => 0),
new Option<bool>("--overwrite", description: "Overwrite existing files", getDefaultValue: () => false),
}
.WithHandler(CommandHandler.Create<string, DirectoryInfo, int, long, bool>(async (url, destination, maxdepth, maxsize, overwrite) =>
{
var token = new CancellationTokenSource().Token;
using var source = await Common.GetProvider(url);
string localPath(string path)
{
var relpath = path.Substring(source.MountedPath.Length).TrimStart(Path.DirectorySeparatorChar);
return Path.Combine(destination.FullName, relpath);
}
var started = DateTime.UtcNow;
var fileCount = 0L;
var folderCount = 0L;
var totalSize = 0L;
var downloadCount = 0L;
var downloadSize = 0L;
await Common.Visit(source, maxdepth, async (entry, level) =>
{
if (entry.IsMetaEntry)
return true;
var path = localPath(entry.Path);
if (entry.IsFolder)
{
folderCount++;
if (!Directory.Exists(path))
Directory.CreateDirectory(path);
}
else
{
fileCount++;
totalSize += entry.Size;
if (entry.Size <= maxsize || maxsize <= 0)
{
var folder = Path.GetDirectoryName(path);
if (!string.IsNullOrWhiteSpace(folder) && !Directory.Exists(folder))
Directory.CreateDirectory(folder);
if (!overwrite && File.Exists(path))
{
Console.WriteLine($"Skipping {entry.Path} as it already exists");
}
else
{
downloadCount++;
downloadSize += entry.Size;
Console.WriteLine($"Downloading {entry.Path} to {path} ({Library.Utility.Utility.FormatSizeString(entry.Size)})");
await using var stream = await entry.OpenRead(token);
await using var dest = File.OpenWrite(path);
await stream.CopyToAsync(dest, token);
}
}
}
return true;
}, token);
Console.WriteLine($"Found {fileCount} files and {folderCount} folders with a total size of {Library.Utility.Utility.FormatSizeString(totalSize)}");
Console.WriteLine($"Downloaded {downloadCount} files with a total size of {Library.Utility.Utility.FormatSizeString(downloadSize)} in {DateTime.UtcNow - started}");
}));
}
Duplicati/CommandLine/SourceTool/Commands/List.cs
+67
View File
@@ -0,0 +1,67 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System.CommandLine;
using System.CommandLine.NamingConventionBinder;
namespace Duplicati.CommandLine.SourceTool.Commands;
/// <summary>
/// The list command
/// </summary>
public static class List
{
/// <summary>
/// Creates the list command
/// </summary>
/// <returns>The command</returns>
public static Command Create() =>
new Command("list", "Lists all paths on the remote")
{
new Argument<string>("url", "The source URL") {
Arity = ArgumentArity.ExactlyOne
},
new Option<int>("--max-depth", description: "The maximum depth to list", getDefaultValue: () => 0),
}
.WithHandler(CommandHandler.Create<string, int>(async (url, maxdepth) =>
{
var token = new CancellationTokenSource().Token;
var folders = 0L;
var files = 0L;
using var source = await Common.GetProvider(url);
await Common.Visit(source, maxdepth, (entry, level) =>
{
if (entry.IsFolder)
folders++;
else
files++;
if (entry.IsFolder && level > 0)
level--;
if (!entry.IsRootEntry)
Console.WriteLine($"{new string(' ', (level + 1) * 2)}{entry.Path}");
return Task.FromResult(true);
}, token);
Console.WriteLine($"Found {folders} folders and {files} files");
})
);
}
Duplicati/CommandLine/SourceTool/Duplicati.CommandLine.SourceTool.csproj
+25
View File
@@ -0,0 +1,25 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<AssemblyName>Duplicati.CommandLine.SourceTool.Implementation</AssemblyName>
<Copyright>Copyright © 2025 Team Duplicati, MIT license</Copyright>
<RootNamespace>Duplicati.CommandLine.Sourcetool</RootNamespace>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
<PackageReference Include="System.CommandLine.NamingConventionBinder" Version="2.0.0-beta4.22272.1" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="../../Library/DynamicLoader/Duplicati.Library.DynamicLoader.csproj" />
<ProjectReference Include="../../Library/SourceProvider/Builtin/Duplicati.Library.SourceProvider.Builtin.csproj" />
<ProjectReference Include="../../Library/SourceProviders/Duplicati.Library.SourceProviders.csproj" />
<ProjectReference Include="../../Library/Utility/Duplicati.Library.Utility.csproj" />
</ItemGroup>
</Project>
Duplicati/CommandLine/SourceTool/GetProvider.cs
+39
View File
@@ -0,0 +1,39 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.DynamicLoader;
using Duplicati.Library.Interface;
namespace Duplicati.CommandLine.SourceTool;
/// <summary>
/// Common methods for the source tool
/// </summary>
public static partial class Common
{
/// <summary>
/// Gets the source provider for the given URL
/// </summary>
/// <param name="url">The URL to get the provider for</param>
/// <returns>The source provider</returns>
public static async Task<ISourceProvider> GetProvider(string url) =>
(await SourceProviderLoader.GetSourceProvider(url, "", [], CancellationToken.None))
?? throw new UserInformationException(string.Format("The source \"{0}\" is not supported", url), "SourceNotSupported");
}
Duplicati/CommandLine/SourceTool/ListEnumerator.cs
+60
View File
@@ -0,0 +1,60 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Interface;
namespace Duplicati.CommandLine.SourceTool;
/// <summary>
/// Common methods for the source tool
/// </summary>
public static partial class Common
{
/// <summary>
/// Visits all entries in the source provider
/// </summary>
/// <param name="source">The source provider to visit</param>
/// <param name="maxdepth">The maximum depth to visit</param>
/// <param name="visitor">The visitor function</param>
/// <param name="token">The cancellation token</param>
/// <returns>An awaitable task</returns>
public static async Task Visit(ISourceProvider source, int maxdepth, Func<ISourceProviderEntry, int, Task<bool>> visitor, CancellationToken token)
{
var visit = new Stack<(ISourceProviderEntry Entry, int Level)>();
await foreach (var item in source.Enumerate(token))
visit.Push((item, 0));
while (visit.Count() != 0)
{
var item = visit.Pop();
var process = await visitor(item.Entry, item.Level);
if (item.Entry.IsFolder && process && (item.Level < maxdepth || maxdepth <= 0))
{
await foreach (var subitem in item.Entry.Enumerate(token))
{
if (subitem.IsFolder)
visit.Push((subitem, item.Level + 1));
else
await visitor(subitem, item.Level);
}
}
}
}
}
Duplicati/CommandLine/SourceTool/Program.cs
+68
View File
@@ -0,0 +1,68 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System.CommandLine;
using System.CommandLine.Builder;
using System.CommandLine.Parsing;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
namespace Duplicati.CommandLine.SourceTool;
/// <summary>
/// The entry point of the application
/// </summary>
public static class Program
{
/// <summary>
/// Executes the program
/// </summary>
/// <param name="args">The arguments</param>
/// <returns>The return code</returns>
public static Task<int> Main(string[] args)
{
Library.AutoUpdater.PreloadSettingsLoader.ConfigurePreloadSettings(ref args, Library.AutoUpdater.PackageHelper.NamedExecutable.ServerUtil);
var rootCmd = new RootCommand("Source CLI tool for Duplicati")
{
Commands.List.Create(),
Commands.Download.Create(),
};
return new CommandLineBuilder(rootCmd)
.UseDefaults()
.UseExceptionHandler((ex, context) =>
{
if (ex is UserInformationException uie)
{
Console.WriteLine(uie.Message);
context.ExitCode = 2;
}
else
{
Console.WriteLine(ex.ToString());
context.ExitCode = 1;
}
})
.UseAdditionalHelpAliases()
.Build()
.InvokeAsync(args);
}
}
Duplicati/Library/AutoUpdater/PackageHelper.cs
+6 -1
View File
@@ -92,7 +92,11 @@ namespace Duplicati.Library.AutoUpdater
/// <summary>
/// The sync tool
/// </summary>
SyncTool
SyncTool,
/// <summary>
/// The source tool
/// </summary>
SourceTool
}
@@ -120,6 +124,7 @@ namespace Duplicati.Library.AutoUpdater
NamedExecutable.Agent => OperatingSystem.IsWindows() ? "Duplicati.Agent.exe" : "duplicati-agent",
NamedExecutable.SecretTool => OperatingSystem.IsWindows() ? "Duplicati.CommandLine.SecretTool.exe" : "duplicati-secret-tool",
NamedExecutable.SyncTool => OperatingSystem.IsWindows() ? "Duplicati.CommandLine.SyncTool.exe" : "duplicati-sync-tool",
NamedExecutable.SourceTool => OperatingSystem.IsWindows() ? "Duplicati.CommandLine.Sourcetool.exe" : "duplicati-source-tool",
_ => throw new ArgumentException($"Named executable not known: {exe}", nameof(exe))
};
Duplicati/Library/Backend/CIFS/CIFS.cs
+24 -3
View File
@@ -29,6 +29,9 @@ using System.Threading.Tasks;
using Duplicati.Library.Backend.CIFS;
using Duplicati.Library.Backend.CIFS.Model;
using SMBLibrary;
using Duplicati.Library.SourceProvider;
using System.Runtime.CompilerServices;
using Duplicati.Library.Common.IO;
using System.Runtime.CompilerServices;
namespace Duplicati.Library.Backend;
@@ -36,7 +39,7 @@ namespace Duplicati.Library.Backend;
/// <summary>
/// Native CIFS/SMB Backend implementation
/// </summary>
public class CIFSBackend : IStreamingBackend
public class CIFSBackend : IStreamingBackend, IFolderEnabledBackend
{
/// <summary>
/// Implementation of interface property for the backend key
@@ -198,7 +201,7 @@ public class CIFSBackend : IStreamingBackend
/// <returns>List of IFileEntry with directory listing result</returns>
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancellationToken)
{
foreach (var v in await GetConnection().ListAsync(cancellationToken).ConfigureAwait(false))
foreach (var v in await GetConnection().ListAsync(_connectionParameters.Path, cancellationToken).ConfigureAwait(false))
yield return v;
}
@@ -288,7 +291,7 @@ public class CIFSBackend : IStreamingBackend
public async Task TestAsync(CancellationToken cancellationToken)
{
// This will throw an exception if the folder is missing
await GetConnection().ListAsync(cancellationToken).ConfigureAwait(false);
await GetConnection().ListAsync(_connectionParameters.Path, cancellationToken).ConfigureAwait(false);
}
/// <summary>
@@ -315,6 +318,8 @@ public class CIFSBackend : IStreamingBackend
/// <returns>An SMB connection that can be used for file operations</returns>
private SMBShareConnection GetConnection() => _sharedConnection ??= new SMBShareConnection(_connectionParameters);
/// <summary>
/// Implementation of Dispose pattern enforced by interface
/// </summary>
@@ -330,4 +335,20 @@ public class CIFSBackend : IStreamingBackend
System.Diagnostics.Debug.WriteLine($"Error disposing CIFS connection: {ex.Message}");
}
}
/// <inheritdoc/>
public async IAsyncEnumerable<IFileEntry> ListAsync(string path, [EnumeratorCancellation] CancellationToken cancellationToken)
{
var sourcePath = _connectionParameters.Path;
if (!string.IsNullOrWhiteSpace(sourcePath))
sourcePath = Util.AppendDirSeparator(sourcePath, "/");
foreach (var v in await GetConnection().ListAsync(sourcePath + BackendSourceFileEntry.NormalizePathTo(path, '/'), cancellationToken).ConfigureAwait(false))
if (v.Name != "." && v.Name != "..")
yield return v;
}
/// <inheritdoc/>
public Task<IFileEntry> GetEntryAsync(string path, CancellationToken cancellationToken)
=> Task.FromResult<IFileEntry>(null);
}
Duplicati/Library/Backend/CIFS/Duplicati.Library.Backend.CIFS.csproj
+1
View File
@@ -10,6 +10,7 @@
<ProjectReference Include="..\..\Interface\Duplicati.Library.Interface.csproj" />
<ProjectReference Include="..\..\Localization\Duplicati.Library.Localization.csproj" />
<ProjectReference Include="..\..\Utility\Duplicati.Library.Utility.csproj" />
<ProjectReference Include="..\..\SourceProvider\Builtin\Duplicati.Library.SourceProvider.Builtin.csproj" />
</ItemGroup>
<ItemGroup>
Duplicati/Library/Backend/CIFS/SMBShareConnection.cs
+8 -7
View File
@@ -79,7 +79,7 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable
if (!_smb2Client.Connect(connectionParameters.ServerName, connectionParameters.TransportType))
throw new UserInformationException($"{LC.L("Failed to connect to server")} {connectionParameters.ServerName}", "ConnectionError");
var status = _smb2Client.Login(connectionParameters.AuthDomain, connectionParameters.AuthUser, connectionParameters.AuthPassword);
var status = _smb2Client.Login(connectionParameters.AuthDomain ?? "", connectionParameters.AuthUser ?? "", connectionParameters.AuthPassword ?? "");
if (status != NTStatus.STATUS_SUCCESS)
throw new UserInformationException($"{LC.L("Failed to authenticate to server")} {connectionParameters.ServerName} with status {status}", "ConnectionError");
@@ -194,9 +194,10 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable
/// <summary>
/// Lists the folder contents of the share and path specified in the connection parameters.
/// </summary>
/// <param name="path">Path to list</param>
/// <param name="cancellationToken">Cancellation Token</param>
/// <exception cref="UserInformationException">Exception to be displayed to user</exception>
public async Task<List<IFileEntry>> ListAsync(CancellationToken cancellationToken)
public async Task<List<IFileEntry>> ListAsync(string path, CancellationToken cancellationToken)
{
await _semaphore.WaitAsync(cancellationToken);
@@ -206,11 +207,10 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable
{
try
{
var status = _smbFileStore.CreateFile(
out directoryHandle,
out fileStatus,
NormalizeSlashes(_connectionParameters.Path),
NormalizeSlashes(path),
AccessMask.GENERIC_READ,
FileAttributes.Directory,
ShareAccess.Read | ShareAccess.Write,
@@ -222,7 +222,7 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable
if (status == NTStatus.STATUS_OBJECT_PATH_NOT_FOUND || status == NTStatus.STATUS_OBJECT_NAME_NOT_FOUND)
throw new FolderMissingException();
else
throw new UserInformationException($"{LC.L("Failed to open directory")} {NormalizeSlashes(_connectionParameters.Path)} with status {status.ToString()}", "DirectoryOpenError");
throw new UserInformationException($"{LC.L("Failed to open directory")} {NormalizeSlashes(path)} with status {status}", "DirectoryOpenError");
List<QueryDirectoryFileInformation> fileList;
status = _smbFileStore.QueryDirectory(
@@ -232,7 +232,7 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable
FileInformationClass.FileDirectoryInformation);
if (status != NTStatus.STATUS_NO_MORE_FILES)
throw new UserInformationException($"{LC.L("Failed to query directory contents")} with status {status.ToString()}", "DirectoryQueryError");
throw new UserInformationException($"{LC.L("Failed to query directory contents")} with status {status}", "DirectoryQueryError");
return
[
@@ -244,7 +244,8 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable
info.LastAccessTime,
info.LastWriteTime)
{
IsFolder = info.FileAttributes == FileAttributes.Directory
IsFolder = info.FileAttributes == FileAttributes.Directory,
Created = info.CreationTime
})
.ToList()
];
Duplicati/Library/Backend/File/Duplicati.Library.Backend.File.csproj
+1
View File
@@ -11,6 +11,7 @@
<ProjectReference Include="..\..\Interface\Duplicati.Library.Interface.csproj" />
<ProjectReference Include="..\..\Localization\Duplicati.Library.Localization.csproj" />
<ProjectReference Include="..\..\Utility\Duplicati.Library.Utility.csproj" />
<ProjectReference Include="..\..\Snapshots\Duplicati.Library.Snapshots.csproj" />
</ItemGroup>
<ItemGroup>
Duplicati/Library/Backend/File/FileBackend.cs
+22 -21
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Common.IO;
@@ -451,14 +451,15 @@ namespace Duplicati.Library.Backend
long? sourceStreamLength = sourceStream == null ? null : Utility.Utility.GetStreamLength(sourceStream, out isStreamPostion);
if (sourceStreamLength.HasValue && targetFileInfo.Length != sourceStreamLength.Value)
throw new FileMissingException($"Target file size ({targetFileInfo.Length:n0}) is different from the source length ({sourceStreamLength.Value:n0}){(isStreamPostion ? " - ending stream position)" : "")}. Target: {targetFilePath}");
throw new FileMissingException($"Target file size ({targetFileInfo.Length:n0}) is different from the source length ({sourceStreamLength.Value:n0}){(isStreamPostion ? " - ending stream position" : "")}. Target: {targetFilePath}");
if (expectedLength.HasValue && targetFileInfo.Length != expectedLength.Value)
throw new FileMissingException($"Target file size ({targetFileInfo.Length:n0}) is different from the expected length ({expectedLength.Value:n0}). Target: {targetFilePath}");
}
catch
{
try { System.IO.File.Delete(targetFilePath); } catch { }
try { System.IO.File.Delete(targetFilePath); }
catch { }
throw;
}
}
Duplicati/Library/Backend/Idrivee2/Duplicati.Library.Backend.Idrivee2.csproj
+1
View File
@@ -8,6 +8,7 @@
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="../../SourceProvider/Builtin/Duplicati.Library.SourceProvider.Builtin.csproj" />
<ProjectReference Include="..\..\Interface\Duplicati.Library.Interface.csproj" />
<ProjectReference Include="..\..\Localization\Duplicati.Library.Localization.csproj" />
<ProjectReference Include="..\..\Utility\Duplicati.Library.Utility.csproj" />
Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs
+17 -11
View File
@@ -31,7 +31,7 @@ using System.Threading.Tasks;
namespace Duplicati.Library.Backend
{
public class Idrivee2Backend : IBackend, IStreamingBackend
public class Idrivee2Backend : IBackend, IStreamingBackend, IFolderEnabledBackend
{
private static readonly string LOGTAG = Logging.Log.LogTagFromType<Idrivee2Backend>();
@@ -80,7 +80,6 @@ namespace Duplicati.Library.Backend
m_s3Client = new S3AwsClient(accessKeyId, accessKeySecret, null, host, null, true, false, options);
}
public string GetRegionEndpoint(string url)
@@ -125,18 +124,10 @@ namespace Duplicati.Library.Backend
public bool SupportsStreaming => true;
/// <inheritdoc />
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancelToken)
{
await foreach (IFileEntry file in Connection.ListBucketAsync(m_bucket, m_prefix, cancelToken).ConfigureAwait(false))
{
((FileEntry)file).Name = file.Name.Substring(m_prefix.Length);
if (file.Name.StartsWith("/", StringComparison.Ordinal) && !m_prefix.StartsWith("/", StringComparison.Ordinal))
((FileEntry)file).Name = file.Name.Substring(1);
await foreach (IFileEntry file in Connection.ListBucketAsync(m_bucket, m_prefix, false, cancelToken).ConfigureAwait(false))
yield return file;
}
}
public async Task PutAsync(string remotename, string localname, CancellationToken cancelToken)
@@ -228,5 +219,20 @@ namespace Duplicati.Library.Backend
//AWS SDK encodes the filenames correctly
return m_prefix + name;
}
/// <inheritdoc/>
public IAsyncEnumerable<IFileEntry> ListAsync(string path, CancellationToken cancellationToken)
{
var filterPath = GetFullKey(path);
if (!string.IsNullOrWhiteSpace(filterPath))
filterPath = Util.AppendDirSeparator(filterPath, "/");
return m_s3Client.ListBucketAsync(m_bucket, filterPath, true, cancellationToken);
}
/// <inheritdoc/>
public Task<IFileEntry> GetEntryAsync(string path, CancellationToken cancellationToken)
=> Task.FromResult<IFileEntry>(null);
}
}
Duplicati/Library/Backend/S3/Duplicati.Library.Backend.S3.csproj
+1 -1
View File
@@ -12,10 +12,10 @@
<PackageReference Include="AWSSDK.IdentityManagement" Version="3.7.402.39" />
<PackageReference Include="AWSSDK.S3" Version="3.7.405.9" />
<PackageReference Include="Minio" Version="3.1.13" />
<PackageReference Include="System.Reactive.Linq" Version="4.3.2" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="../../SourceProvider/Builtin/Duplicati.Library.SourceProvider.Builtin.csproj" />
<ProjectReference Include="..\..\Interface\Duplicati.Library.Interface.csproj" />
<ProjectReference Include="..\..\Localization\Duplicati.Library.Localization.csproj" />
<ProjectReference Include="..\..\Utility\Duplicati.Library.Utility.csproj" />
Duplicati/Library/Backend/S3/IS3Client.cs
+1 -1
View File
@@ -28,7 +28,7 @@ namespace Duplicati.Library.Backend
{
public interface IS3Client : IDisposable
{
IAsyncEnumerable<IFileEntry> ListBucketAsync(string bucketName, string prefix, CancellationToken cancellationToken);
IAsyncEnumerable<IFileEntry> ListBucketAsync(string bucketName, string prefix, bool recursive, CancellationToken cancellationToken);
Task AddBucketAsync(string bucketName, CancellationToken cancelToken);
Duplicati/Library/Backend/S3/S3AwsClient.cs
+77 -18
View File
@@ -22,6 +22,7 @@
using Amazon.S3;
using Amazon.S3.Model;
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
using System;
@@ -49,6 +50,7 @@ namespace Duplicati.Library.Backend
private readonly bool m_useChunkEncoding;
private readonly string m_dnsHost;
private readonly bool m_useV2ListApi;
public S3AwsClient(string awsID, string awsKey, string locationConstraint, string servername,
string storageClass, bool useSSL, bool disableChunkEncoding, Dictionary<string, string> options)
@@ -61,6 +63,7 @@ namespace Duplicati.Library.Backend
m_client = new AmazonS3Client(awsID, awsKey, cfg);
m_useV2ListApi = string.Equals(options.GetValueOrDefault("list-api-version", "v1"), "v2", StringComparison.OrdinalIgnoreCase);
m_locationConstraint = locationConstraint;
m_storageClass = storageClass;
m_dnsHost = string.IsNullOrWhiteSpace(cfg.ServiceURL) ? null : new System.Uri(cfg.ServiceURL).Host;
@@ -192,32 +195,67 @@ namespace Duplicati.Library.Backend
return m_client.DeleteObjectAsync(objectDeleteRequest, cancellationToken);
}
public async IAsyncEnumerable<IFileEntry> ListBucketAsync(string bucketName, string prefix, [EnumeratorCancellation] CancellationToken cancellationToken)
/// <summary>
/// Lists the contents of a bucket, using a plugable API call
/// </summary>
/// <param name="bucketName">The bucket to list</param>
/// <param name="prefix">The prefix to list</param>
/// <param name="recursive">If true, the list is recursive</param>
/// <param name="cancellationToken">The cancellation token</param>
/// <returns>The list of files</returns>
public async IAsyncEnumerable<IFileEntry> ListBucketAsync(string bucketName, string prefix, bool recursive, [EnumeratorCancellation] CancellationToken cancellationToken)
{
bool isTruncated = true;
var isTruncated = true;
string filename = null;
var delimiter = recursive ? "" : "/";
if (!string.IsNullOrWhiteSpace(prefix))
prefix = Util.AppendDirSeparator(prefix, "/");
//TODO: Figure out if this is the case with AWSSDK too
//Unfortunately S3 sometimes reports duplicate values when requesting more than one page of results
//So, track the files that have already been returned and skip any duplicates.
HashSet<string> alreadyReturned = new HashSet<string>();
var alreadyReturned = new HashSet<string>();
//We truncate after ITEM_LIST_LIMIT elements, and then repeat
while (isTruncated)
{
var listRequest = new ListObjectsRequest { BucketName = bucketName };
if (!string.IsNullOrEmpty(filename))
listRequest.Marker = filename;
listRequest.MaxKeys = ITEM_LIST_LIMIT;
if (!string.IsNullOrEmpty(prefix))
listRequest.Prefix = prefix;
ListObjectsResponse listResponse;
ListObjectsV2Response listResponse;
try
{
listResponse = await m_client.ListObjectsAsync(listRequest, cancellationToken).ConfigureAwait(false);
if (m_useV2ListApi)
{
var listRequest = new ListObjectsV2Request
{
BucketName = bucketName,
Prefix = prefix,
ContinuationToken = filename,
MaxKeys = ITEM_LIST_LIMIT,
Delimiter = delimiter
};
listResponse = await m_client.ListObjectsV2Async(listRequest, cancellationToken).ConfigureAwait(false);
}
else
{
// Use the V1 API
var listRequest = new ListObjectsRequest
{
BucketName = bucketName,
Prefix = prefix,
Marker = filename,
MaxKeys = ITEM_LIST_LIMIT,
Delimiter = delimiter
};
var listResponsev1 = await m_client.ListObjectsAsync(listRequest, cancellationToken).ConfigureAwait(false);
// Map the V1 response to the V2 response
listResponse = new ListObjectsV2Response
{
CommonPrefixes = listResponsev1.CommonPrefixes,
IsTruncated = listResponsev1.IsTruncated,
NextContinuationToken = listResponsev1.NextMarker,
S3Objects = listResponsev1.S3Objects
};
}
}
catch (AmazonS3Exception e)
{
@@ -231,16 +269,37 @@ namespace Duplicati.Library.Backend
}
isTruncated = listResponse.IsTruncated;
filename = listResponse.NextMarker;
filename = listResponse.NextContinuationToken;
foreach (var obj in listResponse.CommonPrefixes)
{
if (obj == prefix || !obj.StartsWith(prefix))
continue;
// Because the prefixes are returned, and not the folder objects
// we do not get the folder modification date
yield return new FileEntry(
obj.Substring(prefix.Length),
-1,
new DateTime(0),
new DateTime(0)
)
{ IsFolder = true };
}
foreach (var obj in listResponse.S3Objects.Where(obj => alreadyReturned.Add(obj.Key)))
{
yield return new Common.IO.FileEntry(
obj.Key,
// Skip self-prefix, this discards the folder modification date :/
if (obj.Key == prefix || !obj.Key.StartsWith(prefix))
continue;
yield return new FileEntry(
obj.Key.Substring(prefix.Length),
obj.Size,
obj.LastModified,
obj.LastModified
);
)
{ IsFolder = obj.Key.EndsWith("/") };
}
}
}
Duplicati/Library/Backend/S3/S3Backend.cs
+24 -16
View File
@@ -33,7 +33,7 @@ using System.Threading.Tasks;
namespace Duplicati.Library.Backend
{
public class S3 : IBackend, IStreamingBackend, IRenameEnabledBackend
public class S3 : IBackend, IStreamingBackend, IRenameEnabledBackend, IFolderEnabledBackend
{
private static readonly string LOGTAG = Logging.Log.LogTagFromType<S3>();
@@ -43,6 +43,8 @@ namespace Duplicati.Library.Backend
private const string SSL_OPTION = "use-ssl";
private const string S3_CLIENT_OPTION = "s3-client";
private const string S3_DISABLE_CHUNK_ENCODING_OPTION = "s3-disable-chunk-encoding";
private const string S3_LIST_API_VERSION_OPTION = "s3-list-api-version";
private const string S3_RECURSIVE_LIST = "s3-recursive-list";
public static readonly Dictionary<string, string> KNOWN_S3_PROVIDERS = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase) {
{ "Amazon S3", "s3.amazonaws.com" },
@@ -175,6 +177,7 @@ namespace Duplicati.Library.Backend
private readonly string m_bucket;
private readonly string m_prefix;
private readonly bool m_recurseLists;
private const string DEFAULT_S3_HOST = "s3.amazonaws.com";
private IS3Client s3Client;
@@ -227,6 +230,8 @@ namespace Duplicati.Library.Backend
if (m_prefix.Length != 0)
m_prefix = Util.AppendDirSeparator(m_prefix, "/");
m_recurseLists = Utility.Utility.ParseBoolOption(options, S3_RECURSIVE_LIST);
// Auto-disable DNS lookup for non-AWS configurations
if (!options.ContainsKey("s3-ext-forcepathstyle") && !hostname.EndsWith(".amazonaws.com", StringComparison.OrdinalIgnoreCase))
options["s3-ext-forcepathstyle"] = "true";
@@ -271,21 +276,8 @@ namespace Duplicati.Library.Backend
get { return true; }
}
/// <inheritdoc />
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancelToken)
{
await foreach (IFileEntry file in Connection.ListBucketAsync(m_bucket, m_prefix, cancelToken).ConfigureAwait(false))
{
((FileEntry)file).Name = file.Name.Substring(m_prefix.Length);
//Fix for a bug in Duplicati 1.0 beta 3 and earlier, where filenames are incorrectly prefixed with a slash
if (file.Name.StartsWith("/", StringComparison.Ordinal) && !m_prefix.StartsWith("/", StringComparison.Ordinal))
((FileEntry)file).Name = file.Name.Substring(1);
yield return file;
}
}
public IAsyncEnumerable<IFileEntry> ListAsync(CancellationToken cancelToken)
=> Connection.ListBucketAsync(m_bucket, m_prefix, m_recurseLists, cancelToken);
public async Task PutAsync(string remotename, string localname, CancellationToken cancelToken)
{
@@ -337,6 +329,8 @@ namespace Duplicati.Library.Backend
new CommandLineArgument(SSL_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.S3Backend.DescriptionUseSSLShort, Strings.S3Backend.DescriptionUseSSLLong),
new CommandLineArgument(S3_CLIENT_OPTION, CommandLineArgument.ArgumentType.Enumeration, Strings.S3Backend.S3ClientDescriptionShort, Strings.S3Backend.S3ClientDescriptionLong, "aws", null, new string[] { "aws", "minio" }),
new CommandLineArgument(S3_DISABLE_CHUNK_ENCODING_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.S3Backend.DescriptionDisableChunkEncodingShort, Strings.S3Backend.DescriptionDisableChunkEncodingLong, "false"),
new CommandLineArgument(S3_LIST_API_VERSION_OPTION, CommandLineArgument.ArgumentType.Enumeration, Strings.S3Backend.DescriptionListApiVersionShort, Strings.S3Backend.DescriptionListApiVersionLong, "v1", null, ["v1", "v2"]),
new CommandLineArgument(S3_RECURSIVE_LIST, CommandLineArgument.ArgumentType.Boolean, Strings.S3Backend.DescriptionRecursiveListShort, Strings.S3Backend.DescriptionRecursiveListLong, "false"),
new CommandLineArgument("auth-password", CommandLineArgument.ArgumentType.Password, Strings.S3Backend.AuthPasswordDescriptionShort, Strings.S3Backend.AuthPasswordDescriptionLong),
new CommandLineArgument("auth-username", CommandLineArgument.ArgumentType.String, Strings.S3Backend.AuthUsernameDescriptionShort, Strings.S3Backend.AuthUsernameDescriptionLong),
};
@@ -396,5 +390,19 @@ namespace Duplicati.Library.Backend
//AWS SDK encodes the filenames correctly
return m_prefix + name;
}
/// <inheritdoc/>
public IAsyncEnumerable<IFileEntry> ListAsync(string path, CancellationToken cancellationToken)
{
var filterPath = GetFullKey(path);
if (!string.IsNullOrWhiteSpace(filterPath))
filterPath = Util.AppendDirSeparator(filterPath, "/");
return s3Client.ListBucketAsync(m_bucket, filterPath, m_recurseLists, cancellationToken);
}
/// <inheritdoc/>
public Task<IFileEntry> GetEntryAsync(string path, CancellationToken cancellationToken)
=> Task.FromResult<IFileEntry>(null);
}
}
Duplicati/Library/Backend/S3/S3MinioClient.cs
+13 -5
View File
@@ -26,6 +26,7 @@ using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Channels;
using System.Threading.Tasks;
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
using Minio;
@@ -81,19 +82,26 @@ namespace Duplicati.Library.Backend
}
}
public async IAsyncEnumerable<IFileEntry> ListBucketAsync(string bucketName, string prefix, [EnumeratorCancellation] CancellationToken cancellationToken)
public async IAsyncEnumerable<IFileEntry> ListBucketAsync(string bucketName, string prefix, bool recursive, [EnumeratorCancellation] CancellationToken cancellationToken)
{
ThrowExceptionIfBucketDoesNotExist(bucketName);
var observable = m_client.ListObjectsAsync(bucketName, prefix, true, cancellationToken);
if (!string.IsNullOrWhiteSpace(prefix))
prefix = Util.AppendDirSeparator(prefix, "/");
var observable = m_client.ListObjectsAsync(bucketName, prefix, recursive, cancellationToken);
await foreach (var obj in ToAsyncEnumerable(observable, cancellationToken).ConfigureAwait(false))
{
yield return new Common.IO.FileEntry(
obj.Key,
if (obj.Key == prefix || !obj.Key.StartsWith(prefix))
continue;
yield return new FileEntry(
obj.Key.Substring(prefix.Length),
(long)obj.Size,
Convert.ToDateTime(obj.LastModified),
Convert.ToDateTime(obj.LastModified)
);
)
{ IsFolder = obj.Key.EndsWith("/") };
}
}
Duplicati/Library/Backend/S3/Strings.cs
+23 -19
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Localization.Short;
namespace Duplicati.Library.Backend.Strings
@@ -49,6 +49,10 @@ namespace Duplicati.Library.Backend.Strings
public static string DescriptionDisableChunkEncodingShort { get { return LC.L(@"Disable chunk encoding (aws client only)"); } }
public static string S3StorageclassDescriptionLong { get { return LC.L(@"Use this option to specify a storage class. If this option is not used, the server will choose a default storage class."); } }
public static string S3StorageclassDescriptionShort { get { return LC.L(@"Specify storage class"); } }
public static string DescriptionListApiVersionShort { get { return LC.L(@"Specify the S3 list API version to use"); } }
public static string DescriptionListApiVersionLong { get { return LC.L(@"Use this option to specify the S3 list API version to use. This can be used to work around issues with some S3 providers."); } }
public static string DescriptionRecursiveListShort { get { return LC.L(@"Use this option to list all files in the bucket"); } }
public static string DescriptionRecursiveListLong { get { return LC.L(@"To reduce the number of objects listed, the default is to only list the first level of objects. Use this option to list all objects in the bucket."); } }
public static string UnknownS3ClientError(string client) { return LC.L(@"Unknown S3 client: {0}", client); }
}
}
Duplicati/Library/Backend/SSHv2/Duplicati.Library.Backend.SSHv2.csproj
+1
View File
@@ -12,6 +12,7 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="../../SourceProvider/Builtin/Duplicati.Library.SourceProvider.Builtin.csproj" />
<ProjectReference Include="..\..\Interface\Duplicati.Library.Interface.csproj" />
<ProjectReference Include="..\..\Localization\Duplicati.Library.Localization.csproj" />
<ProjectReference Include="..\..\Utility\Duplicati.Library.Utility.csproj" />
Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs
+69 -15
View File
@@ -22,6 +22,7 @@
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.SourceProvider;
using Renci.SshNet;
using Renci.SshNet.Common;
using System;
@@ -34,11 +35,12 @@ using System.Threading.Tasks;
namespace Duplicati.Library.Backend
{
public class SSHv2 : IStreamingBackend, IRenameEnabledBackend
public class SSHv2 : IStreamingBackend, IRenameEnabledBackend, IFolderEnabledBackend
{
private const string SSH_KEYFILE_OPTION = "ssh-keyfile";
private const string SSH_KEYFILE_INLINE = "ssh-key";
private const string SSH_FINGERPRINT_OPTION = "ssh-fingerprint";
private const string SSH_RELATIVE_PATH_OPTION = "ssh-relative-path";
private const string SSH_FINGERPRINT_ACCEPT_ANY_OPTION = "ssh-accept-any-fingerprints";
public const string KEYFILE_URI = "sshkey://";
private const string SSH_TIMEOUT_OPTION = "ssh-operation-timeout";
@@ -56,6 +58,8 @@ namespace Duplicati.Library.Backend
private readonly TimeSpan m_keepaliveinterval;
private readonly int m_port = 22;
private readonly bool m_useRelativePath;
private string m_initialDirectory;
private SftpClient m_con;
@@ -82,6 +86,7 @@ namespace Duplicati.Library.Backend
m_password = uri.Password;
m_fingerprintallowall = Utility.Utility.ParseBoolOption(options, SSH_FINGERPRINT_ACCEPT_ANY_OPTION);
m_useRelativePath = Utility.Utility.ParseBoolOption(options, SSH_RELATIVE_PATH_OPTION);
m_path = uri.Path;
@@ -90,8 +95,15 @@ namespace Duplicati.Library.Backend
m_path = Util.AppendDirSeparator(m_path, "/");
}
if (!m_path.StartsWith("/", StringComparison.Ordinal))
m_path = "/" + m_path;
if (m_useRelativePath)
{
m_path = m_path.TrimStart('/');
}
else
{
if (!m_path.StartsWith("/", StringComparison.Ordinal))
m_path = "/" + m_path;
}
m_server = uri.Host;
@@ -163,7 +175,7 @@ namespace Duplicati.Library.Backend
try
{
await CreateConnection(cancelToken).ConfigureAwait(false);
await ChangeDirectory(m_path).ConfigureAwait(false);
await SetWorkingDirectory(cancelToken).ConfigureAwait(false);
await m_con.DeleteFileAsync(remotename, cancelToken).ConfigureAwait(false);
}
catch (SftpPathNotFoundException ex)
@@ -202,6 +214,9 @@ namespace Duplicati.Library.Backend
new CommandLineArgument(SSH_KEEPALIVE_OPTION, CommandLineArgument.ArgumentType.Timespan,
Strings.SSHv2Backend.DescriptionSshkeepaliveShort,
Strings.SSHv2Backend.DescriptionSshkeepaliveLong, "0"),
new CommandLineArgument(SSH_RELATIVE_PATH_OPTION, CommandLineArgument.ArgumentType.Boolean,
Strings.SSHv2Backend.DescriptionRelativePathShort,
Strings.SSHv2Backend.DescriptionRelativePathLong)
});
}
}
@@ -239,7 +254,7 @@ namespace Duplicati.Library.Backend
public async Task PutAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken)
{
await CreateConnection(cancelToken).ConfigureAwait(false);
await ChangeDirectory(m_path).ConfigureAwait(false);
await SetWorkingDirectory(cancelToken).ConfigureAwait(false);
await Task.Factory.FromAsync(
(cb, state) => m_con.BeginUploadFile(stream, remotename, cb, state, _ => cancelToken.ThrowIfCancellationRequested()),
m_con.EndUploadFile,
@@ -249,7 +264,7 @@ namespace Duplicati.Library.Backend
public async Task GetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken)
{
await CreateConnection(cancelToken).ConfigureAwait(false);
await ChangeDirectory(m_path).ConfigureAwait(false);
await SetWorkingDirectory(cancelToken).ConfigureAwait(false);
try
{
@@ -271,7 +286,7 @@ namespace Duplicati.Library.Backend
public async Task RenameAsync(string source, string target, CancellationToken cancelToken)
{
await CreateConnection(cancelToken).ConfigureAwait(false);
await ChangeDirectory(m_path).ConfigureAwait(false);
await SetWorkingDirectory(cancelToken).ConfigureAwait(false);
await m_con.RenameFileAsync(source, target, cancelToken).ConfigureAwait(false);
}
@@ -342,6 +357,9 @@ namespace Duplicati.Library.Backend
con.KeepAliveInterval = m_keepaliveinterval;
await this.TryConnect(con, cancelToken).ConfigureAwait(false);
if (m_useRelativePath && (string.IsNullOrWhiteSpace(con.WorkingDirectory) || !con.WorkingDirectory.StartsWith("/")))
throw new UserInformationException("Server does not report absolute initial directory, please switch to absolute paths", "RelativePathNotSupported");
m_initialDirectory = con.WorkingDirectory;
m_con = con;
}
@@ -349,23 +367,27 @@ namespace Duplicati.Library.Backend
private Task TryConnect(SftpClient client, CancellationToken cancelToken)
=> client.ConnectAsync(cancelToken);
private async Task ChangeDirectory(string path)
private async Task SetWorkingDirectory(CancellationToken cancelToken)
{
if (string.IsNullOrEmpty(path))
if (string.IsNullOrEmpty(m_path))
return;
var targetPath = m_useRelativePath
? Util.AppendDirSeparator(m_initialDirectory, "/") + m_path
: m_path;
var workingDir = Util.AppendDirSeparator(m_con.WorkingDirectory, "/");
if (workingDir == path)
if (workingDir == targetPath)
return;
try
{
await m_con.ChangeDirectoryAsync(path);
await m_con.ChangeDirectoryAsync(targetPath, cancelToken).ConfigureAwait(false);
}
catch (Exception ex)
{
throw new Interface.FolderMissingException(
Strings.SSHv2Backend.FolderNotFoundManagedError(path, ex.Message), ex);
Strings.SSHv2Backend.FolderNotFoundManagedError(m_path, ex.Message), ex);
}
}
@@ -375,7 +397,7 @@ namespace Duplicati.Library.Backend
var path = ".";
await CreateConnection(cancelToken).ConfigureAwait(false);
await ChangeDirectory(m_path).ConfigureAwait(false);
await SetWorkingDirectory(cancelToken).ConfigureAwait(false);
await foreach (var ls in m_con.ListDirectoryAsync(path, cancelToken).ConfigureAwait(false))
{
@@ -392,7 +414,7 @@ namespace Duplicati.Library.Backend
{
if (!filename.StartsWith(KEYFILE_URI, StringComparison.OrdinalIgnoreCase))
{
return String.IsNullOrEmpty(password)
return string.IsNullOrEmpty(password)
? new PrivateKeyFile(filename)
: new PrivateKeyFile(filename, password);
}
@@ -405,7 +427,7 @@ namespace Duplicati.Library.Backend
ms.Position = 0;
return String.IsNullOrEmpty(password) ? new PrivateKeyFile(ms) : new PrivateKeyFile(ms, password);
return string.IsNullOrEmpty(password) ? new PrivateKeyFile(ms) : new PrivateKeyFile(ms, password);
}
}
catch (Exception ex)
@@ -425,5 +447,37 @@ namespace Duplicati.Library.Backend
}
public Task<string[]> GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new[] { m_server });
/// <inheritdoc/>
public async IAsyncEnumerable<IFileEntry> ListAsync(string path, [EnumeratorCancellation] CancellationToken cancellationToken)
{
await CreateConnection(cancellationToken).ConfigureAwait(false);
await SetWorkingDirectory(cancellationToken).ConfigureAwait(false);
var prefixPath = m_useRelativePath ? "" : m_path;
if (!string.IsNullOrWhiteSpace(prefixPath))
prefixPath = Util.AppendDirSeparator(prefixPath, "/");
var filterPath = prefixPath + BackendSourceFileEntry.NormalizePathTo(path, '/');
if (!string.IsNullOrWhiteSpace(filterPath))
filterPath = Util.AppendDirSeparator(filterPath, "/");
if (string.IsNullOrWhiteSpace(filterPath))
filterPath = ".";
await foreach (var x in Client.ListDirectoryAsync(filterPath, cancellationToken).ConfigureAwait(false))
yield return new FileEntry(
x.Name,
x.Attributes.Size,
x.Attributes.LastAccessTimeUtc,
x.Attributes.LastWriteTimeUtc)
{
IsFolder = x.Attributes.IsDirectory
};
}
/// <inheritdoc/>
public Task<IFileEntry> GetEntryAsync(string path, CancellationToken cancellationToken)
=> Task.FromResult<IFileEntry>(null);
}
}
Duplicati/Library/Backend/SSHv2/Strings.cs
+21 -19
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Localization.Short;
namespace Duplicati.Library.Backend.Strings
@@ -61,6 +61,8 @@ namespace Duplicati.Library.Backend.Strings
public static string DescriptionSshtimeoutShort { get { return LC.L(@"Set the operation timeout value"); } }
public static string DescriptionSshkeepaliveLong { get { return LC.L(@"Use this option to enable the keep-alive interval for the SSH connection. If the connection is idle, aggressive firewalls might close the connection. Using keep-alive will keep the connection open in this scenario. If this value is set to zero, the keep-alive is disabled."); } }
public static string DescriptionSshkeepaliveShort { get { return LC.L(@"Set a keepalive value"); } }
public static string DescriptionRelativePathShort { get { return LC.L(@"Treat source path as relative to the initial path"); } }
public static string DescriptionRelativePathLong { get { return LC.L(@"Use this option to treat the source path as relative to the initial path. This is useful when the full path of the system is not known."); } }
public static string FolderNotFoundManagedError(string foldername, string message) { return LC.L(@"Unable to set folder to {0}, error message: {1}", foldername, message); }
public static string FingerprintNotMatchManagedError(string fingerprint) { return LC.L(@"Validation of server fingerprint failed. Server returned fingerprint ""{0}"". Cause of this message is either not correct configuration or Man-in-the-middle attack!", fingerprint); }
public static string FingerprintNotSpecifiedManagedError(string fingerprint, string hostkeyoption, string allkeysoptions) { return LC.L(@"Please add --{1}=""{0}"" to trust this host. Optionally you can use --{2} (NOT SECURE) for testing!", fingerprint, hostkeyoption, allkeysoptions); }
Duplicati/Library/Backend/Storj/StorjFile.cs
+1
View File
@@ -33,6 +33,7 @@ namespace Duplicati.Library.Backend.Storj
public DateTime LastAccess { get; set; }
public DateTime LastModification { get; set; }
public DateTime Created { get; set; }
public string Name { get; set; }
Duplicati/Library/Backend/pCloud/Model/pCloudFileEntry.cs
+9 -4
View File
@@ -33,22 +33,27 @@ internal class pCloudFileEntry : IFileEntry
/// Indicates it is a folder on the remote filesystem
/// </summary>
public bool IsFolder { get; set; }
/// <summary>
/// Last access date of the file
/// </summary>
public DateTime LastAccess { get; set; }
/// <summary>
/// Last modification of the file
/// </summary>
public DateTime LastModification { get; set; }
/// <summary>
/// Last modification of the file
/// </summary>
public DateTime Created { get; set; }
/// <summary>
/// Filename or folder name
/// </summary>
public string Name { get; set; }
/// <summary>
/// Size of the object in bytes
/// </summary>
Duplicati/Library/Common/IO/FileEntry.cs
+30 -20
View File
@@ -1,23 +1,23 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using Duplicati.Library.Interface;
@@ -32,6 +32,7 @@ namespace Duplicati.Library.Common.IO
private string m_name;
private DateTime m_lastAccess;
private DateTime m_lastModification;
private DateTime m_created;
private long m_size;
private bool m_isFolder;
@@ -62,6 +63,15 @@ namespace Duplicati.Library.Common.IO
set { m_lastModification = value; }
}
/// <summary>
/// Gets or sets the time the file or folder was created
/// </summary>
public DateTime Created
{
get { return m_created; }
set { m_created = value; }
}
/// <summary>
/// Gets or sets the size of the file or folder
/// </summary>
Duplicati/Library/DynamicLoader/Duplicati.Library.DynamicLoader.csproj
+1
View File
@@ -17,6 +17,7 @@
<ProjectReference Include="..\Modules\Builtin\Duplicati.Library.Modules.Builtin.csproj" />
<ProjectReference Include="..\AutoUpdater\Duplicati.Library.AutoUpdater.csproj" />
<ProjectReference Include="..\SecretProvider\Duplicati.Library.SecretProvider.csproj" />
<ProjectReference Include="..\SourceProviders\Duplicati.Library.SourceProviders.csproj" />
</ItemGroup>
<ItemGroup>
Duplicati/Library/DynamicLoader/SourceProviderLoader.cs
+219
View File
@@ -0,0 +1,219 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Collections.Generic;
using Duplicati.Library.Interface;
using System.Linq;
using System.Runtime.ExceptionServices;
using Duplicati.Library.SourceProviders;
using Duplicati.Library.SourceProvider;
using System.Threading.Tasks;
using System.Threading;
namespace Duplicati.Library.DynamicLoader
{
/// <summary>
/// Loads all SourceProviders dynamically and exposes a list of those
/// </summary>
public class SourceProviderLoader
{
/// <summary>
/// Implementation overrides specific to SourceProviders
/// </summary>
private class SourceProviderLoaderSub : DynamicLoader<ISourceProviderModule>
{
/// <summary>
/// Returns the protocol key
/// </summary>
/// <param name="item">The item to load the key for</param>
/// <returns>The protocol key</returns>
protected override string GetInterfaceKey(ISourceProviderModule item)
{
return item.Key;
}
/// <summary>
/// Returns the subfolders searched for SourceProviders
/// </summary>
protected override string[] Subfolders => ["SourceProviders"];
/// <summary>
/// The built-in modules
/// </summary>
protected override IEnumerable<ISourceProviderModule> BuiltInModules => SourceProviderModules.BuiltInSourceProviderModules;
/// <summary>
/// Instanciates a specific SourceProvider, given the url and options
/// </summary>
/// <param name="url">The url to create the instance for</param>
/// <param name="options">The options to pass to the instance constructor</param>
/// <returns>The instanciated SourceProvider or null if the url is not supported</returns>
public ISourceProviderModule GetSourceProvider(string url, string mountPoint, Dictionary<string, string> options)
{
var uri = new Utility.Uri(url);
LoadInterfaces();
var newOpts = new Dictionary<string, string>(options);
foreach (var key in uri.QueryParameters.AllKeys)
newOpts[key] = uri.QueryParameters[key];
lock (m_lock)
{
try
{
if (m_interfaces.ContainsKey(uri.Scheme))
return (ISourceProviderModule)Activator.CreateInstance(m_interfaces[uri.Scheme].GetType(), url, mountPoint, newOpts);
}
catch (System.Reflection.TargetInvocationException tex)
{
if (tex.InnerException != null)
{
// Unwrap exceptions for nicer display. The ExceptionDispatchInfo class allows us to
// rethrow an exception without changing the stack trace.
ExceptionDispatchInfo.Capture(tex.InnerException).Throw();
}
throw;
}
return null;
}
}
/// <summary>
/// Gets the supported commands for a certain url
/// </summary>
/// <param name="url">The url to find commands for</param>
/// <returns>The supported commands or null if the url scheme was not supported</returns>
public IReadOnlyList<ICommandLineArgument> GetSupportedCommands(string url)
{
var uri = new Utility.Uri(url);
LoadInterfaces();
// TODO: The loading logic is replicated in the "GetSourceProvider" method, should be refactored
lock (m_lock)
{
ISourceProviderModule b;
if (m_interfaces.TryGetValue(uri.Scheme, out b) && b != null)
return GetSupportedCommandsCached(b).ToList();
else if (uri.Scheme.EndsWith("s", StringComparison.Ordinal))
{
var tmpscheme = uri.Scheme.Substring(0, uri.Scheme.Length - 1);
if (m_interfaces.TryGetValue(tmpscheme, out b) && b != null)
return GetSupportedCommandsCached(b).ToList();
}
return null;
}
}
}
/// <summary>
/// The static instance used to access SourceProvider information
/// </summary>
private static readonly SourceProviderLoaderSub _SourceProviderLoader = new SourceProviderLoaderSub();
#region Public static API
/// <summary>
/// Gets a list of loaded SourceProviders, the instances can be used to extract interface information, not used to interact with the SourceProvider.
/// </summary>
public static ISourceProviderModule[] SourceProviders { get { return _SourceProviderLoader.Interfaces; } }
/// <summary>
/// Gets a list of keys supported
/// </summary>
public static string[] Keys { get { return _SourceProviderLoader.Keys; } }
/// <summary>
/// Gets the supported commands for a given SourceProvider
/// </summary>
/// <param name="url">The url to find the commands for</param>
/// <returns>The supported commands or null if the url is not supported</returns>
public static IReadOnlyList<ICommandLineArgument> GetSupportedCommands(string url)
{
if (string.IsNullOrEmpty(url))
throw new ArgumentNullException(nameof(url));
var commands = _SourceProviderLoader.GetSupportedCommands(url);
if (commands != null)
return commands;
var backend = BackendLoader.GetBackend(url, []);
if (backend is IFolderEnabledBackend folderBackend)
commands = folderBackend.SupportedCommands.AsReadOnly();
backend?.Dispose();
return commands;
}
/// <summary>
/// Instanciates a specific SourceProvider, given the url and options
/// </summary>
/// <param name="url">The url to create the instance for</param>
/// <param name="mountPoint">The mount point to use</param>
/// <param name="options">The options to pass to the instance constructor</param>
/// <param name="cancellationToken">The cancellation token</param>
/// <returns>The instanciated SourceProvider or null if the url is not supported</returns>
public static async Task<ISourceProviderModule> GetSourceProvider(string url, string mountPoint, Dictionary<string, string> options, CancellationToken cancellationToken)
{
// Source providers are preferred over backends
var provider = _SourceProviderLoader.GetSourceProvider(url, mountPoint, options);
if (provider == null)
{
// See if there is a backend that can also be a source
var backend = BackendLoader.GetBackend(url, options);
if (backend is IFolderEnabledBackend folderBackend)
provider = new BackendSourceProvider(folderBackend, mountPoint);
else
backend?.Dispose();
}
if (provider == null)
return null;
try
{
await provider.Initialize(cancellationToken).ConfigureAwait(false);
return provider;
}
catch
{
provider.Dispose();
throw;
}
}
#endregion
/// <summary>
/// Adds a SourceProvider to the loader
/// </summary>
/// <param name="SourceProvider">The SourceProvider to add</param>
public static void AddSourceProvider(ISourceProviderModule SourceProvider)
{
_SourceProviderLoader.AddModule(SourceProvider);
}
}
}
Duplicati/Library/Interface/IDestinationProvider.cs
+43
View File
@@ -0,0 +1,43 @@
#nullable enable
using System.IO;
using System.Threading;
using System.Threading.Tasks;
namespace Duplicati.Library.Interface
{
/// <summary>
/// Implements a destination where data can be restored to
/// </summary>
public interface IDestinationProvider : IDynamicModule
{
/// <summary>
/// Gets the module key
/// </summary>
string Key { get; }
/// <summary>
/// Gets the display name of the module
/// </summary>
string DisplayName { get; }
/// <summary>
/// Checks if a restore is needed
/// </summary>
/// <param name="path">The path to restore to</param>
/// <param name="dataHash">The hash of the data to restore</param>
/// <param name="metadataHash">The hash of the metadata to restore</param>
/// <param name="cancel">The cancellation token</param>
/// <returns>True if a restore is needed, false otherwise</returns>
Task<bool> IsRestoreNeededAsync(string path, string dataHash, string? metadataHash, CancellationToken cancel);
/// <summary>
/// Restores a file to the given path
/// </summary>
/// <param name="path">The path to restore to</param>
/// <param name="data">The data stream to restore</param>
/// <param name="metadata">The metadata stream to restore</param>
/// <param name="cancel">The cancellation token</param>
/// <returns>A task that completes when the restore is done</returns>
Task WriteAsync(string path, Stream data, Stream? metadata, CancellationToken cancel);
}
}
Duplicati/Library/Interface/IFileEntry.cs
+24 -20
View File
@@ -1,23 +1,23 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
@@ -41,6 +41,10 @@ namespace Duplicati.Library.Interface
/// </summary>
DateTime LastModification { get; }
/// <summary>
/// The time the file or folder was created
/// </summary>
DateTime Created { get; }
/// <summary>
/// The name of the file or folder
/// </summary>
string Name { get; }
Duplicati/Library/Interface/IFolderEnabledBackend.cs
+61
View File
@@ -0,0 +1,61 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
namespace Duplicati.Library.Interface;
/// <summary>
/// An interface a backend may implement if it supports streaming operations.
/// Backends that implement this interface can be throttled and correctly shows
/// the progressbar when transferring data.
/// </summary>
public interface IFolderEnabledBackend : IBackend
{
/// <summary>
/// Enumerates a list of files and folders found on the remote location
/// </summary>
/// <param name="path">The path to list, empty or null for the root folder</param>
/// <param name="cancellationToken">The cancellation token</param>
/// <returns>The list of files, where paths are scoped to the backend.</returns>
/// <remarks>
/// The paths returned should be only the filenames or folder names, not the full path.
/// Folders should end with the operating system's directory separator.
/// </remarks>
IAsyncEnumerable<IFileEntry> ListAsync(string? path, CancellationToken cancellationToken);
/// <summary>
/// Gets a single file or folder entry
/// </summary>
/// <param name="path">The path to get</param>
/// <param name="cancellationToken">The cancellation token</param>
/// <returns>The file or folder entry</returns>
/// <remarks>
/// The paths returned should be only the filenames or folder names, not the full path.
/// Folders should end with the operating system's directory separator.
/// </remarks>
Task<IFileEntry?> GetEntryAsync(string path, CancellationToken cancellationToken);
}
Duplicati/Library/Interface/ISourceProvider.cs
+63
View File
@@ -0,0 +1,63 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
namespace Duplicati.Library.Interface;
/// <summary>
/// Implements a data source that can be backed up
/// </summary>
public interface ISourceProvider : IDisposable
{
/// <summary>
/// Returns the path where this provider is logically mounted
/// </summary>
string MountedPath { get; }
/// <summary>
/// Initializes the provider, if needed
/// </summary>
/// <param name="cancellationToken">The cancellation token</param>
/// <returns>An awaitable task</returns>
Task Initialize(CancellationToken cancellationToken);
/// <summary>
/// Gets the root entries
/// </summary>
/// <param name="cancellationToken">The cancellation token</param>
/// <returns>An enumerable of file entries</returns>
IAsyncEnumerable<ISourceProviderEntry> Enumerate(CancellationToken cancellationToken);
/// <summary>
/// Gets a specific entry
/// </summary>
/// <param name="path">The path to use</param>
/// <param name="isFolder">True if the path is a folder</param>
/// <param name="cancellationToken">The cancellation token</param>
/// <returns>The file entry</returns>
Task<ISourceProviderEntry?> GetEntry(string path, bool isFolder, CancellationToken cancellationToken);
}
Duplicati/Library/Interface/ISourceProviderEntry.cs
+126
View File
@@ -0,0 +1,126 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
#nullable enable
namespace Duplicati.Library.Interface;
/// <summary>
/// Interface for an instance of a file or folder
/// </summary>
public interface ISourceProviderEntry
{
/// <summary>
/// True if the entry represents a folder, false otherwise
/// </summary>
bool IsFolder { get; }
/// <summary>
/// True if the entry is a meta entry, false otherwise
/// </summary>
bool IsMetaEntry { get; }
/// <summary>
/// True if the entry is the root entry, false otherwise
/// </summary>
bool IsRootEntry { get; }
/// <summary>
/// The creation time of the file or folder
/// </summary>
DateTime CreatedUtc { get; }
/// <summary>
/// The time the file or folder was last modified
/// </summary>
DateTime LastModificationUtc { get; }
/// <summary>
/// The name of the file or folder
/// </summary>
string Path { get; }
/// <summary>
/// The size of the file or folder
/// </summary>
long Size { get; }
/// <summary>
/// True if the file is a symlink, false otherwise
/// </summary>
bool IsSymlink { get; }
/// <summary>
/// The target of a symlink, if the file is a symlink
/// </summary>
string? SymlinkTarget { get; }
/// <summary>
/// The entry attributes
/// </summary>
FileAttributes Attributes { get; }
/// <summary>
/// The minor metadata of the file, should be less than 1KB
/// </summary>
Dictionary<string, string> MinorMetadata { get; }
/// <summary>
/// True if the file is a block device, false otherwise
/// </summary>
bool IsBlockDevice { get; }
/// <summary>
/// True if the file is a character device, false otherwise
/// </summary>
bool IsCharacterDevice { get; }
/// <summary>
/// True if the files is an alternate stream, false otherwise
/// </summary>
bool IsAlternateStream { get; }
/// <summary>
/// The hardlink target id, if the file is a hardlink
/// </summary>
string? HardlinkTargetId { get; }
/// <summary>
/// Opens the file for reading
/// </summary>
/// <param name="cancellationToken">A token that can be used to cancel the operation</param>
/// <returns>A stream that can be read</returns>
Task<Stream> OpenRead(CancellationToken cancellationToken);
/// <summary>
/// Opens the metadata for reading
/// </summary>
/// <param name="cancellationToken">A token that can be used to cancel the operation</param>
/// <returns>A stream that can be read</returns>
Task<Stream?> OpenMetadataRead(CancellationToken cancellationToken);
/// <summary>
/// Checks if the specified file exists
/// </summary>
/// <param name="filename">The file to check for existence</param>
/// <param name="cancellationToken">A token that can be used to cancel the operation</param>
/// <returns><c>true</c> if the file exists, <c>false</c> otherwise</returns>
Task<bool> FileExists(string filename, CancellationToken cancellationToken);
/// <summary>
/// Returns the contents of the object, if it is a folder
/// </summary>
/// <param name="cancellationToken">A token that can be used to cancel the operation</param>
/// <returns>The contents of the folder</returns>
IAsyncEnumerable<ISourceProviderEntry> Enumerate(CancellationToken cancellationToken);
}
Duplicati/Library/Interface/ISourceProviderModule.cs
+39
View File
@@ -0,0 +1,39 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
namespace Duplicati.Library.Interface;
/// <summary>
/// Implements a data source that can be backed up
/// </summary>
public interface ISourceProviderModule : ISourceProvider, IDynamicModule
{
/// <summary>
/// Gets the module key
/// </summary>
string Key { get; }
/// <summary>
/// Gets the display name of the module
/// </summary>
string DisplayName { get; }
}
Duplicati/Library/Main/Controller.cs
+14 -1
View File
@@ -948,6 +948,19 @@ namespace Duplicati.Library.Main
string source;
try
{
// Check if this is a mounted path
if (expandedSource.StartsWith("@"))
{
// TODO: If the remote source fails to load,
// this will be an enumeration warning, but will result
// in the backup being recorded without files from the source
// Eventually, this could lead to retention deletion,
// causing the last backup with the data from the source to be deleted
foundAnyPaths = true;
sources.Add(expandedSource);
continue;
}
// TODO: This expands "C:" to CWD, but not C:\
source = System.IO.Path.GetFullPath(expandedSource);
}
@@ -999,7 +1012,7 @@ namespace Duplicati.Library.Main
//Sanity check for duplicate files/folders
ISet<string> pathDuplicates;
sources = Library.Utility.Utility.GetUniqueItems(sources, Library.Utility.Utility.ClientFilenameStringComparer, out pathDuplicates).OrderBy(a => a).ToList();
sources = Library.Utility.Utility.GetUniqueItems(sources, Library.Utility.Utility.ClientFilenameStringComparer, out pathDuplicates).ToList();
foreach (var pathDuplicate in pathDuplicates)
Logging.Log.WriteVerboseMessage(LOGTAG, "RemoveDuplicateSource", "Removing duplicate source: {0}", pathDuplicate);
Duplicati/Library/Main/Database/LocalBackupDatabase.cs
+2 -1
View File
@@ -22,6 +22,7 @@ using System;
using System.Collections.Generic;
using System.Data;
using System.Linq;
using Duplicati.Library.Common.IO;
namespace Duplicati.Library.Main.Database
{
@@ -881,7 +882,7 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist
{
using (var cmd = m_connection.CreateCommand())
{
cmd.CommandText = @"SELECT ""Path"" FROM ""File"" ORDER BY LENGTH(""Path"") DESC LIMIT 1";
cmd.CommandText = @$"SELECT ""Path"" FROM ""File"" ORDER BY LENGTH(""Path"") DESC LIMIT 1";
var v0 = cmd.ExecuteScalar();
if (v0 == null || v0 == DBNull.Value)
return null;
Duplicati/Library/Main/Duplicati.Library.Main.csproj
+2
View File
@@ -19,6 +19,7 @@
<ProjectReference Include="..\Localization\Duplicati.Library.Localization.csproj" />
<ProjectReference Include="..\UsageReporter\Duplicati.Library.UsageReporter.csproj" />
<ProjectReference Include="..\Modules\Builtin\Duplicati.Library.Modules.Builtin.csproj" />
<ProjectReference Include="..\SourceProvider\Builtin\Duplicati.Library.SourceProvider.Builtin.csproj" />
</ItemGroup>
<ItemGroup>
@@ -47,6 +48,7 @@
<ItemGroup>
<PackageReference Include="CoCoL" Version="1.8.1" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
</ItemGroup>
</Project>
Duplicati/Library/Main/Operation/Backup/Channels.cs
+2 -1
View File
@@ -20,6 +20,7 @@
// DEALINGS IN THE SOFTWARE.
using CoCoL;
using Duplicati.Library.Interface;
namespace Duplicati.Library.Main.Operation.Backup
{
@@ -51,7 +52,7 @@ namespace Duplicati.Library.Main.Operation.Backup
/// <summary>
/// When enumerating the source folders, all discovered paths are sent by the <see cref="FileEnumerationProcess"/> to the <see cref="MetadataPreProcess"/>
/// </summary>
public readonly IChannel<string> SourcePaths = ChannelManager.CreateChannel<string>();
public readonly IChannel<ISourceProviderEntry> SourcePaths = ChannelManager.CreateChannel<ISourceProviderEntry>();
/// <summary>
/// All progress events are communicated over this channel, to ensure that parallel progress is reported as a if it was sequential
/// </summary>
Duplicati/Library/Main/Operation/Backup/CountFilesHandler.cs
+28 -30
View File
@@ -1,42 +1,40 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System.Collections.Generic;
using System.Threading.Tasks;
using Duplicati.Library.Utility;
using Duplicati.Library.Snapshots;
using CoCoL;
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.Snapshots.USN;
namespace Duplicati.Library.Main.Operation.Backup
{
internal static class CountFilesHandler
{
public static async Task Run(
IEnumerable<string> sources,
ISnapshotService snapshot,
ISourceProvider sources,
UsnJournalService journalService,
BackupResults result,
Options options,
IFilter sourcefilter,
IFilter filter,
HashSet<string> blacklistPaths,
Common.ITaskReader taskreader,
@@ -47,9 +45,8 @@ namespace Duplicati.Library.Main.Operation.Backup
using (Logging.Log.StartIsolatingScope(true))
{
Channels channels = new();
var enumeratorTask = Backup.FileEnumerationProcess.Run(
channels, sources, snapshot, journalService,
options.FileAttributeFilter, sourcefilter, filter,
var enumeratorTask = FileEnumerationProcess.Run(
channels, sources, journalService, options.FileAttributeFilter, filter,
options.SymlinkPolicy, options.HardlinkPolicy,
options.ExcludeEmptyFolders, options.IgnoreFilenames,
blacklistPaths, options.ChangedFilelist, taskreader, null, token);
@@ -68,14 +65,15 @@ namespace Duplicati.Library.Main.Operation.Backup
{
while (await taskreader.ProgressRendevouz() && !token.IsCancellationRequested)
{
var path = await self.Input.ReadAsync();
var entry = await self.Input.ReadAsync();
if (entry.IsFolder)
continue;
count++;
try
{
if (SystemIO.IO_OS.FileExists(path))
size += snapshot.GetFileSize(path);
size += entry.Size;
}
catch
{
Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs
+15 -16
View File
@@ -23,7 +23,6 @@ using System;
using CoCoL;
using Duplicati.Library.Main.Operation.Common;
using System.Threading.Tasks;
using System.Threading;
namespace Duplicati.Library.Main.Operation.Backup
{
@@ -37,7 +36,7 @@ namespace Duplicati.Library.Main.Operation.Backup
/// </summary>
private static readonly string FILELOGTAG = Logging.Log.LogTagFromType(typeof(FileBlockProcessor)) + ".FileEntry";
public static Task Run(Channels channels, Snapshots.ISnapshotService snapshot, Options options, BackupDatabase database, BackupStatsCollector stats, ITaskReader taskreader)
public static Task Run(Channels channels, Options options, BackupDatabase database, BackupStatsCollector stats, ITaskReader taskreader)
{
return AutomationExtensions.RunTask(
new
@@ -57,7 +56,7 @@ namespace Duplicati.Library.Main.Operation.Backup
try
{
var hint = options.GetCompressionHintFromFilename(e.Path);
var hint = options.GetCompressionHintFromFilename(e.Entry.Path);
var oldHash = e.OldId < 0 ? null : await database.GetFileHashAsync(e.OldId);
StreamProcessResult filestreamdata;
@@ -73,15 +72,15 @@ namespace Duplicati.Library.Main.Operation.Backup
if (res.Item1)
return res.Item2;
Logging.Log.WriteWarningMessage(FILELOGTAG, "UnexpectedMetadataLookup", null, "Metadata was reported as not changed, but still requires being added?\nHash: {0}, Length: {1}, ID: {2}, Path: {3}", e.MetaHashAndSize.FileHash, e.MetaHashAndSize.Blob.Length, res.Item2, e.Path);
Logging.Log.WriteWarningMessage(FILELOGTAG, "UnexpectedMetadataLookup", null, "Metadata was reported as not changed, but still requires being added?\nHash: {0}, Length: {1}, ID: {2}, Path: {3}", e.MetaHashAndSize.FileHash, e.MetaHashAndSize.Blob.Length, res.Item2, e.Entry.Path);
e.MetadataChanged = true;
}
return (await MetadataPreProcess.AddMetadataToOutputAsync(e.Path, e.MetaHashAndSize, database, self.StreamBlockChannel)).Item2;
return (await MetadataPreProcess.AddMetadataToOutputAsync(e.Entry.Path, e.MetaHashAndSize, database, self.StreamBlockChannel)).Item2;
});
using (var fs = snapshot.OpenRead(e.Path))
filestreamdata = await StreamBlock.ProcessStream(self.StreamBlockChannel, e.Path, fs, false, hint);
using (var fs = await e.Entry.OpenRead(taskreader.ProgressToken))
filestreamdata = await StreamBlock.ProcessStream(self.StreamBlockChannel, e.Entry.Path, fs, false, hint);
await stats.AddOpenedFile(filestreamdata.Streamlength);
@@ -92,42 +91,42 @@ namespace Duplicati.Library.Main.Operation.Backup
if (oldHash != filekey)
{
if (oldHash == null)
Logging.Log.WriteVerboseMessage(FILELOGTAG, "NewFile", "New file {0}", e.Path);
Logging.Log.WriteVerboseMessage(FILELOGTAG, "NewFile", "New file {0}", e.Entry.Path);
else
Logging.Log.WriteVerboseMessage(FILELOGTAG, "ChangedFile", "File has changed {0}", e.Path);
Logging.Log.WriteVerboseMessage(FILELOGTAG, "ChangedFile", "File has changed {0}", e.Entry.Path);
if (e.OldId < 0)
{
await stats.AddAddedFile(filesize);
if (options.Dryrun)
Logging.Log.WriteVerboseMessage(FILELOGTAG, "WouldAddNewFile", "Would add new file {0}, size {1}", e.Path, Library.Utility.Utility.FormatSizeString(filesize));
Logging.Log.WriteVerboseMessage(FILELOGTAG, "WouldAddNewFile", "Would add new file {0}, size {1}", e.Entry.Path, Library.Utility.Utility.FormatSizeString(filesize));
}
else
{
await stats.AddModifiedFile(filesize);
if (options.Dryrun)
Logging.Log.WriteVerboseMessage(FILELOGTAG, "WouldAddChangedFile", "Would add changed file {0}, size {1}", e.Path, Library.Utility.Utility.FormatSizeString(filesize));
Logging.Log.WriteVerboseMessage(FILELOGTAG, "WouldAddChangedFile", "Would add changed file {0}, size {1}", e.Entry.Path, Library.Utility.Utility.FormatSizeString(filesize));
}
await database.AddFileAsync(e.PathPrefixID, e.Filename, e.LastWrite, filestreamdata.Blocksetid, metadataid);
}
else if (e.MetadataChanged)
{
Logging.Log.WriteVerboseMessage(FILELOGTAG, "FileMetadataChanged", "File has only metadata changes {0}", e.Path);
Logging.Log.WriteVerboseMessage(FILELOGTAG, "FileMetadataChanged", "File has only metadata changes {0}", e.Entry.Path);
await database.AddFileAsync(e.PathPrefixID, e.Filename, e.LastWrite, filestreamdata.Blocksetid, metadataid);
}
else if (e.TimestampChanged)
{
await stats.AddTimestampChangedFile();
Logging.Log.WriteVerboseMessage(FILELOGTAG, "FileTimestampChanged", "File has only timestamp changes {0}", e.Path);
Logging.Log.WriteVerboseMessage(FILELOGTAG, "FileTimestampChanged", "File has only timestamp changes {0}", e.Entry.Path);
await database.AddUnmodifiedAsync(e.OldId, e.LastWrite);
}
else /*if (e.OldId >= 0)*/
{
// When we write the file to output, update the last modified time
Logging.Log.WriteVerboseMessage(FILELOGTAG, "NoFileChanges", "File has not changed {0}", e.Path);
Logging.Log.WriteVerboseMessage(FILELOGTAG, "NoFileChanges", "File has not changed {0}", e.Entry.Path);
try
{
@@ -135,7 +134,7 @@ namespace Duplicati.Library.Main.Operation.Backup
}
catch (Exception ex)
{
Logging.Log.WriteWarningMessage(FILELOGTAG, "FailedToAddFile", ex, "Failed while attempting to add unmodified file to database: {0}", e.Path);
Logging.Log.WriteWarningMessage(FILELOGTAG, "FailedToAddFile", ex, "Failed while attempting to add unmodified file to database: {0}", e.Entry.Path);
}
}
}
@@ -144,7 +143,7 @@ namespace Duplicati.Library.Main.Operation.Backup
if (ex.IsRetiredException())
return;
else
Logging.Log.WriteWarningMessage(FILELOGTAG, "PathProcessingFailed", ex, "Failed to process path: {0}", e.Path);
Logging.Log.WriteWarningMessage(FILELOGTAG, "PathProcessingFailed", ex, "Failed to process path: {0}", e.Entry.Path);
}
}
});
Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs
+237 -119
View File
@@ -19,6 +19,8 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
using System;
using CoCoL;
using System.Threading.Tasks;
@@ -27,9 +29,10 @@ using System.Collections.Generic;
using System.Linq;
using System.Threading;
using Duplicati.Library.Main.Operation.Common;
using Duplicati.Library.Snapshots;
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using System.Runtime.CompilerServices;
using Duplicati.Library.SourceProvider;
using Duplicati.Library.Snapshots.USN;
namespace Duplicati.Library.Main.Operation.Backup
{
@@ -47,20 +50,18 @@ namespace Duplicati.Library.Main.Operation.Backup
public static Task Run(
Channels channels,
IEnumerable<string> sources,
ISnapshotService snapshot,
UsnJournalService journalService,
FileAttributes fileAttributes,
Library.Utility.IFilter sourcefilter,
ISourceProvider sourceProvider,
UsnJournalService? journalService,
FileAttributes fileAttributeFilter,
Library.Utility.IFilter emitfilter,
Options.SymlinkStrategy symlinkPolicy,
Options.HardlinkStrategy hardlinkPolicy,
bool excludeemptyfolders,
string[] ignorenames,
string[]? ignorenames,
HashSet<string> blacklistPaths,
string[] changedfilelist,
IEnumerable<string>? changedfilelist,
ITaskReader taskreader,
Action onStopRequested,
Action? onStopRequested,
CancellationToken token)
{
return AutomationExtensions.RunTask(
@@ -73,9 +74,19 @@ namespace Duplicati.Library.Main.Operation.Backup
{
if (!token.IsCancellationRequested)
{
// The hardlink map tracks the hardlink targets we have seen
// and avoid multiple processing of the same contents
var hardlinkmap = new Dictionary<string, string>();
var mixinqueue = new Queue<string>();
Duplicati.Library.Utility.IFilter enumeratefilter = emitfilter;
// The mixin queue is used to store symlinks that should be processed
// The symlinks are emitted during the enumeration process when they are found
var mixinqueue = new Queue<ISourceProviderEntry>();
// The enumeration filter is used to determine what paths to
// recurse into. If the emit filter only has includes,
// the enumeration filter will also include all folders,
// as nothing will match otherwise
var enumeratefilter = emitfilter;
Library.Utility.FilterExpression.AnalyzeFilters(emitfilter, out var includes, out var excludes);
if (includes && !excludes)
@@ -85,59 +96,82 @@ namespace Duplicati.Library.Main.Operation.Backup
if (ignorenames != null && ignorenames.Length == 0)
ignorenames = null;
// Shared filter function with bound variables
ValueTask<bool> FilterEntry(ISourceProviderEntry entry)
=> SourceFileEntryFilter(entry, blacklistPaths, hardlinkPolicy, symlinkPolicy, hardlinkmap, fileAttributeFilter, enumeratefilter, ignorenames, mixinqueue, token);
// Prepare the work list
IAsyncEnumerable<ISourceProviderEntry> worklist;
// If we have a specific list, use that instead of enumerating the filesystem
IEnumerable<string> worklist;
if (changedfilelist != null && changedfilelist.Length > 0)
if (changedfilelist != null && changedfilelist.Any())
{
worklist = changedfilelist.Where(x =>
async IAsyncEnumerable<ISourceProviderEntry> ExpandSources(IEnumerable<string> list)
{
var fa = FileAttributes.Normal;
try
{
fa = snapshot.GetAttributes(x);
}
catch
foreach (var s in list)
{
var r = await sourceProvider.GetEntry(s, s.EndsWith(Path.DirectorySeparatorChar), token).ConfigureAwait(false);
if (r != null)
{
//TODO: Set r.IsRoot = true for source elements
yield return r;
}
}
}
worklist = ExpandSources(changedfilelist).WhereAwait(FilterEntry);
}
else if (journalService != null)
{
if (!OperatingSystem.IsWindows())
throw new NotSupportedException("USN is only supported on Windows");
if (token.IsCancellationRequested)
{
return false;
}
var fileProviders = (sourceProvider is Combiner c ? c.Providers.AsEnumerable() : [sourceProvider])
.OfType<SourceProvider.LocalFileSource>()
.ToList();
return AttributeFilter(x, fa, snapshot, sourcefilter, blacklistPaths, hardlinkPolicy, symlinkPolicy, hardlinkmap, fileAttributes, enumeratefilter, ignorenames, mixinqueue);
});
if (fileProviders.Count <= 0)
throw new InvalidOperationException("No file providers found, but USN was enabled?");
if (fileProviders.Count > 1)
throw new InvalidOperationException("Multiple file providers found, but USN only supports one");
// TODO: This is not as effecient as possible.
// If the root folder is marked changed by USN, the expansion with RecurseEntries
// will cause a full regular scan. It should be possible to *only* process the
// changed elements as returned from the USN journal.
// It should be possible to remove RecurseEntries from the GetModifiedSources()
// enumeration result.
// Such a change requires significant testing as there are many pitfalls with USN.
worklist = RecurseEntries(journalService.GetModifiedSources(FilterEntry, token),
FilterEntry,
token
)
.Concat(
RecurseEntries(journalService.GetFullScanSources(token),
FilterEntry,
token)
);
}
else
{
Library.Utility.Utility.EnumerationFilterDelegate attributeFilter = (root, path, attr) =>
AttributeFilter(path, attr, snapshot, sourcefilter, blacklistPaths, hardlinkPolicy, symlinkPolicy, hardlinkmap, fileAttributes, enumeratefilter, ignorenames, mixinqueue);
if (journalService != null)
{
if (!OperatingSystem.IsWindows())
throw new UserInformationException("USN journal is only supported on Windows", "USNJournalNotSupported");
// filter sources using USN journal, to obtain a sub-set of files / folders that may have been modified
sources = journalService.GetModifiedSources(attributeFilter);
}
worklist = snapshot.EnumerateFilesAndFolders(sources, attributeFilter, (rootpath, errorpath, ex) =>
{
Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "FileAccessError", ex, "Error reported while accessing file: {0}", errorpath);
}).Distinct(Library.Utility.Utility.IsFSCaseSensitive ? StringComparer.Ordinal : StringComparer.OrdinalIgnoreCase);
worklist = RecurseEntries(sourceProvider.Enumerate(token),
FilterEntry,
token
);
}
if (token.IsCancellationRequested)
{
return;
}
var source = ExpandWorkList(worklist, mixinqueue, emitfilter, enumeratefilter);
var source = ExpandWorkList(worklist, mixinqueue, emitfilter, enumeratefilter, token);
// TODO: There was a call to DistinctBy here, but this would cause all paths to be stored in memory
//.DistinctBy(x => x.Path, Library.Utility.Utility.IsFSCaseSensitive ? StringComparer.Ordinal : StringComparer.OrdinalIgnoreCase);
if (excludeemptyfolders)
source = ExcludeEmptyFolders(source);
source = ExcludeEmptyFolders(source, token);
// Process each path, and dequeue the mixins with symlinks as we go
foreach (var s in source)
await foreach (var s in source.WithCancellation(token).ConfigureAwait(false))
{
#if DEBUG
// For testing purposes, we need exact control
@@ -146,7 +180,7 @@ namespace Duplicati.Library.Main.Operation.Backup
// if the process is the real file enumeration process
// because the counter processe does not have a callback
if (onStopRequested != null)
taskreader.TestMethodCallback?.Invoke(s);
taskreader.TestMethodCallback?.Invoke(s.Path);
#endif
// Stop if requested
if (token.IsCancellationRequested || !await taskreader.ProgressRendevouz().ConfigureAwait(false))
@@ -167,13 +201,13 @@ namespace Duplicati.Library.Main.Operation.Backup
private class DirectoryStackEntry
{
/// <summary>
/// The path for the folder
/// The item being tracked
/// </summary>
public string Path;
public required ISourceProviderEntry Item;
/// <summary>
/// A flag indicating if any items are found in this folder
/// </summary>
public bool AnyEntries;
public required bool AnyEntries;
}
@@ -182,17 +216,17 @@ namespace Duplicati.Library.Main.Operation.Backup
/// </summary>
/// <returns>The list without empty folders.</returns>
/// <param name="source">The list with potential empty folders.</param>
private static IEnumerable<string> ExcludeEmptyFolders(IEnumerable<string> source)
private static async IAsyncEnumerable<ISourceProviderEntry> ExcludeEmptyFolders(IAsyncEnumerable<ISourceProviderEntry> source, [EnumeratorCancellation] CancellationToken cancellationToken)
{
var pathstack = new Stack<DirectoryStackEntry>();
foreach (var s in source)
await foreach (var s in source.WithCancellation(cancellationToken).ConfigureAwait(false))
{
// Keep track of directories
var isDirectory = s[s.Length - 1] == System.IO.Path.DirectorySeparatorChar;
var isDirectory = s.Path[s.Path.Length - 1] == System.IO.Path.DirectorySeparatorChar;
if (isDirectory)
{
while (pathstack.Count > 0 && !s.StartsWith(pathstack.Peek().Path, Library.Utility.Utility.ClientFilenameStringComparison))
while (pathstack.Count > 0 && !s.Path.StartsWith(pathstack.Peek().Item.Path, Library.Utility.Utility.ClientFilenameStringComparison))
{
var e = pathstack.Pop();
if (e.AnyEntries || pathstack.Count == 0)
@@ -201,15 +235,15 @@ namespace Duplicati.Library.Main.Operation.Backup
if (pathstack.Count > 0)
pathstack.Peek().AnyEntries = true;
yield return e.Path;
yield return e.Item;
}
else
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingEmptyFolder", "Excluding empty folder {0}", e.Path);
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingEmptyFolder", "Excluding empty folder {0}", e.Item);
}
if (pathstack.Count == 0 || s.StartsWith(pathstack.Peek().Path, Library.Utility.Utility.ClientFilenameStringComparison))
if (pathstack.Count == 0 || s.Path.StartsWith(pathstack.Peek().Item.Path, Library.Utility.Utility.ClientFilenameStringComparison))
{
pathstack.Push(new DirectoryStackEntry() { Path = s });
pathstack.Push(new DirectoryStackEntry() { Item = s, AnyEntries = false });
continue;
}
}
@@ -231,7 +265,46 @@ namespace Duplicati.Library.Main.Operation.Backup
if (pathstack.Count > 0)
pathstack.Peek().AnyEntries = true;
yield return e.Path;
yield return e.Item;
}
}
}
/// <summary>
/// Performs recursive traversal of the sources
/// </summary>
/// <param name="entries">The entries to recurse</param>
/// <param name="filter">The filter to apply</param>
/// <returns></returns>
private static async IAsyncEnumerable<ISourceProviderEntry> RecurseEntries(IAsyncEnumerable<ISourceProviderEntry> entries, Func<ISourceProviderEntry, ValueTask<bool>> filter, [EnumeratorCancellation] CancellationToken cancellationToken)
{
var work = new Stack<ISourceProviderEntry>();
await foreach (var e in entries.WithCancellation(cancellationToken).ConfigureAwait(false))
if (await filter(e).ConfigureAwait(false))
work.Push(e);
while (work.Count > 0)
{
var e = work.Pop();
// Process meta entry contents, but don't emit them for processing
if (!e.IsMetaEntry)
yield return e;
if (e.IsFolder)
{
try
{
// We only filter new items, as we assume the input is already filtered
await foreach (var r in e.Enumerate(cancellationToken).ConfigureAwait(false))
if (await filter(r).ConfigureAwait(false))
work.Push(r);
}
catch (Exception ex)
{
Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingErrorEnumerate", ex, "Failed to enumerate path: {0}", e.Path);
}
}
}
}
@@ -244,16 +317,18 @@ namespace Duplicati.Library.Main.Operation.Backup
/// <param name="mixinqueue">The mix in queue.</param>
/// <param name="emitfilter">The emitfilter.</param>
/// <param name="enumeratefilter">The enumeratefilter.</param>
private static IEnumerable<string> ExpandWorkList(IEnumerable<string> worklist, Queue<string> mixinqueue, Library.Utility.IFilter emitfilter, Library.Utility.IFilter enumeratefilter)
private static async IAsyncEnumerable<ISourceProviderEntry> ExpandWorkList(IAsyncEnumerable<ISourceProviderEntry> worklist, Queue<ISourceProviderEntry> mixinqueue, Library.Utility.IFilter emitfilter, Library.Utility.IFilter enumeratefilter, [EnumeratorCancellation] CancellationToken cancellationToken)
{
// Process each path, and dequeue the mixins with symlinks as we go
foreach (var s in worklist)
await foreach (var s in worklist.WithCancellation(cancellationToken).ConfigureAwait(false))
{
while (mixinqueue.Count > 0)
yield return mixinqueue.Dequeue();
Library.Utility.IFilter m;
if (emitfilter != enumeratefilter && !Library.Utility.FilterExpression.Matches(emitfilter, s, out m))
// If there are only includes in the filter, check if the item is in the original filter
// Since the enumerate filter also includes all folders, we need to ensure we do not emit
// any entries that are filtered explicitly by the user
if (emitfilter != enumeratefilter && !Library.Utility.FilterExpression.Matches(emitfilter, s.Path, out var _))
continue;
yield return s;
@@ -265,165 +340,208 @@ namespace Duplicati.Library.Main.Operation.Backup
}
/// <summary>
/// Plugin filter for enumerating a list of files.
/// Performs a pre-filter on the source entry to see if it should be included in the backup
/// </summary>
/// <returns>True if the path should be returned, false otherwise.</returns>
/// <param name="path">The current path.</param>
/// <param name="attributes">The file or folder attributes.</param>
/// <param name="snapshot">The snapshot service.</param>
/// <param name="sourcefilter">The source filter.</param>
/// <param name="entry">The entry to evaluate.</param>
/// <param name="blacklistPaths">The blacklist paths.</param>
/// <param name="hardlinkPolicy">The hardlink policy.</param>
/// <param name="symlinkPolicy">The symlink policy.</param>
/// <param name="hardlinkmap">The hardlink map.</param>
/// <param name="fileAttributes">The file attributes to exclude.</param>
/// <param name="enumeratefilter">The enumerate filter.</param>
/// <param name="ignorenames">The ignore names.</param>
/// <param name="mixinqueue">The mixin queue.</param>
private static bool AttributeFilter(string path, FileAttributes attributes, Snapshots.ISnapshotService snapshot, Library.Utility.IFilter sourcefilter, HashSet<string> blacklistPaths, Options.HardlinkStrategy hardlinkPolicy, Options.SymlinkStrategy symlinkPolicy, Dictionary<string, string> hardlinkmap, FileAttributes fileAttributes, Duplicati.Library.Utility.IFilter enumeratefilter, string[] ignorenames, Queue<string> mixinqueue)
/// <returns>True if the path should be returned, false otherwise.</returns>
private static bool PreFilterSourceEntry(ISourceProviderEntry entry, HashSet<string> blacklistPaths)
{
// Don't filter meta stuff
if (entry.IsMetaEntry)
return true;
// Exclude any blacklisted paths
if (blacklistPaths.Contains(path))
if (blacklistPaths.Contains(entry.Path))
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingBlacklistedPath", "Excluding blacklisted path: {0}", path);
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingBlacklistedPath", "Excluding blacklisted path: {0}", entry.Path);
return false;
}
// Exclude block devices
try
{
if (snapshot.IsBlockDevice(path))
if (entry.IsBlockDevice)
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingBlockDevice", "Excluding block device: {0}", path);
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingBlockDevice", "Excluding block device: {0}", entry.Path);
return false;
}
}
catch (Exception ex)
{
Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingErrorBlockDevice", ex, "Failed to process path: {0}", path);
Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingErrorBlockDevice", ex, "Failed to process path: {0}", entry.Path);
return false;
}
// Check if we explicitly include this entry
Duplicati.Library.Utility.IFilter sourcematch;
bool sourcematches;
if (sourcefilter.Matches(path, out sourcematches, out sourcematch) && sourcematches)
// Exclude character devices
try
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "IncludingSourcePath", "Including source path: {0}", path);
return true;
if (entry.IsCharacterDevice)
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingCharacterDevice", "Excluding character device: {0}", entry.Path);
return false;
}
}
catch (Exception ex)
{
Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingErrorCharacterDevice", ex, "Failed to process path: {0}", entry.Path);
return false;
}
return true;
}
/// <summary>
/// Evaluates a single entry for inclusion in the backup
/// </summary>
/// <param name="entry">The current entry.</param>
/// <param name="snapshot">The snapshot service.</param>
/// <param name="blacklistPaths">The blacklist paths.</param>
/// <param name="hardlinkPolicy">The hardlink policy.</param>
/// <param name="symlinkPolicy">The symlink policy.</param>
/// <param name="hardlinkmap">The hardlink map.</param>
/// <param name="fileAttributeFilter">The file attributes to exclude.</param>
/// <param name="enumeratefilter">The enumerate filter.</param>
/// <param name="ignorenames">The ignore names.</param>
/// <param name="mixinqueue">The mixin queue.</param>
/// <returns>True if the path should be returned, false otherwise.</returns>
private static async ValueTask<bool> SourceFileEntryFilter(ISourceProviderEntry entry, HashSet<string> blacklistPaths, Options.HardlinkStrategy hardlinkPolicy, Options.SymlinkStrategy symlinkPolicy, Dictionary<string, string> hardlinkmap, FileAttributes fileAttributeFilter, Duplicati.Library.Utility.IFilter enumeratefilter, string[]? ignorenames, Queue<ISourceProviderEntry> mixinqueue, CancellationToken cancellationToken)
{
// Do the course pre-filtering first
if (!PreFilterSourceEntry(entry, blacklistPaths))
return false;
// Never exclude the root entries
if (entry.IsRootEntry)
return true;
// If we have a hardlink strategy, obey it
if (hardlinkPolicy != Options.HardlinkStrategy.All)
{
try
{
var id = snapshot.HardlinkTargetID(path);
var id = entry.HardlinkTargetId;
if (id != null)
{
if (hardlinkPolicy == Options.HardlinkStrategy.None)
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingHardlinkByPolicy", "Excluding hardlink: {0} ({1})", path, id);
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingHardlinkByPolicy", "Excluding hardlink: {0} ({1})", entry.Path, id);
return false;
}
else if (hardlinkPolicy == Options.HardlinkStrategy.First)
{
string prevPath;
if (hardlinkmap.TryGetValue(id, out prevPath))
if (hardlinkmap.TryGetValue(id, out var prevPath))
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingDuplicateHardlink", "Excluding hardlink ({1}) for: {0}, previous hardlink: {2}", path, id, prevPath);
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingDuplicateHardlink", "Excluding hardlink ({1}) for: {0}, previous hardlink: {2}", entry.Path, id, prevPath);
return false;
}
else
{
hardlinkmap.Add(id, path);
hardlinkmap.Add(id, entry.Path);
}
}
}
}
catch (Exception ex)
{
Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingErrorHardLink", ex, "Failed to process path: {0}", path);
Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingErrorHardLink", ex, "Failed to process path: {0}", entry.Path);
return false;
}
}
if (ignorenames != null && (attributes & FileAttributes.Directory) != 0)
// Check if there is an ignore marker file
if (ignorenames != null && entry.IsFolder)
{
try
{
foreach (var n in ignorenames)
{
var ignorepath = SystemIO.IO_OS.PathCombine(path, n);
if (snapshot.FileExists(ignorepath))
if (await entry.FileExists(n, cancellationToken).ConfigureAwait(false))
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingPathDueToIgnoreFile", "Excluding path because ignore file was found: {0}", ignorepath);
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingPathDueToIgnoreFile", "Excluding path because ignore file {0} was found in: {1}", n, entry.Path);
return false;
}
}
}
catch (Exception ex)
{
Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingErrorIgnoreFile", ex, "Failed to process path: {0}", path);
Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingErrorIgnoreFile", ex, "Failed to process path: {0}", entry.Path);
}
}
// If we exclude files based on attributes, filter that
if ((fileAttributes & attributes) != 0)
// Setup some basic processing attributes
var attributes = entry.IsFolder
? FileAttributes.Directory
: FileAttributes.Normal;
try
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingPathFromAttributes", "Excluding path due to attribute filter: {0}", path);
attributes = entry.Attributes;
}
catch (Exception ex)
{
Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingErrorAttributes", ex, "Failed to process path, using default attributes: {0}", entry.Path);
}
// If we exclude files based on attributes, filter that
if ((fileAttributeFilter & attributes) != 0)
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingPathFromAttributes", "Excluding path due to attribute filter: {0}", entry.Path);
return false;
}
// Then check if the filename is not explicitly excluded by a filter
Library.Utility.IFilter match;
var filtermatch = false;
if (!Library.Utility.FilterExpression.Matches(enumeratefilter, path, out match))
if (!Library.Utility.FilterExpression.Matches(enumeratefilter, entry.Path, out var match))
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingPathFromFilter", "Excluding path due to filter: {0} => {1}", path, match == null ? "null" : match.ToString());
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingPathFromFilter", "Excluding path due to filter: {0} => {1}", entry.Path, match == null ? "null" : match.ToString());
return false;
}
else if (match != null)
{
filtermatch = true;
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "IncludingPathFromFilter", "Including path due to filter: {0} => {1}", path, match.ToString());
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "IncludingPathFromFilter", "Including path due to filter: {0} => {1}", entry.Path, match.ToString());
}
// If the file is a symlink, apply special handling
var isSymlink = snapshot.IsSymlink(path, attributes);
string symlinkTarget = null;
if (isSymlink)
try { symlinkTarget = snapshot.GetSymlinkTarget(path); }
catch (Exception ex) { Logging.Log.WriteExplicitMessage(FILTER_LOGTAG, "SymlinkTargetReadError", ex, "Failed to read symlink target for path: {0}", path); }
string? symlinkTarget = null;
try
{
symlinkTarget = entry.SymlinkTarget;
}
catch (Exception ex)
{
Logging.Log.WriteExplicitMessage(FILTER_LOGTAG, "SymlinkTargetReadError", ex, "Failed to read symlink target for path: {0}", entry.Path);
}
if (isSymlink)
if (symlinkTarget != null)
{
if (!string.IsNullOrWhiteSpace(symlinkTarget))
{
if (symlinkPolicy == Options.SymlinkStrategy.Ignore)
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludeSymlink", "Excluding symlink: {0}", path);
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludeSymlink", "Excluding symlink: {0}", entry.Path);
return false;
}
if (symlinkPolicy == Options.SymlinkStrategy.Store)
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "StoreSymlink", "Storing symlink: {0}", path);
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "StoreSymlink", "Storing symlink: {0}", entry.Path);
// We return false because we do not want to recurse into the path,
// but we add the symlink to the mixin so we process the symlink itself
mixinqueue.Enqueue(path);
mixinqueue.Enqueue(entry);
return false;
}
}
else
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "FollowingEmptySymlink", "Treating empty symlink as regular path {0}", path);
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "FollowingEmptySymlink", "Treating empty symlink as regular path {0}", entry.Path);
}
}
if (!filtermatch)
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "IncludingPath", "Including path as no filters matched: {0}", path);
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "IncludingPath", "Including path as no filters matched: {0}", entry.Path);
// All the way through, yes!
return true;
Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs
+13 -13
View File
@@ -40,7 +40,7 @@ namespace Duplicati.Library.Main.Operation.Backup
/// </summary>
private static readonly string FILELOGTAG = Logging.Log.LogTagFromType(typeof(FilePreFilterProcess)) + ".FileEntry";
public static Task Run(Channels channels, Snapshots.ISnapshotService snapshot, Options options, BackupStatsCollector stats, BackupDatabase database)
public static Task Run(Channels channels, Options options, BackupStatsCollector stats, BackupDatabase database)
{
return AutomationExtensions.RunTask(
new
@@ -73,11 +73,11 @@ namespace Duplicati.Library.Main.Operation.Backup
long filestatsize = -1;
try
{
filestatsize = snapshot.GetFileSize(e.Path);
filestatsize = e.Entry.Size;
}
catch (Exception ex)
{
Logging.Log.WriteExplicitMessage(FILELOGTAG, "FailedToReadSize", ex, "Failed to read size of file: {0}", e.Path);
Logging.Log.WriteExplicitMessage(FILELOGTAG, "FailedToReadSize", ex, "Failed to read size of file: {0}", e.Entry.Path);
}
await stats.AddExaminedFile(filestatsize);
@@ -86,8 +86,8 @@ namespace Duplicati.Library.Main.Operation.Backup
var tooLargeFile = SKIPFILESLARGERTHAN != 0 && filestatsize >= 0 && filestatsize > SKIPFILESLARGERTHAN;
if (tooLargeFile)
{
Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckTooLarge", "Skipped checking file, because the size exceeds limit {0}", e.Path);
await self.ProgressChannel.WriteAsync(new ProgressEvent() { Filepath = e.Path, Length = filestatsize, Type = EventType.FileSkipped });
Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckTooLarge", "Skipped checking file, because the size exceeds limit {0}", e.Entry.Path);
await self.ProgressChannel.WriteAsync(new ProgressEvent() { Filepath = e.Entry.Path, Length = filestatsize, Type = EventType.FileSkipped });
continue;
}
@@ -102,7 +102,7 @@ namespace Duplicati.Library.Main.Operation.Backup
// and we only check for timestamp changes
if (CHECKFILETIMEONLY && !e.TimestampChanged && !isNewFile)
{
Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoTimestampChange", "Skipped checking file, because timestamp was not updated {0}", e.Path);
Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoTimestampChange", "Skipped checking file, because timestamp was not updated {0}", e.Entry.Path);
try
{
await database.AddUnmodifiedAsync(e.OldId, e.LastWrite);
@@ -111,9 +111,9 @@ namespace Duplicati.Library.Main.Operation.Backup
{
if (ex.IsRetiredException())
throw;
Logging.Log.WriteWarningMessage(FILELOGTAG, "FailedToAddFile", ex, "Failed while attempting to add unmodified file to database: {0}", e.Path);
Logging.Log.WriteWarningMessage(FILELOGTAG, "FailedToAddFile", ex, "Failed while attempting to add unmodified file to database: {0}", e.Entry.Path);
}
await self.ProgressChannel.WriteAsync(new ProgressEvent() { Filepath = e.Path, Length = filestatsize, Type = EventType.FileSkipped });
await self.ProgressChannel.WriteAsync(new ProgressEvent() { Filepath = e.Entry.Path, Length = filestatsize, Type = EventType.FileSkipped });
continue;
}
@@ -130,28 +130,28 @@ namespace Duplicati.Library.Main.Operation.Backup
}
// Compute current metadata
e.MetaHashAndSize = SKIPMETADATA ? EMPTY_METADATA : Utility.WrapMetadata(MetadataGenerator.GenerateMetadata(e.Path, e.Attributes, options, snapshot), options);
e.MetaHashAndSize = SKIPMETADATA ? EMPTY_METADATA : Utility.WrapMetadata(MetadataGenerator.GenerateMetadata(e.Entry, e.Attributes, options), options);
e.MetadataChanged = !SKIPMETADATA && (e.MetaHashAndSize.Blob.Length != e.OldMetaSize || e.MetaHashAndSize.FileHash != e.OldMetaHash);
// Check if the file is new, or something indicates a change
var filesizeChanged = filestatsize < 0 || e.LastFileSize < 0 || filestatsize != e.LastFileSize;
if (isNewFile || e.TimestampChanged || filesizeChanged || e.MetadataChanged)
{
Logging.Log.WriteVerboseMessage(FILELOGTAG, "CheckFileForChanges", "Checking file for changes {0}, new: {1}, timestamp changed: {2}, size changed: {3}, metadatachanged: {4}, {5} vs {6}", e.Path, isNewFile, e.TimestampChanged, filesizeChanged, e.MetadataChanged, e.LastWrite, e.OldModified);
Logging.Log.WriteVerboseMessage(FILELOGTAG, "CheckFileForChanges", "Checking file for changes {0}, new: {1}, timestamp changed: {2}, size changed: {3}, metadatachanged: {4}, {5} vs {6}", e.Entry.Path, isNewFile, e.TimestampChanged, filesizeChanged, e.MetadataChanged, e.LastWrite, e.OldModified);
await self.Output.WriteAsync(e);
}
else
{
Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoMetadataChange", "Skipped checking file, because no metadata was updated {0}", e.Path);
Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoMetadataChange", "Skipped checking file, because no metadata was updated {0}", e.Entry.Path);
try
{
await database.AddUnmodifiedAsync(e.OldId, e.LastWrite);
}
catch (Exception ex)
{
Logging.Log.WriteWarningMessage(FILELOGTAG, "FailedToAddFile", ex, "Failed while attempting to add unmodified file to database: {0}", e.Path);
Logging.Log.WriteWarningMessage(FILELOGTAG, "FailedToAddFile", ex, "Failed while attempting to add unmodified file to database: {0}", e.Entry.Path);
}
await self.ProgressChannel.WriteAsync(new ProgressEvent() { Filepath = e.Path, Length = filestatsize, Type = EventType.FileSkipped });
await self.ProgressChannel.WriteAsync(new ProgressEvent() { Filepath = e.Entry.Path, Length = filestatsize, Type = EventType.FileSkipped });
}
}
});
Duplicati/Library/Main/Operation/Backup/MetadataGenerator.cs
+28 -31
View File
@@ -1,30 +1,27 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Threading.Tasks;
using System.Collections.Generic;
using CoCoL;
using Duplicati.Library.Main.Operation.Common;
using Duplicati.Library.Snapshots;
using Duplicati.Library.Interface;
namespace Duplicati.Library.Main.Operation.Backup
{
@@ -35,7 +32,7 @@ namespace Duplicati.Library.Main.Operation.Backup
{
private static readonly string METALOGTAG = Logging.Log.LogTagFromType(typeof(MetadataGenerator)) + ".Metadata";
public static Dictionary<string, string> GenerateMetadata(string path, System.IO.FileAttributes attributes, Options options, Snapshots.ISnapshotService snapshot)
public static Dictionary<string, string> GenerateMetadata(ISourceProviderEntry entry, System.IO.FileAttributes attributes, Options options)
{
try
{
@@ -43,7 +40,7 @@ namespace Duplicati.Library.Main.Operation.Backup
if (!options.SkipMetadata)
{
metadata = snapshot.GetMetadata(path, snapshot.IsSymlink(path, attributes), options.SymlinkPolicy == Options.SymlinkStrategy.Follow);
metadata = entry.MinorMetadata;
if (metadata == null)
metadata = new Dictionary<string, string>();
@@ -54,11 +51,11 @@ namespace Duplicati.Library.Main.Operation.Backup
{
try
{
metadata["CoreLastWritetime"] = snapshot.GetLastWriteTimeUtc(path).Ticks.ToString();
metadata["CoreLastWritetime"] = entry.LastModificationUtc.Ticks.ToString();
}
catch (Exception ex)
{
Logging.Log.WriteWarningMessage(METALOGTAG, "TimestampReadFailed", ex, "Failed to read timestamp on \"{0}\"", path);
Logging.Log.WriteWarningMessage(METALOGTAG, "TimestampReadFailed", ex, "Failed to read timestamp on \"{0}\"", entry.Path);
}
}
@@ -66,11 +63,11 @@ namespace Duplicati.Library.Main.Operation.Backup
{
try
{
metadata["CoreCreatetime"] = snapshot.GetCreationTimeUtc(path).Ticks.ToString();
metadata["CoreCreatetime"] = entry.CreatedUtc.Ticks.ToString();
}
catch (Exception ex)
{
Logging.Log.WriteWarningMessage(METALOGTAG, "TimestampReadFailed", ex, "Failed to read timestamp on \"{0}\"", path);
Logging.Log.WriteWarningMessage(METALOGTAG, "TimestampReadFailed", ex, "Failed to read timestamp on \"{0}\"", entry.Path);
}
}
}
@@ -81,9 +78,9 @@ namespace Duplicati.Library.Main.Operation.Backup
return metadata;
}
catch(Exception ex)
catch (Exception ex)
{
Logging.Log.WriteWarningMessage(METALOGTAG, "MetadataProcessFailed", ex, "Failed to process metadata for \"{0}\", storing empty metadata", path);
Logging.Log.WriteWarningMessage(METALOGTAG, "MetadataProcessFailed", ex, "Failed to process metadata for \"{0}\", storing empty metadata", entry.Path);
return new Dictionary<string, string>();
}
}
Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs
+29 -28
View File
@@ -24,11 +24,8 @@ using CoCoL;
using System.Threading.Tasks;
using System.IO;
using System.Collections.Generic;
using System.Security.Principal;
using System.Threading;
using Duplicati.Library.Interface;
using Duplicati.Library.Main.Operation.Common;
using Duplicati.Library.Snapshots;
namespace Duplicati.Library.Main.Operation.Backup
{
@@ -43,7 +40,7 @@ namespace Duplicati.Library.Main.Operation.Backup
public class FileEntry
{
// From input
public string Path;
public ISourceProviderEntry Entry;
// Split
public long PathPrefixID;
@@ -66,7 +63,7 @@ namespace Duplicati.Library.Main.Operation.Backup
public bool TimestampChanged;
}
public static Task Run(Channels channels, Snapshots.ISnapshotService snapshot, Options options, BackupDatabase database, long lastfilesetid, ITaskReader taskReader)
public static Task Run(Channels channels, Options options, BackupDatabase database, long lastfilesetid, ITaskReader taskReader)
{
return AutomationExtensions.RunTask(new
{
@@ -85,37 +82,40 @@ namespace Duplicati.Library.Main.Operation.Backup
while (true)
{
var path = await self.Input.ReadAsync();
var entry = await self.Input.ReadAsync();
// We ignore the stop signal, but not the pause and terminate
await taskReader.ProgressRendevouz().ConfigureAwait(false);
var lastwrite = new DateTime(0, DateTimeKind.Utc);
var attributes = default(FileAttributes);
var attributes = entry.IsFolder
? FileAttributes.Directory
: FileAttributes.Normal;
try
{
lastwrite = snapshot.GetLastWriteTimeUtc(path);
lastwrite = entry.LastModificationUtc;
}
catch (Exception ex)
{
Logging.Log.WriteWarningMessage(FILELOGTAG, "TimestampReadFailed", ex, "Failed to read timestamp on \"{0}\"", path);
Logging.Log.WriteWarningMessage(FILELOGTAG, "TimestampReadFailed", ex, "Failed to read timestamp on \"{0}\"", entry.Path);
}
try
{
attributes = snapshot.GetAttributes(path);
attributes = entry.Attributes;
}
catch (Exception ex)
{
Logging.Log.WriteVerboseMessage(FILELOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", path, ex.Message);
Logging.Log.WriteVerboseMessage(FILELOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", entry.Path, ex.Message);
}
// If we only have metadata, stop here
if (await ProcessMetadata(path, attributes, lastwrite, options, snapshot, emptymetadata, database, self.StreamBlockChannel).ConfigureAwait(false))
if (await ProcessMetadata(entry, attributes, lastwrite, options, emptymetadata, database, self.StreamBlockChannel).ConfigureAwait(false))
{
try
{
var split = Database.LocalDatabase.SplitIntoPrefixAndName(path);
var split = Database.LocalDatabase.SplitIntoPrefixAndName(entry.Path);
long prefixid;
if (string.Equals(prevprefix.Key, split.Key, StringComparison.Ordinal))
@@ -132,7 +132,7 @@ namespace Duplicati.Library.Main.Operation.Backup
await self.Output.WriteAsync(new FileEntry
{
OldId = tmp.Item1,
Path = path,
Entry = entry,
PathPrefixID = prefixid,
Filename = split.Value,
Attributes = attributes,
@@ -149,7 +149,7 @@ namespace Duplicati.Library.Main.Operation.Backup
await self.Output.WriteAsync(new FileEntry
{
OldId = res == null ? -1 : res.id,
Path = path,
Entry = entry,
PathPrefixID = prefixid,
Filename = split.Value,
Attributes = attributes,
@@ -167,7 +167,7 @@ namespace Duplicati.Library.Main.Operation.Backup
continue;
Logging.Log.WriteWarningMessage(FILELOGTAG, "ProcessingMetadataFailed", ex,
"Failed to process entry, path: {0}", path);
"Failed to process entry, path: {0}", entry.Path);
}
}
}
@@ -180,9 +180,9 @@ namespace Duplicati.Library.Main.Operation.Backup
/// Processes the metadata for the given path.
/// </summary>
/// <returns><c>True</c> if the path should be submitted to more analysis, <c>false</c> if there is nothing else to do</returns>
private static async Task<bool> ProcessMetadata(string path, FileAttributes attributes, DateTime lastwrite, Options options, Snapshots.ISnapshotService snapshot, IMetahash emptymetadata, BackupDatabase database, IWriteChannel<StreamBlock> streamblockchannel)
private static async Task<bool> ProcessMetadata(ISourceProviderEntry entry, FileAttributes attributes, DateTime lastwrite, Options options, IMetahash emptymetadata, BackupDatabase database, IWriteChannel<StreamBlock> streamblockchannel)
{
if (snapshot.IsSymlink(path, attributes))
if (entry.IsSymlink)
{
// Not all reparse points are symlinks.
// For example, on Windows 10 Fall Creator's Update, the OneDrive folder (and all subfolders)
@@ -191,57 +191,58 @@ namespace Duplicati.Library.Main.Operation.Backup
string symlinkTarget = null;
try
{
symlinkTarget = snapshot.GetSymlinkTarget(path);
symlinkTarget = entry.SymlinkTarget;
}
catch (Exception ex)
{
Logging.Log.WriteExplicitMessage(FILELOGTAG, "SymlinkTargetReadFailure", ex, "Failed to read symlink target for path: {0}", path);
Logging.Log.WriteExplicitMessage(FILELOGTAG, "SymlinkTargetReadFailure", ex, "Failed to read symlink target for path: {0}", entry.Path);
}
if (!string.IsNullOrWhiteSpace(symlinkTarget))
{
if (options.SymlinkPolicy == Options.SymlinkStrategy.Ignore)
{
Logging.Log.WriteVerboseMessage(FILELOGTAG, "IgnoreSymlink", "Ignoring symlink {0}", path);
Logging.Log.WriteVerboseMessage(FILELOGTAG, "IgnoreSymlink", "Ignoring symlink {0}", entry.Path);
return false;
}
if (options.SymlinkPolicy == Options.SymlinkStrategy.Store)
{
var metadata = MetadataGenerator.GenerateMetadata(path, attributes, options, snapshot);
var metadata = MetadataGenerator.GenerateMetadata(entry, attributes, options);
if (!metadata.ContainsKey("CoreSymlinkTarget"))
metadata["CoreSymlinkTarget"] = symlinkTarget;
var metahash = Utility.WrapMetadata(metadata, options);
await AddSymlinkToOutputAsync(path, DateTime.UtcNow, metahash, database, streamblockchannel).ConfigureAwait(false);
await AddSymlinkToOutputAsync(entry.Path, DateTime.UtcNow, metahash, database, streamblockchannel).ConfigureAwait(false);
Logging.Log.WriteVerboseMessage(FILELOGTAG, "StoreSymlink", "Stored symlink {0}", path);
Logging.Log.WriteVerboseMessage(FILELOGTAG, "StoreSymlink", "Stored symlink {0}", entry.Path);
// Don't process further
return false;
}
}
else
{
Logging.Log.WriteVerboseMessage(FILELOGTAG, "FollowingEmptySymlink", "Treating empty symlink as regular path {0}", path);
Logging.Log.WriteVerboseMessage(FILELOGTAG, "FollowingEmptySymlink", "Treating empty symlink as regular path {0}", entry.Path);
}
}
if ((attributes & FileAttributes.Directory) == FileAttributes.Directory)
{
IMetahash metahash;
if (!options.SkipMetadata)
{
metahash = Utility.WrapMetadata(MetadataGenerator.GenerateMetadata(path, attributes, options, snapshot), options);
metahash = Utility.WrapMetadata(MetadataGenerator.GenerateMetadata(entry, attributes, options), options);
}
else
{
metahash = emptymetadata;
}
Logging.Log.WriteVerboseMessage(FILELOGTAG, "AddDirectory", "Adding directory {0}", path);
await AddFolderToOutputAsync(path, lastwrite, metahash, database, streamblockchannel).ConfigureAwait(false);
Logging.Log.WriteVerboseMessage(FILELOGTAG, "AddDirectory", "Adding directory {0}", entry.Path);
await AddFolderToOutputAsync(entry.Path, lastwrite, metahash, database, streamblockchannel).ConfigureAwait(false);
return false;
}
Duplicati/Library/Main/Operation/BackupHandler.cs
+130 -38
View File
@@ -34,6 +34,11 @@ using Duplicati.Library.Common.IO;
using Duplicati.Library.Logging;
using Duplicati.Library.Main.Operation.Common;
using System.Data;
using Duplicati.Library.DynamicLoader;
using Duplicati.Library.SourceProvider;
using Duplicati.Library.Snapshots.USN;
using System.Text.RegularExpressions;
using System.IO;
namespace Duplicati.Library.Main.Operation
{
@@ -74,53 +79,142 @@ namespace Duplicati.Library.Main.Operation
throw new UserInformationException(Strings.Common.PassphraseChangeUnsupported, "PassphraseChangeUnsupported");
}
public static Snapshots.ISnapshotService GetSnapshot(string[] sources, Options options)
/// <summary>
/// Gets a single source provider for the given sources
/// </summary>
/// <param name="sources">The sources to get providers for</param>
/// <param name="options">The options to use</param>
/// <returns>The source providers</returns>
public static async Task<ISourceProvider> GetSourceProvider(IEnumerable<string> sources, Options options, CancellationToken cancellationToken)
=> Combiner.Combine(await GetSourceProviders(sources, options, cancellationToken));
/// <summary>
/// Gets a snapshot service for the given sources
/// </summary>
/// <param name="sources">The sources to get the snapshot for</param>
/// <param name="options">The options to use</param>
/// <returns>The source provider</returns>
private static ISnapshotService GetFileSnapshotService(IEnumerable<string> sources, Options options)
{
try
{
if (options.SnapShotStrategy != Options.OptimizationStrategy.Off)
return Duplicati.Library.Snapshots.SnapshotUtility.CreateSnapshot(sources, options.RawOptions);
return SnapshotUtility.CreateSnapshot(sources, options.RawOptions, options.SymlinkPolicy == Options.SymlinkStrategy.Follow);
}
catch (Exception ex)
{
if (options.SnapShotStrategy == Options.OptimizationStrategy.Required)
throw new UserInformationException(Strings.Common.SnapshotFailedError(ex.Message), "SnapshotFailed", ex);
else if (options.SnapShotStrategy == Options.OptimizationStrategy.On)
Logging.Log.WriteWarningMessage(LOGTAG, "SnapshotFailed", ex, Strings.Common.SnapshotFailedError(ex.Message));
Log.WriteWarningMessage(LOGTAG, "SnapshotFailed", ex, Strings.Common.SnapshotFailedError(ex.Message));
else if (options.SnapShotStrategy == Options.OptimizationStrategy.Auto)
Logging.Log.WriteInformationMessage(LOGTAG, "SnapshotFailed", Strings.Common.SnapshotFailedError(ex.Message));
Log.WriteInformationMessage(LOGTAG, "SnapshotFailed", Strings.Common.SnapshotFailedError(ex.Message));
}
if (OperatingSystem.IsMacOS() || OperatingSystem.IsLinux())
{
return new NoSnapshotLinux(options.IgnoreAdvisoryLocking);
}
else if (OperatingSystem.IsWindows())
{
return new NoSnapshotWindows();
}
else
{
throw new NotSupportedException("Unsupported Operating System");
}
return SnapshotUtility.CreateNoSnapshot(sources, options.IgnoreAdvisoryLocking, options.SymlinkPolicy == Options.SymlinkStrategy.Follow);
}
/// <summary>
/// Create instance of USN journal service
/// Gets all source providers for the given sources
/// </summary>
/// <param name="sources"></param>
/// <param name="snapshot"></param>
/// <param name="filter"></param>
/// <param name="lastfilesetid"></param>
/// <returns></returns>
private UsnJournalService GetJournalService(IEnumerable<string> sources, ISnapshotService snapshot, IFilter filter, long lastfilesetid)
/// <param name="sources">The sources to get providers for</param>
/// <param name="options">The options to use</param>
/// <param name="filter">The filter to use</param>
/// <param name="lastfilesetid">The last fileset id</param>
/// <returns>The source providers</returns>
private static async Task<List<ISourceProvider>> GetSourceProviders(IEnumerable<string> sources, Options options, CancellationToken cancellationToken)
{
// Group the sources by their type, so we can combine all snapshot paths into a single snapshot
var sourceTypes = sources.GroupBy(x => x.StartsWith("@") ? "@" : Library.Utility.Utility.GuessScheme(x) ?? "file", StringComparer.OrdinalIgnoreCase);
// To avoid leaking snapshot instances, we create all instances first and then dispose them if an exception occurs
// The number of instances is expected to be low, so the memory overhead is acceptable
var results = new List<ISourceProvider>();
try
{
foreach (var entry in sourceTypes)
{
if ("file".Equals(entry.Key, StringComparison.OrdinalIgnoreCase))
results.Add(new LocalFileSource(GetFileSnapshotService(entry, options)));
else if ("vss".Equals(entry.Key, StringComparison.OrdinalIgnoreCase) || "lvm".Equals(entry.Key, StringComparison.OrdinalIgnoreCase))
results.Add(new LocalFileSource(SnapshotUtility.CreateSnapshot(entry, options.RawOptions, options.SymlinkPolicy == Options.SymlinkStrategy.Follow)));
else if ("@".Equals(entry.Key, StringComparison.OrdinalIgnoreCase))
{
foreach (var url in entry)
{
var sanitizedUrl = Library.Utility.Utility.GetUrlWithoutCredentials(url);
var m = Regex.Match(url, @"^@(?<mountpoint>[^|]+)\|(?<url>.+)$", RegexOptions.IgnoreCase);
if (m.Success)
{
var mountpoint = m.Groups["mountpoint"].Value;
if (mountpoint.Any(x => Path.GetInvalidPathChars().Contains(x)))
throw new UserInformationException(string.Format("The mountpoint \"{0}\" contains invalid characters", mountpoint), "InvalidMountpoint");
if (!Path.IsPathRooted(mountpoint))
throw new UserInformationException(string.Format("The mountpoint \"{0}\" is not a valid rooted mountpoint", mountpoint), "InvalidMountpoint");
var backendurl = m.Groups["url"].Value;
ISourceProvider provider;
try
{
provider = await SourceProviderLoader.GetSourceProvider(backendurl, Path.GetFullPath(mountpoint), options.RawOptions, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
if (options.AllowMissingSource)
{
Log.WriteWarningMessage(LOGTAG, "SourceProviderFailed", ex, "Failed to load source provider for \"{0}\"", sanitizedUrl);
continue;
}
throw new UserInformationException($"Failed to load source provider for \"{sanitizedUrl}\": {ex.Message}", "SourceProviderFailed", ex);
}
// Don't accept missing providers
results.Add(provider ?? throw new UserInformationException($"The source \"{sanitizedUrl}\" is not supported", "SourceNotSupported"));
}
else
throw new UserInformationException($"The source \"{sanitizedUrl}\" is not a supported format", "SourceFormatNotSupported");
}
}
else
throw new UserInformationException($"The source type \"{entry.Key}\" is not supported", "SourceTypeNotSupported");
}
}
catch
{
foreach (var provider in results)
(provider as IDisposable)?.Dispose();
throw;
}
if (results.Count == 0)
throw new UserInformationException("No sources were available for the backup", "NoSourcesAvailable");
return results;
}
public UsnJournalService GetJournalService(ISourceProvider provider, IFilter filter, long lastfilesetid)
{
if (m_options.UsnStrategy == Options.OptimizationStrategy.Off) return null;
if (!OperatingSystem.IsWindows())
throw new UserInformationException("USN journal is only supported on Windows", "UsnJournalNotSupported");
var providers = (provider is Combiner c ? c.Providers : [provider])
.Select((x, i) => new { Provider = x, Index = i })
.Where(x => x.Provider is LocalFileSource)
.ToList();
if (providers.Count == 0)
return null;
if (providers.Count > 1)
throw new UserInformationException("Multiple USN journal services are not supported", "MultipleUSNJournalServices");
var fileProvider = providers.First().Provider as LocalFileSource;
var journalData = m_database.GetChangeJournalData(lastfilesetid);
var service = new UsnJournalService(sources, snapshot, filter, m_options.FileAttributeFilter, m_options.SkipFilesLargerThan,
var service = new UsnJournalService(fileProvider.SnapshotService, filter, m_options.FileAttributeFilter, m_options.SkipFilesLargerThan,
journalData, cancellationTokenSource.Token);
foreach (var volumeData in service.VolumeDataList)
@@ -288,7 +382,7 @@ namespace Duplicati.Library.Main.Operation
/// <summary>
/// Performs the bulk of work by starting all relevant processes
/// </summary>
private static async Task RunMainOperation(Backup.Channels channels, IEnumerable<string> sources, Snapshots.ISnapshotService snapshot, UsnJournalService journalService, Backup.BackupDatabase database, IBackendManager backendManager, Backup.BackupStatsCollector stats, Options options, IFilter sourcefilter, IFilter filter, BackupResults result, ITaskReader taskreader, long filesetid, long lastfilesetid)
private static async Task RunMainOperation(Backup.Channels channels, ISourceProvider source, UsnJournalService journalService, Backup.BackupDatabase database, IBackendManager backendManager, Backup.BackupStatsCollector stats, Options options, IFilter sourcefilter, IFilter filter, BackupResults result, ITaskReader taskreader, long filesetid, long lastfilesetid)
{
using (new Logging.Timer(LOGTAG, "BackupMainOperation", "BackupMainOperation"))
{
@@ -299,15 +393,15 @@ namespace Duplicati.Library.Main.Operation
new[]
{
Backup.DataBlockProcessor.Run(channels, database, backendManager, options, taskreader),
Backup.FileBlockProcessor.Run(channels, snapshot, options, database, stats, taskreader),
Backup.FileBlockProcessor.Run(channels, options, database, stats, taskreader),
Backup.StreamBlockSplitter.Run(channels, options, database, taskreader),
Backup.FileEnumerationProcess.Run(channels, sources, snapshot, journalService,
options.FileAttributeFilter, sourcefilter, filter, options.SymlinkPolicy,
Backup.FileEnumerationProcess.Run(channels, source, journalService,
options.FileAttributeFilter, filter, options.SymlinkPolicy,
options.HardlinkPolicy, options.ExcludeEmptyFolders, options.IgnoreFilenames,
GetBlacklistedPaths(options), options.ChangedFilelist, taskreader,
() => result.PartialBackup = true, CancellationToken.None),
Backup.FilePreFilterProcess.Run(channels, snapshot, options, stats, database),
Backup.MetadataPreProcess.Run(channels, snapshot, options, database, lastfilesetid, taskreader),
Backup.FilePreFilterProcess.Run(channels, options, stats, database),
Backup.MetadataPreProcess.Run(channels, options, database, lastfilesetid, taskreader),
Backup.SpillCollectorProcess.Run(channels, options, database, backendManager, taskreader),
Backup.ProgressHandler.Run(channels, result)
}
@@ -324,7 +418,7 @@ namespace Duplicati.Library.Main.Operation
// Spawn additional file processors
.Concat(
Enumerable.Range(0, options.ConcurrencyFileprocessors - 1).Select(x =>
Backup.FileBlockProcessor.Run(channels, snapshot, options, database, stats, taskreader)
Backup.FileBlockProcessor.Run(channels, options, database, stats, taskreader)
)
)
);
@@ -351,9 +445,8 @@ namespace Duplicati.Library.Main.Operation
return true;
if (fileSize >= 0)
{
stats.AddExaminedFile(fileSize);
}
return false;
});
@@ -528,7 +621,7 @@ namespace Duplicati.Library.Main.Operation
{
long filesetid;
using var counterToken = CancellationTokenSource.CreateLinkedTokenSource(m_taskReader.ProgressToken);
using (var snapshot = GetSnapshot(sources, m_options))
using (var source = await GetSourceProvider(sources, m_options, m_taskReader.ProgressToken).ConfigureAwait(false))
{
try
{
@@ -558,8 +651,7 @@ namespace Duplicati.Library.Main.Operation
var filesetvolumeid = await db.RegisterRemoteVolumeAsync(filesetvolume.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary);
filesetid = await db.CreateFilesetAsync(filesetvolumeid, VolumeBase.ParseFilename(filesetvolume.RemoteFilename).Time);
// create USN-based scanner if enabled
var journalService = GetJournalService(sources, snapshot, filter, lastfilesetid);
var journalService = GetJournalService(source, filter, lastfilesetid);
// Start parallel scan, or use the database
if (m_options.DisableFileScanner)
@@ -569,13 +661,13 @@ namespace Duplicati.Library.Main.Operation
}
else
{
parallelScanner = Backup.CountFilesHandler.Run(sources, snapshot, journalService, m_result, m_options, m_sourceFilter, m_filter, GetBlacklistedPaths(), m_result.TaskControl, counterToken.Token);
parallelScanner = Backup.CountFilesHandler.Run(source, journalService, m_result, m_options, m_filter, GetBlacklistedPaths(), m_result.TaskControl, counterToken.Token);
}
// Run the backup operation
if (await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
{
await RunMainOperation(channels, sources, snapshot, journalService, db, backendManager, stats, m_options, m_sourceFilter, m_filter, m_result, m_result.TaskControl, filesetid, lastfilesetid).ConfigureAwait(false);
await RunMainOperation(channels, source, journalService, db, backendManager, stats, m_options, m_sourceFilter, m_filter, m_result, m_result.TaskControl, filesetid, lastfilesetid).ConfigureAwait(false);
}
}
finally
Duplicati/Library/Main/Operation/FilelistProcessor.cs
-1
View File
@@ -26,7 +26,6 @@ using System.Data;
using System.Linq;
using Duplicati.Library.Interface;
using System.Threading;
using Duplicati.Library.Utility;
using System.Threading.Tasks;
namespace Duplicati.Library.Main.Operation
Duplicati/Library/Main/Operation/TestFilterHandler.cs
+55 -44
View File
@@ -1,27 +1,28 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
using System;
using System.IO;
using Duplicati.Library.Snapshots;
using CoCoL;
using System.Threading.Tasks;
@@ -45,14 +46,13 @@ namespace Duplicati.Library.Main.Operation
public async Task RunAsync(string[] sources, Library.Utility.IFilter filter)
{
var sourcefilter = new Library.Utility.FilterExpression(sources, true);
var stopToken = m_result.TaskControl.ProgressToken;
using (var snapshot = BackupHandler.GetSnapshot(sources, m_options))
using (var provider = await BackupHandler.GetSourceProvider(sources, m_options, stopToken).ConfigureAwait(false))
{
Backup.Channels channels = new();
var source = Backup.FileEnumerationProcess.Run(channels, sources, snapshot, null,
m_options.FileAttributeFilter, sourcefilter, filter, m_options.SymlinkPolicy,
var source = Backup.FileEnumerationProcess.Run(channels, provider, null,
m_options.FileAttributeFilter, filter, m_options.SymlinkPolicy,
m_options.HardlinkPolicy, m_options.ExcludeEmptyFolders, m_options.IgnoreFilenames,
BackupHandler.GetBlacklistedPaths(m_options), null, m_result.TaskControl, null, stopToken);
@@ -64,20 +64,20 @@ namespace Duplicati.Library.Main.Operation
{
while (await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
{
var path = await self.source.ReadAsync();
var fa = FileAttributes.Normal;
try { fa = snapshot.GetAttributes(path); }
catch (Exception ex) { Logging.Log.WriteVerboseMessage(LOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", path, ex.Message); }
var entry = await self.source.ReadAsync();
var fa = entry.IsFolder
? FileAttributes.Directory
: FileAttributes.Normal;
try { fa = entry.Attributes; }
catch (Exception ex) { Logging.Log.WriteVerboseMessage(LOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", entry.Path, ex.Message); }
// Analyze symlinks
var isSymlink = snapshot.IsSymlink(path, fa);
string symlinkTarget = null;
string? symlinkTarget = null;
try { symlinkTarget = entry.SymlinkTarget; }
catch (Exception ex) { Logging.Log.WriteExplicitMessage(LOGTAG, "SymlinkTargetReadFailure", ex, "Failed to read symlink target for path: {0}", entry.Path); }
if (isSymlink)
try { symlinkTarget = snapshot.GetSymlinkTarget(path); }
catch (Exception ex) { Logging.Log.WriteExplicitMessage(LOGTAG, "SymlinkTargetReadFailure", ex, "Failed to read symlink target for path: {0}", path); }
if (isSymlink && m_options.SymlinkPolicy == Options.SymlinkStrategy.Store && !string.IsNullOrWhiteSpace(symlinkTarget))
if (m_options.SymlinkPolicy == Options.SymlinkStrategy.Store && !string.IsNullOrWhiteSpace(symlinkTarget))
{
// Skip stored symlinks
continue;
@@ -86,33 +86,44 @@ namespace Duplicati.Library.Main.Operation
// Go for the symlink target, as we know we follow symlinks
if (!string.IsNullOrWhiteSpace(symlinkTarget))
{
path = symlinkTarget;
var targetEntry = await provider.GetEntry(symlinkTarget, false, stopToken).ConfigureAwait(false);
fa = FileAttributes.Normal;
try { fa = snapshot.GetAttributes(path); }
catch (Exception ex) { Logging.Log.WriteVerboseMessage(LOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", path, ex.Message); }
try { fa = targetEntry!.Attributes; }
catch (Exception ex) { Logging.Log.WriteVerboseMessage(LOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", entry.Path, ex.Message); }
// If we guessed wrong and the symlink target is a folder, we need to fetch it with the correct flag
if (fa.HasFlag(FileAttributes.Directory))
targetEntry = await provider.GetEntry(symlinkTarget, true, stopToken).ConfigureAwait(false);
// No such target
if (targetEntry == null)
{
Logging.Log.WriteVerboseMessage(LOGTAG, "SymlinkTargetNotFound", "Symlink target not found: {0}", symlinkTarget);
continue;
}
}
// Proceed with non-folders
if (!((fa & FileAttributes.Directory) == FileAttributes.Directory))
if (!entry.IsFolder)
{
m_result.FileCount++;
var size = -1L;
try
{
size = snapshot.GetFileSize(path);
size = entry.Size;
m_result.FileSize += size;
}
catch (Exception ex)
{
Logging.Log.WriteVerboseMessage(LOGTAG, "SizeReadFailed", "Failed to read length of file {0}: {1}", path, ex.Message);
Logging.Log.WriteVerboseMessage(LOGTAG, "SizeReadFailed", "Failed to read length of file {0}: {1}", entry.Path, ex.Message);
}
if (m_options.SkipFilesLargerThan == long.MaxValue || m_options.SkipFilesLargerThan == 0 || size < m_options.SkipFilesLargerThan)
Logging.Log.WriteVerboseMessage(LOGTAG, "IncludeFile", "Including file: {0} ({1})", path, size < 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(size));
Logging.Log.WriteVerboseMessage(LOGTAG, "IncludeFile", "Including file: {0} ({1})", entry.Path, size < 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(size));
else
Logging.Log.WriteVerboseMessage(LOGTAG, "ExcludeLargeFile", "Excluding file due to size: {0} ({1})", path, size < 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(size));
Logging.Log.WriteVerboseMessage(LOGTAG, "ExcludeLargeFile", "Excluding file due to size: {0} ({1})", entry.Path, size < 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(size));
}
}
}
Duplicati/Library/Main/Options.cs
+20 -1
View File
@@ -345,6 +345,7 @@ namespace Duplicati.Library.Main
new CommandLineArgument("upload-unchanged-backups", CommandLineArgument.ArgumentType.Boolean, Strings.Options.UploadUnchangedBackupsShort, Strings.Options.UploadUnchangedBackupsLong, "false"),
new CommandLineArgument("snapshot-policy", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.SnapshotpolicyShort, Strings.Options.SnapshotpolicyLong, "off", null, Enum.GetNames(typeof(OptimizationStrategy))),
new CommandLineArgument("snapshot-provider", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.SnapshotproviderShort, Strings.Options.SnapshotproviderLong, OperatingSystem.IsWindows() ? Snapshots.SnapshotProvider.AlphaVSS.ToString() : Snapshots.SnapshotProvider.LVM.ToString(), null, (OperatingSystem.IsWindows() ? [Snapshots.SnapshotProvider.AlphaVSS, Snapshots.SnapshotProvider.Wmic] : new [] { Snapshots.SnapshotProvider.LVM }).Select(x => x.ToString()).ToArray()),
new CommandLineArgument("vss-exclude-writers", CommandLineArgument.ArgumentType.String, Strings.Options.VssexcludewritersShort, Strings.Options.VssexcludewritersLong, "{e8132975-6f93-4464-a53e-1050253ae220}"),
new CommandLineArgument("vss-use-mapping", CommandLineArgument.ArgumentType.Boolean, Strings.Options.VssusemappingShort, Strings.Options.VssusemappingLong, "false"),
new CommandLineArgument("usn-policy", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.UsnpolicyShort, Strings.Options.UsnpolicyLong, "off", null, Enum.GetNames(typeof(OptimizationStrategy))),
@@ -1025,6 +1026,24 @@ namespace Duplicati.Library.Main
}
}
/// <summary>
/// Gets the snapshot strategy to use
/// </summary>
public Snapshots.SnapshotProvider SnapShotProvider
{
get
{
if (!m_options.TryGetValue("snapshot-provider", out var provider))
provider = "";
Snapshots.SnapshotProvider r;
if (!Enum.TryParse(provider, true, out r))
r = OperatingSystem.IsWindows() ? Snapshots.SnapshotProvider.AlphaVSS : Snapshots.SnapshotProvider.LVM;
return r;
}
}
/// <summary>
/// Gets a flag indicating if advisory locking should be ignored
/// </summary>
@@ -1687,7 +1706,7 @@ namespace Duplicati.Library.Main
}
/// <summary>
/// Gets a flag indicating if compacting should not be done automatically
/// Gets a flag indicating if missing source elements should be ignored
/// </summary>
public bool AllowMissingSource
{
Duplicati/Library/Main/Strings.cs
+2
View File
@@ -116,6 +116,8 @@ namespace Duplicati.Library.Main.Strings
public static string EnablemoduleShort { get { return LC.L(@"Enable one or more modules"); } }
public static string SnapshotpolicyLong { get { return LC.L(@"This setting controls the usage of snapshots, which allows Duplicati to backup files that are locked by other programs. If this is set to ""off"", Duplicati will not attempt to create a disk snapshot. Setting this to ""auto"" makes Duplicati attempt to create a snapshot, and fail silently if that was not allowed or supported (note that the OS may still log system warnings). A setting of ""on"" will also make Duplicati attempt to create a snapshot, but will produce a warning message in the log if it fails. Setting it to ""required"" will make Duplicati abort the backup if the snapshot creation fails. On windows this uses the Volume Shadow Copy Services (VSS) and requires administrative privileges. On Linux this uses Logical Volume Management (LVM) and requires root privileges."); } }
public static string SnapshotpolicyShort { get { return LC.L(@"Control the use of disk snapshots"); } }
public static string SnapshotproviderLong { get { return LC.L(@"The snapshot provider implementation for Windows. The AlphaVSS is the most feature complete, but is not supported on Arm64. The WMIC based snapshot has less features, but is more portable. On Linux, only LVM is supported"); } }
public static string SnapshotproviderShort { get { return LC.L(@"The snapshot provider implementation to use"); } }
public static string AsynchronousuploadfolderLong { get { return LC.L(@"The pre-generated volumes will be placed into the temporary folder by default. This option can set a different folder for placing the temporary volumes. Despite the name, this also works for synchronous runs."); } }
public static string AsynchronousuploadfolderShort { get { return LC.L(@"The path where ready volumes are placed until uploaded"); } }
public static string AsynchronousuploadlimitLong { get { return LC.L(@"When performing asynchronous uploads, Duplicati will create volumes that can be uploaded. To prevent Duplicati from generating too many volumes, this option limits the number of pending uploads. Set to zero to disable the limit. The volume(s) that are being created are not counted in this limit. Use the option --concurrency-compressors=1 to limit the number of volumes being created."); } }
Duplicati/Library/Modules/Builtin/HyperVOptions.cs
+22 -21
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
@@ -24,8 +24,8 @@ using System.Collections.Generic;
using System.Linq;
using System.Runtime.Versioning;
using System.Text.RegularExpressions;
using Duplicati.Library.Common;
using Duplicati.Library.Snapshots;
using Duplicati.Library.Snapshots.Windows;
namespace Duplicati.Library.Modules.Builtin
{
@@ -149,7 +149,8 @@ namespace Duplicati.Library.Modules.Builtin
Logging.Log.WriteWarningMessage(LOGTAG, "HyperVOnServerOnly", null, "This is client version of Windows. Hyper-V VSS writer is present only on Server version. Backup will continue, but will be crash consistent only in opposite to application consistent in Server version");
Logging.Log.WriteInformationMessage(LOGTAG, "StartingHyperVQuery", "Starting to gather Hyper-V information");
hypervUtility.QueryHyperVGuestsInfo(true);
var provider = Utility.Utility.ParseEnumOption(changedOptions.AsReadOnly(), "snapshot-provider", SnapshotProvider.AlphaVSS);
hypervUtility.QueryHyperVGuestsInfo(provider, true);
Logging.Log.WriteInformationMessage(LOGTAG, "HyperVMachineCount", "Found {0} virtual machines on Hyper-V", hypervUtility.Guests.Count);
foreach (var guest in hypervUtility.Guests)
Duplicati/Library/Modules/Builtin/MSSQLOptions.cs
+23 -22
View File
@@ -1,31 +1,31 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.Versioning;
using System.Text.RegularExpressions;
using Duplicati.Library.Common;
using Duplicati.Library.Snapshots;
using Duplicati.Library.Snapshots.Windows;
namespace Duplicati.Library.Modules.Builtin
{
@@ -144,7 +144,8 @@ namespace Duplicati.Library.Modules.Builtin
}
Logging.Log.WriteInformationMessage(LOGTAG, "StartingMsSqlQuery", "Starting to gather Microsoft SQL Server information", Logging.LogMessageType.Information);
mssqlUtility.QueryDBsInfo();
var provider = Utility.Utility.ParseEnumOption(changedOptions.AsReadOnly(), "snapshot-provider", SnapshotProvider.AlphaVSS);
mssqlUtility.QueryDBsInfo(provider);
Logging.Log.WriteInformationMessage(LOGTAG, "MsSqlDatabaseCount", "Found {0} databases on Microsoft SQL Server", mssqlUtility.DBs.Count);
foreach(var db in mssqlUtility.DBs)
Duplicati/Library/Modules/Builtin/ReportHelper.cs
+1 -1
View File
@@ -411,7 +411,7 @@ namespace Duplicati.Library.Modules.Builtin
case DESTINATION_TYPE:
// Only return the url scheme, as the rest could contain sensitive information
var ix = m_remoteurl?.IndexOf("://", StringComparison.OrdinalIgnoreCase) ?? -1;
return ix >= 0 && ix < 15 ? m_remoteurl.Substring(0, ix) : "file";
return Utility.Utility.GuessScheme(m_remoteurl) ?? "file";
default:
return null;
}
Duplicati/Library/Modules/Builtin/SendJabberMessage.cs
+1 -2
View File
@@ -195,8 +195,7 @@ namespace Duplicati.Library.Modules.Builtin
protected override void SendMessage(string subject, string body)
{
var uri = new Library.Utility.Uri(m_username.Contains("://") ? m_username : "http://" + m_username);
var uri = new Utility.Uri(m_username.Contains("://") ? m_username : "http://" + m_username);
var resource = uri.Path ?? "";
if (resource.StartsWith("/", StringComparison.Ordinal))
resource = resource.Substring(1);
Duplicati/Library/RestAPI/Scheduler.cs
+1 -1
View File
@@ -152,7 +152,7 @@ namespace Duplicati.Server
/// </summary>
public List<Runner.IRunnerData> WorkerQueue
{
get { return (from t in m_worker?.CurrentTasks where t != null select t)?.ToList() ?? []; }
get { return m_worker?.CurrentTasks?.Where(t => t != null)?.ToList() ?? []; }
}
/// <summary>
Duplicati/Library/Snapshots/Duplicati.Library.Snapshots.csproj
+3 -1
View File
@@ -3,10 +3,12 @@
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<Copyright>Copyright © 2025 Team Duplicati, MIT license</Copyright>
<RootNamespace>Duplicati.Library.Snapshots</RootNamespace>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
</ItemGroup>
<ItemGroup>
@@ -14,7 +16,7 @@
<ProjectReference Include="..\Common\Duplicati.Library.Common.csproj" />
<ProjectReference Include="..\Interface\Duplicati.Library.Interface.csproj" />
<ProjectReference Include="..\Localization\Duplicati.Library.Localization.csproj" />
<ProjectReference Include="..\Logging\Duplicati.Library.Logging.csproj" />
<ProjectReference Include="..\Logging\Duplicati.Library.Logging.csproj" />
<ProjectReference Include="..\Utility\Duplicati.Library.Utility.csproj" />
</ItemGroup>
Duplicati/Library/Snapshots/ISnapshotService.cs
+59 -43
View File
@@ -1,27 +1,32 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using Duplicati.Library.Interface;
namespace Duplicati.Library.Snapshots
{
@@ -31,12 +36,28 @@ namespace Duplicati.Library.Snapshots
public interface ISnapshotService : IDisposable
{
/// <summary>
/// Enumerates all files and folders in the snapshot, restricted to sources
/// Gets the source entries
/// </summary>
/// <param name="sources">Sources to enumerate</param>
/// <param name="callback">The callback to invoke with each found path</param>
/// <param name="errorCallback">The callback used to report errors</param>
IEnumerable<string> EnumerateFilesAndFolders(IEnumerable<string> sources, Utility.Utility.EnumerationFilterDelegate callback, Utility.Utility.ReportAccessError errorCallback);
IEnumerable<string> SourceEntries { get; }
/// <summary>
/// Enumerates the root source files and folders
/// </summary>
/// <returns>The source files and folders</returns>
IEnumerable<ISourceProviderEntry> EnumerateFilesystemEntries();
/// <summary>
/// Enumerates the files and folders in a given folder
/// </summary>
/// <param name="parent">The parent folder to enumerate</param>
/// <returns>The files and folders in the parent folder</returns>
IEnumerable<ISourceProviderEntry> EnumerateFilesystemEntries(ISourceProviderEntry parent);
/// <summary>
/// Gets a filesystem entry for a given path
/// </summary>
/// <param name="path">The path to get</param>
/// <param name="isFolder">True if the path is a folder</param>
/// <returns>The filesystem entry</returns>
ISourceProviderEntry? GetFilesystemEntry(string path, bool isFolder);
/// <summary>
/// Gets the last write time of a given file in UTC
@@ -59,6 +80,14 @@ namespace Duplicati.Library.Snapshots
/// <returns>An open filestream that can be read and seeked</returns>
Stream OpenRead(string localPath);
/// <summary>
/// Opens a file for reading
/// </summary>
/// <param name="localPath">The full path to the file in non-snapshot format</param>
/// <param name="cancellationToken">A token that can be used to cancel the operation</param>
/// <returns>An open filestream that can be read and seeked</returns>
Task<Stream> OpenReadAsync(string localPath, CancellationToken cancellationToken);
/// <summary>
/// Returns the size of a file
/// </summary>
@@ -79,29 +108,28 @@ namespace Duplicati.Library.Snapshots
/// <returns>The file attributes</returns>
/// <param name="localPath">The file or folder to examine</param>
FileAttributes GetAttributes(string localPath);
/// <summary>
/// Gets the metadata for the given file or folder
/// </summary>
/// <returns>The metadata for the given file or folder</returns>
/// <param name="localPath">The file or folder to examine</param>
/// <param name="isSymlink">A flag indicating if the target is a symlink</param>
/// <param name="followSymlink">A flag indicating if a symlink should be followed</param>
Dictionary<string, string> GetMetadata(string localPath, bool isSymlink, bool followSymlink);
Dictionary<string, string> GetMetadata(string localPath, bool isSymlink);
/// <summary>
/// Gets a value indicating if the path points to a block device
/// </summary>
/// <returns><c>true</c> if this instance is a block device; otherwise, <c>false</c>.</returns>
/// <param name="localPath">The file or folder to examine</param>
bool IsBlockDevice(string localPath);
/// <summary>
/// Gets a unique hardlink target ID
/// </summary>
/// <returns>The hardlink ID</returns>
/// <param name="localPath">The file or folder to examine</param>
string HardlinkTargetID(string localPath);
string? HardlinkTargetID(string localPath);
/// <summary>
/// Converts a snapshot path to a local path
@@ -130,17 +158,5 @@ namespace Duplicati.Library.Snapshots
/// <param name="localFolderPath">The local path</param>
/// <returns>True if folder exists, false otherwise</returns>
bool DirectoryExists(string localFolderPath);
/// <summary>
/// Tests if a file or folder exists in the snapshot
/// </summary>
/// <param name="localFileOrFolderPath">The local path</param>
/// <returns>True if file or folder exists, false otherwise</returns>
bool PathExists(string localFileOrFolderPath);
/// <summary>
/// Is true if the actual service is a real snapshot service
/// </summary>
bool IsSnapshot { get; }
}
}
Duplicati/Library/Snapshots/LinuxSnapshot.cs
+28 -8
View File
@@ -23,9 +23,11 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Runtime.Versioning;
using System.Text;
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
namespace Duplicati.Library.Snapshots
{
@@ -59,7 +61,9 @@ namespace Duplicati.Library.Snapshots
/// Constructs a new snapshot module using LVM
/// </summary>
/// <param name="sources">The list of folders to create snapshots for</param>
public LinuxSnapshot(IEnumerable<string> sources)
/// <param name="followSymlinks">A flag indicating if symlinks should be followed</param>
public LinuxSnapshot(IEnumerable<string> sources, bool followSymlinks)
: base(followSymlinks)
{
try
{
@@ -79,7 +83,7 @@ namespace Duplicati.Library.Snapshots
m_entries.Add(new KeyValuePair<string, SnapShot>(path, snap));
}
m_snapShots = new List<SnapShot>(snaps.Values);
m_snapShots = [.. snaps.Values];
// We have all the snapshots that we need, lets activate them
foreach (var snap in m_snapShots)
@@ -103,6 +107,26 @@ namespace Duplicati.Library.Snapshots
}
}
/// <summary>
/// Gets the source folders
/// </summary>
public override IEnumerable<string> SourceEntries => m_entries.Select(x => x.Key);
/// <summary>
/// Enumerates the root source files and folders
/// </summary>
/// <returns>The source files and folders</returns>
public override IEnumerable<ISourceProviderEntry> EnumerateFilesystemEntries()
{
foreach (var sourceEntry in SourceEntries)
{
if (DirectoryExists(sourceEntry) || sourceEntry.EndsWith(System.IO.Path.DirectorySeparatorChar))
yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(sourceEntry), true, true);
else
yield return new SnapshotSourceFileEntry(this, sourceEntry, false, true);
}
}
/// <inheritdoc />
protected override void Dispose(bool disposing)
{
@@ -475,10 +499,9 @@ namespace Duplicati.Library.Snapshots
/// <returns>The metadata for the given file or folder</returns>
/// <param name="localPath">The file or folder to examine</param>
/// <param name="isSymlink">A flag indicating if the target is a symlink</param>
/// <param name="followSymlink">A flag indicating if a symlink should be followed</param>
public override Dictionary<string, string> GetMetadata(string localPath, bool isSymlink, bool followSymlink)
public override Dictionary<string, string> GetMetadata(string localPath, bool isSymlink)
{
return SystemIO.IO_OS.GetMetadata(ConvertToSnapshotPath(localPath), isSymlink, followSymlink);
return SystemIO.IO_OS.GetMetadata(ConvertToSnapshotPath(localPath), isSymlink, FollowSymlinks);
}
/// <summary>
@@ -537,9 +560,6 @@ namespace Duplicati.Library.Snapshots
return FindSnapshotByLocalPath(localPath).ConvertToSnapshotPath(localPath);
}
/// <inheritdoc />
public override bool IsSnapshot => true;
#endregion
}
}
Duplicati/Library/Snapshots/NoSnapshotLinux.cs
+32 -4
View File
@@ -25,6 +25,7 @@ using System.IO;
using System.Runtime.InteropServices;
using System.Runtime.Versioning;
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Microsoft.Win32.SafeHandles;
namespace Duplicati.Library.Snapshots
@@ -65,15 +66,43 @@ namespace Duplicati.Library.Snapshots
/// </summary>
private readonly bool m_ignoreAdvisoryLocks;
/// <summary>
/// The list of folders to create snapshots of
/// </summary>
private readonly IEnumerable<string> m_sources;
/// <summary>
/// Initializes a new instance of the <see cref="NoSnapshotLinux"/> class.
/// </summary>
/// <param name="sources">The list of sources to create snapshots of</param>
/// <param name="ignoreAdvisoryLocks">A flag indicating if advisory locks should be ignored</param>
public NoSnapshotLinux(bool ignoreAdvisoryLocks)
/// <param name="followSymlinks">A flag indicating if symlinks should be followed</param>
public NoSnapshotLinux(IEnumerable<string> sources, bool ignoreAdvisoryLocks, bool followSymlinks)
: base(followSymlinks)
{
m_sources = sources;
m_ignoreAdvisoryLocks = ignoreAdvisoryLocks;
}
/// <summary>
/// Gets the source folders
/// </summary>
public override IEnumerable<string> SourceEntries => m_sources;
/// <summary>
/// Enumerates the root source files and folders
/// </summary>
/// <returns>The source files and folders</returns>
public override IEnumerable<ISourceProviderEntry> EnumerateFilesystemEntries()
{
foreach (var folder in m_sources)
{
if (DirectoryExists(folder) || folder.EndsWith(Path.DirectorySeparatorChar))
yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(folder), true, true);
else
yield return new SnapshotSourceFileEntry(this, folder, false, true);
}
}
/// <summary>
/// Returns the symlink target if the entry is a symlink, and null otherwise
@@ -91,10 +120,9 @@ namespace Duplicati.Library.Snapshots
/// <returns>The metadata for the given file or folder</returns>
/// <param name="localPath">The file or folder to examine</param>
/// <param name="isSymlink">A flag indicating if the target is a symlink</param>
/// <param name="followSymlink">A flag indicating if a symlink should be followed</param>
public override Dictionary<string, string> GetMetadata(string localPath, bool isSymlink, bool followSymlink)
public override Dictionary<string, string> GetMetadata(string localPath, bool isSymlink)
{
return SystemIO.IO_OS.GetMetadata(localPath, isSymlink, followSymlink);
return SystemIO.IO_OS.GetMetadata(localPath, isSymlink, FollowSymlinks);
}
/// <summary>
Duplicati/Library/Snapshots/NoSnapshotWindows.cs
+43 -38
View File
@@ -24,6 +24,7 @@ using System.Collections.Generic;
using System.Linq;
using System.Runtime.Versioning;
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
namespace Duplicati.Library.Snapshots
{
@@ -34,9 +35,20 @@ namespace Duplicati.Library.Snapshots
public sealed class NoSnapshotWindows : SnapshotBase
{
/// <summary>
/// The system IO implementation for Windows
/// The list of folders to create snapshots of
/// </summary>
private static readonly ISystemIO IO_WIN = SystemIO.IO_OS;
private readonly IEnumerable<string> m_sources;
/// <summary>
/// Initializes a new instance of the <see cref="NoSnapshotWindows"/> class.
/// </summary>
/// <param name="sources">The list of entries to create snapshots of</param>
/// <param name="followSymlinks">A flag indicating if symlinks should be followed</param>
public NoSnapshotWindows(IEnumerable<string> sources, bool followSymlinks)
: base(followSymlinks)
{
m_sources = sources;
}
/// <summary>
/// Returns the symlink target if the entry is a symlink, and null otherwise
@@ -44,9 +56,7 @@ namespace Duplicati.Library.Snapshots
/// <param name="localPath">The file or folder to examine</param>
/// <returns>The symlink target</returns>
public override string GetSymlinkTarget(string localPath)
{
return IO_WIN.GetSymlinkTarget(localPath);
}
=> SystemIO.IO_OS.GetSymlinkTarget(localPath);
/// <summary>
/// Gets the attributes for the given file or folder
@@ -54,9 +64,7 @@ namespace Duplicati.Library.Snapshots
/// <returns>The file attributes</returns>
/// <param name="localPath">The file or folder to examine</param>
public override System.IO.FileAttributes GetAttributes(string localPath)
{
return IO_WIN.GetFileAttributes(localPath);
}
=> SystemIO.IO_OS.GetFileAttributes(localPath);
/// <summary>
/// Returns the size of a file
@@ -64,31 +72,37 @@ namespace Duplicati.Library.Snapshots
/// <param name="localPath">The full path to the file in non-snapshot format</param>
/// <returns>The length of the file</returns>
public override long GetFileSize(string localPath)
{
return IO_WIN.FileLength(localPath);
}
=> SystemIO.IO_OS.FileLength(localPath);
/// <summary>
/// Enumerates all files and folders in the snapshot, restricted to sources
/// Gets the source folders
/// </summary>
/// <param name="sources">Sources to enumerate</param>
/// <param name="callback">The callback to invoke with each found path</param>
/// <param name="errorCallback">The callback used to report errors</param>
public override IEnumerable<string> EnumerateFilesAndFolders(IEnumerable<string> sources, Utility.Utility.EnumerationFilterDelegate callback, Utility.Utility.ReportAccessError errorCallback)
public override IEnumerable<string> SourceEntries
=> m_sources;
/// <summary>
/// Enumerates the root source files and folders
/// </summary>
/// <returns>The source files and folders</returns>
public override IEnumerable<ISourceProviderEntry> EnumerateFilesystemEntries()
{
// For Windows, ensure we don't store paths with extended device path prefixes (i.e., @"\\?\" or @"\\?\UNC\")
return base.EnumerateFilesAndFolders(sources.Select(SystemIOWindows.RemoveExtendedDevicePathPrefix), callback, errorCallback);
foreach (var folder in m_sources.Select(SystemIOWindows.RemoveExtendedDevicePathPrefix))
{
if (DirectoryExists(folder) || folder.EndsWith(System.IO.Path.DirectorySeparatorChar))
yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(folder), true, true);
else
yield return new SnapshotSourceFileEntry(this, folder, false, true);
}
}
/// <summary>
/// Gets the last write time of a given file in UTC
/// </summary>
/// <param name="localPath">The full path to the file in non-snapshot format</param>
/// <returns>The last write time of the file</returns>
public override DateTime GetLastWriteTimeUtc(string localPath)
{
return IO_WIN.FileGetLastWriteTimeUtc(localPath);
}
=> SystemIO.IO_OS.FileGetLastWriteTimeUtc(localPath);
/// <summary>
/// Gets the last write time of a given file in UTC
@@ -96,9 +110,7 @@ namespace Duplicati.Library.Snapshots
/// <param name="localPath">The full path to the file in non-snapshot format</param>
/// <returns>The last write time of the file</returns>
public override DateTime GetCreationTimeUtc(string localPath)
{
return IO_WIN.FileGetCreationTimeUtc(localPath);
}
=> SystemIO.IO_OS.FileGetCreationTimeUtc(localPath);
/// <summary>
/// Opens a file for reading
@@ -106,9 +118,7 @@ namespace Duplicati.Library.Snapshots
/// <param name="localPath">The full path to the file in non-snapshot format</param>
/// <returns>An open filestream that can be read</returns>
public override System.IO.Stream OpenRead(string localPath)
{
return IO_WIN.FileOpenRead(localPath);
}
=> SystemIO.IO_OS.FileOpenRead(localPath);
/// <summary>
/// Lists all files in the given folder
@@ -117,7 +127,7 @@ namespace Duplicati.Library.Snapshots
/// <param name='localFolderPath'>The folder to examinate</param>
protected override string[] ListFiles(string localFolderPath)
{
string[] tmp = IO_WIN.GetFiles(localFolderPath);
string[] tmp = SystemIO.IO_OS.GetFiles(localFolderPath);
string[] res = new string[tmp.Length];
for (int i = 0; i < tmp.Length; i++)
res[i] = SystemIOWindows.RemoveExtendedDevicePathPrefix(tmp[i]);
@@ -133,7 +143,7 @@ namespace Duplicati.Library.Snapshots
/// <param name='localFolderPath'>The folder to examinate</param>
protected override string[] ListFolders(string localFolderPath)
{
string[] tmp = IO_WIN.GetDirectories(SystemIOWindows.AddExtendedDevicePathPrefix(localFolderPath));
string[] tmp = SystemIO.IO_OS.GetDirectories(SystemIOWindows.AddExtendedDevicePathPrefix(localFolderPath));
string[] res = new string[tmp.Length];
for (int i = 0; i < tmp.Length; i++)
res[i] = SystemIOWindows.RemoveExtendedDevicePathPrefix(tmp[i]);
@@ -147,11 +157,8 @@ namespace Duplicati.Library.Snapshots
/// <returns>The metadata for the given file or folder</returns>
/// <param name="localPath">The file or folder to examine</param>
/// <param name="isSymlink">A flag indicating if the target is a symlink</param>
/// <param name="followSymlink">A flag indicating if a symlink should be followed</param>
public override Dictionary<string, string> GetMetadata(string localPath, bool isSymlink, bool followSymlink)
{
return IO_WIN.GetMetadata(localPath, isSymlink, followSymlink);
}
public override Dictionary<string, string> GetMetadata(string localPath, bool isSymlink)
=> SystemIO.IO_OS.GetMetadata(localPath, isSymlink, FollowSymlinks);
/// <inheritdoc />
public override string ConvertToLocalPath(string snapshotPath)
@@ -160,11 +167,9 @@ namespace Duplicati.Library.Snapshots
}
/// <inheritdoc />
public override string ConvertToSnapshotPath(string localPath)
{
public override string ConvertToSnapshotPath(string localPath) =>
// For Windows, ensure we don't store paths with extended device path prefixes (i.e., @"\\?\" or @"\\?\UNC\")
return SystemIOWindows.RemoveExtendedDevicePathPrefix(localPath);
}
SystemIOWindows.RemoveExtendedDevicePathPrefix(localPath);
}
}
Duplicati/Library/Snapshots/Program.cs
+1 -3
View File
@@ -21,9 +21,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Duplicati.Library.AutoUpdater;
using Duplicati.Library.Common;
using Duplicati.Library.Utility;
namespace Duplicati.Library.Snapshots
@@ -110,7 +108,7 @@ Where <test-folder> is the folder where files will be locked/created etc");
Console.WriteLine("Creating snapshot for folder: {0}", args[0]);
Console.WriteLine("If this fails, try to run as " + ((OperatingSystem.IsMacOS() || OperatingSystem.IsLinux()) ? "root" : "Administrator"));
using (ISnapshotService snapshot = SnapshotUtility.CreateSnapshot(new[] { args[0] }, options))
using (ISnapshotService snapshot = SnapshotUtility.CreateSnapshot(new[] { args[0] }, options, false))
{
Console.WriteLine("Attempting to read locked file via snapshot");
try
Duplicati/Library/Snapshots/SnapshotBase.cs
+70 -25
View File
@@ -20,9 +20,11 @@
// DEALINGS IN THE SOFTWARE.
using System;
using System.Linq;
using System.Collections.Generic;
using System.IO;
using Duplicati.Library.Interface;
using System.Threading.Tasks;
using System.Threading;
using Duplicati.Library.Common.IO;
namespace Duplicati.Library.Snapshots
@@ -40,19 +42,63 @@ namespace Duplicati.Library.Snapshots
Dispose(false);
}
#region ISnapshotService
/// <summary>
/// Enumerates all files and folders in the shadow copy
/// Initializes a new instance of the <see cref="SnapshotBase"/> class.
/// </summary>
/// <param name="sources">Source paths to enumerate</param>
/// <param name="callback">The callback to invoke with each found path</param>
/// <param name="errorCallback">The callback used to report errors</param>
public virtual IEnumerable<string> EnumerateFilesAndFolders(IEnumerable<string> sources, Utility.Utility.EnumerationFilterDelegate callback, Utility.Utility.ReportAccessError errorCallback)
/// <param name="followSymlinks">A flag indicating if symlinks should be followed</param>
protected SnapshotBase(bool followSymlinks)
{
return sources.SelectMany(s => Utility.Utility.EnumerateFileSystemEntries(s, callback, ListFolders, ListFiles, GetAttributes, errorCallback));
FollowSymlinks = followSymlinks;
}
#region ISnapshotService
protected bool FollowSymlinks { get; }
/// <summary>
/// Gets the source entries
/// </summary>
public abstract IEnumerable<string> SourceEntries { get; }
/// <summary>
/// Enuemrates the root source files and folders
/// </summary>
/// <returns>The source files and folders</returns>
public abstract IEnumerable<ISourceProviderEntry> EnumerateFilesystemEntries();
/// <summary>
/// Gets a filesystem entry for a given path
/// </summary>
/// <param name="path">The path to get the entry for</param>
/// <param name="isFolder">A flag indicating if the path is a folder</param>
/// <returns>The filesystem entry</returns>
public virtual ISourceProviderEntry GetFilesystemEntry(string path, bool isFolder)
{
if (isFolder && !DirectoryExists(path))
return null;
else if (!isFolder && !FileExists(path))
return null;
return new SnapshotSourceFileEntry(this, isFolder ? Util.AppendDirSeparator(path) : path, isFolder, false);
}
/// <summary>
/// Enumerates files and folders in the given folder
/// </summary>
/// <param name="source">Source to enumerate</param>
/// <returns>The files and folders in the given folder</returns>
public virtual IEnumerable<ISourceProviderEntry> EnumerateFilesystemEntries(ISourceProviderEntry source)
{
if (source.IsFolder)
{
foreach (var f in ListFiles(source.Path))
yield return new SnapshotSourceFileEntry(this, f, false, false);
foreach (var d in ListFolders(source.Path))
yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(d), true, false);
}
}
/// <summary>
/// Gets the last write time of a given file in UTC
/// </summary>
@@ -72,7 +118,7 @@ namespace Duplicati.Library.Snapshots
{
return File.GetCreationTimeUtc(localPath);
}
/// <summary>
/// Opens a file for reading
/// </summary>
@@ -83,6 +129,15 @@ namespace Duplicati.Library.Snapshots
return File.Open(localPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
}
/// <summary>
/// Opens a file for reading
/// </summary>
/// <param name="localPath">The full path to the file in non-snapshot format</param>
/// <param name="cancellationToken">A token that can be used to cancel the operation</param>
/// <returns>An open filestream that can be read</returns>
public virtual Task<Stream> OpenReadAsync(string localPath, CancellationToken cancellationToken)
=> Task.FromResult(OpenRead(localPath));
/// <summary>
/// Returns the size of a file
/// </summary>
@@ -116,9 +171,8 @@ namespace Duplicati.Library.Snapshots
/// <returns>The metadata for the given file or folder</returns>
/// <param name="localPath">The file or folder to examine</param>
/// <param name="isSymlink">A flag indicating if the target is a symlink</param>
/// <param name="followSymlink">A flag indicating if a symlink should be followed</param>
public abstract Dictionary<string, string> GetMetadata(string localPath, bool isSymlink, bool followSymlink);
public abstract Dictionary<string, string> GetMetadata(string localPath, bool isSymlink);
/// <summary>
/// Gets a value indicating if the path points to a block device
/// </summary>
@@ -128,7 +182,7 @@ namespace Duplicati.Library.Snapshots
{
return false;
}
/// <summary>
/// Gets a unique hardlink target ID
/// </summary>
@@ -157,15 +211,6 @@ namespace Duplicati.Library.Snapshots
return Directory.Exists(ConvertToSnapshotPath(localFolderPath));
}
/// <inheritdoc />
public virtual bool PathExists(string localFileOrFolderPath)
{
return FileExists(localFileOrFolderPath) || DirectoryExists(localFileOrFolderPath);
}
/// <inheritdoc />
public virtual bool IsSnapshot => false;
#endregion
/// <summary>
@@ -198,7 +243,7 @@ namespace Duplicati.Library.Snapshots
// Now suppress finalization for this object, since we've already handled our resource cleanup tasks
GC.SuppressFinalize(this);
}
}
#endregion
Duplicati/Library/Snapshots/SnapshotProvider.cs
+30
View File
@@ -0,0 +1,30 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
namespace Duplicati.Library.Snapshots
{
public enum SnapshotProvider
{
AlphaVSS,
Wmic,
LVM
}
}
Duplicati/Library/Snapshots/SnapshotSourceFileEntry.cs
+143
View File
@@ -0,0 +1,143 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Duplicati.Library.Interface;
namespace Duplicati.Library.Snapshots;
/// <summary>
/// A snapshot-based source file entry
/// </summary>
/// <param name="service">The snapshot service</param>
/// <param name="path">The path of the file</param>
/// <param name="isFolder">True if the entry is a folder, false otherwise</param>
/// <param name="isRoot">True if the entry is the root entry, false otherwise</param>
public class SnapshotSourceFileEntry(ISnapshotService service, string path, bool isFolder, bool isRoot) : ISourceProviderEntry
{
/// <summary>
/// The symlink target
/// </summary>
private string? symlinkTarget;
/// <summary>
/// The file attributes
/// </summary>
private FileAttributes? attributes;
/// <summary>
/// Gets the symlink target or null if the entry is not a symlink
/// </summary>
public string? SymlinkTarget
{
get
{
if (symlinkTarget != null)
return symlinkTarget;
if (!service.IsSymlink(path, Attributes))
return null;
return symlinkTarget = service.GetSymlinkTarget(path);
}
}
/// <summary>
/// True if the entry is a symlink, false otherwise
/// </summary>
public bool IsSymlink => SymlinkTarget != null;
/// <summary>
/// Gets the entry attributes
/// </summary>
public FileAttributes Attributes
{
get
{
if (attributes != null)
return attributes.Value;
attributes = service.GetAttributes(path);
return attributes.Value;
}
}
/// <summary>
/// Gets the minor metadata for the file
/// </summary>
public Dictionary<string, string> MinorMetadata => service.GetMetadata(path, IsSymlink);
/// <summary>
/// Gets a value indicating if the entry is a block device
/// </summary>
public bool IsBlockDevice => service.IsBlockDevice(path);
/// <summary>
/// Gets a value indicating if the entry is a character device
/// </summary>
public bool IsCharacterDevice => false;
/// <summary>
/// Gets a value indicating if the entry is an alternate stream
/// </summary>
public bool IsAlternateStream => false;
/// <summary>
/// Gets the hardlink target ID, or null if the entry is not a hardlink
/// </summary>
public string? HardlinkTargetId => service.HardlinkTargetID(path);
/// <inheritdoc/>
public bool IsFolder => isFolder;
/// <inheritdoc/>
public bool IsMetaEntry => false;
/// <inheritdoc/>
public bool IsRootEntry => isRoot;
/// <inheritdoc/>
public DateTime CreatedUtc => service.GetCreationTimeUtc(path);
/// <inheritdoc/>
public DateTime LastModificationUtc => service.GetLastWriteTimeUtc(path);
/// <inheritdoc/>
public string Path => path;
/// <inheritdoc/>
public long Size => IsFolder ? -1 : service.GetFileSize(path);
/// <inheritdoc/>
public Task<Stream> OpenRead(CancellationToken cancellationToken) => service.OpenReadAsync(path, cancellationToken);
/// <inheritdoc/>
public Task<Stream?> OpenMetadataRead(CancellationToken cancellationToken) => Task.FromResult<Stream?>(new MemoryStream(System.Text.Json.JsonSerializer.SerializeToUtf8Bytes(MinorMetadata)));
/// <inheritdoc/>
public IAsyncEnumerable<ISourceProviderEntry> Enumerate(CancellationToken cancellationToken) => service.EnumerateFilesystemEntries(this).ToAsyncEnumerable();
/// <inheritdoc/>
public Task<bool> FileExists(string filename, CancellationToken cancellationToken) => Task.FromResult(service.FileExists(System.IO.Path.Combine(path, filename)));
}
Duplicati/Library/Snapshots/SnapshotUtility.cs
+50 -42
View File
@@ -1,30 +1,29 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Collections.Generic;
using System.IO;
using System.Runtime.Versioning;
using Duplicati.Library.Common;
using Duplicati.Library.Common.IO;
namespace Duplicati.Library.Snapshots
@@ -37,24 +36,42 @@ namespace Duplicati.Library.Snapshots
/// <summary>
/// Loads a snapshot implementation for the current OS
/// </summary>
/// <param name="folders">The list of folders to create snapshots of</param>
/// <param name="paths">The list of paths to create snapshots of</param>
/// <param name="options">A set of commandline options</param>
/// <param name="followSymlinks">Whether to follow symlinks</param>
/// <returns>The ISnapshotService implementation</returns>
public static ISnapshotService CreateSnapshot(IEnumerable<string> folders, Dictionary<string, string> options)
public static ISnapshotService CreateSnapshot(IEnumerable<string> paths, Dictionary<string, string> options, bool followSymlinks)
{
if (OperatingSystem.IsMacOS() || OperatingSystem.IsLinux())
if (OperatingSystem.IsLinux())
{
return CreateLinuxSnapshot(folders);
return CreateLinuxSnapshot(paths, followSymlinks);
}
else if (OperatingSystem.IsWindows())
{
return CreateWindowsSnapshot(folders, options);
return CreateWindowsSnapshot(paths, options, followSymlinks);
}
else
{
throw new NotSupportedException("Unsupported Operating System");
}
}
/// <summary>
/// Creates a snapshot implementation that does not use snapshots (i.e., regular file access)
/// </summary>
/// <param name="paths">The list of paths to create snapshots of</param>
/// <param name="ignoreAdvisoryLocking">Flag to ignore advisory locking</param>
/// <param name="followSymlinks">Whether to follow symlinks</param>
/// <returns>The ISnapshotService implementation</returns>
public static ISnapshotService CreateNoSnapshot(IEnumerable<string> paths, bool ignoreAdvisoryLocking, bool followSymlinks)
{
if (OperatingSystem.IsMacOS() || OperatingSystem.IsLinux())
return new NoSnapshotLinux(paths, ignoreAdvisoryLocking, followSymlinks);
else if (OperatingSystem.IsWindows())
return new NoSnapshotWindows(paths, followSymlinks);
else
throw new NotSupportedException("Unsupported Operating System");
}
// The two loader methods below guard against the type system attempting to load types
@@ -65,13 +82,14 @@ namespace Duplicati.Library.Snapshots
/// Loads a snapshot implementation for Linux
/// </summary>
/// <param name="folders">The list of folders to create snapshots of</param>
/// <param name="followSymlinks">Whether to follow symlinks</param>
/// <returns>The ISnapshotService implementation</returns>
[System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)]
[SupportedOSPlatform("linux")]
[SupportedOSPlatform("macOS")]
private static ISnapshotService CreateLinuxSnapshot(IEnumerable<string> folders)
private static ISnapshotService CreateLinuxSnapshot(IEnumerable<string> folders, bool followSymlinks)
{
return new LinuxSnapshot(folders);
return new LinuxSnapshot(folders, followSymlinks);
}
/// <summary>
@@ -79,23 +97,13 @@ namespace Duplicati.Library.Snapshots
/// </summary>
/// <param name="folders">The list of folders to create snapshots of</param>
/// <param name="options">A set of commandline options</param>
/// <param name="followSymlinks">Whether to follow symlinks</param>
/// <returns>The ISnapshotService implementation</returns>
[System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)]
[SupportedOSPlatform("windows")]
private static ISnapshotService CreateWindowsSnapshot(IEnumerable<string> folders, Dictionary<string, string> options)
private static ISnapshotService CreateWindowsSnapshot(IEnumerable<string> folders, Dictionary<string, string> options, bool followSymlinks)
{
return new WindowsSnapshot(folders, options);
}
/// <summary>
/// Extension method for ISnapshotService which determines whether the given path is a symlink.
/// </summary>
/// <param name="snapshot">ISnapshotService implementation</param>
/// <param name="path">File or folder path</param>
/// <returns>Whether the path is a symlink</returns>
public static bool IsSymlink(this ISnapshotService snapshot, string path)
{
return snapshot.IsSymlink(path, snapshot.GetAttributes(path));
return new WindowsSnapshot(folders, options, followSymlinks);
}
/// <summary>
Duplicati/Library/Snapshots/USNJournal.cs → Duplicati/Library/Snapshots/USN/USNJournal.cs
+1 -1
View File
@@ -31,7 +31,7 @@ using System.Runtime.Versioning;
using Duplicati.Library.Common.IO;
using Microsoft.Win32.SafeHandles;
namespace Duplicati.Library.Snapshots
namespace Duplicati.Library.Snapshots.USN
{
/// <summary>
/// Class that encapsulates USN journal access to a single volume
Duplicati/Library/Snapshots/UsnJournalService.cs → Duplicati/Library/Snapshots/USN/UsnJournalService.cs
+167 -104
View File
@@ -19,6 +19,7 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
using System;
using System.Collections.Generic;
@@ -29,8 +30,10 @@ using System.Threading;
using Duplicati.Library.Interface;
using Duplicati.Library.Common.IO;
using Duplicati.Library.Utility;
using System.Threading.Tasks;
using System.Runtime.CompilerServices;
namespace Duplicati.Library.Snapshots
namespace Duplicati.Library.Snapshots.USN
{
[SupportedOSPlatform("windows")]
public class UsnJournalService
@@ -40,30 +43,39 @@ namespace Duplicati.Library.Snapshots
/// </summary>
private static readonly string FILTER_LOGTAG = Logging.Log.LogTagFromType(typeof(UsnJournalService));
/// <summary>
/// The snapshot service
/// </summary>
private readonly ISnapshotService m_snapshot;
private readonly IEnumerable<string> m_sources;
/// <summary>
/// The volume data dictionary
/// </summary>
private readonly Dictionary<string, VolumeData> m_volumeDataDict;
/// <summary>
/// The cancellation token
/// </summary>
private readonly CancellationToken m_token;
/// <summary>
/// Constructor.
/// </summary>
/// <param name="sources">Sources to filter</param>
/// <param name="snapshot"></param>
/// <param name="emitFilter">Emit filter</param>
/// <param name="fileAttributeFilter"></param>
/// <param name="skipFilesLargerThan"></param>
/// <param name="prevJournalData">Journal-data of previous fileset</param>
/// <param name="token"></param>
public UsnJournalService(IEnumerable<string> sources, ISnapshotService snapshot, IFilter emitFilter, FileAttributes fileAttributeFilter,
public UsnJournalService(ISnapshotService snapshot, IFilter emitFilter, FileAttributes fileAttributeFilter,
long skipFilesLargerThan, IEnumerable<USNJournalDataEntry> prevJournalData, CancellationToken token)
{
m_sources = sources;
m_snapshot = snapshot;
m_volumeDataDict = Initialize(emitFilter, fileAttributeFilter, skipFilesLargerThan, prevJournalData);
m_token = token;
}
/// <summary>
/// The volume data lists for the mapped root drives
/// </summary>
public IEnumerable<VolumeData> VolumeDataList => m_volumeDataDict.Select(e => e.Value);
/// <summary>
@@ -84,8 +96,8 @@ namespace Duplicati.Library.Snapshots
// get hash identifying current source filter / sources configuration
var configHash = Utility.Utility.ByteArrayAsHexString(MD5HashHelper.GetHash(new string[] {
emitFilter == null ? string.Empty : emitFilter.ToString(),
string.Join("; ", m_sources),
emitFilter?.ToString() ?? string.Empty,
string.Join("; ", m_snapshot.SourceEntries),
fileAttributeFilter.ToString(),
skipFilesLargerThan.ToString()
}));
@@ -94,7 +106,7 @@ namespace Duplicati.Library.Snapshots
var journalDataDict = prevJournalData.ToDictionary(data => data.Volume);
// iterate over volumes
foreach (var sourcesPerVolume in SortByVolume(m_sources))
foreach (var sourcesPerVolume in SortByVolume(m_snapshot.SourceEntries))
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "UsnInitialize", "Reading USN journal for volume: {0}", sourcesPerVolume.Key);
@@ -180,8 +192,7 @@ namespace Duplicati.Library.Snapshots
// Since that folder is going to be fully scanned, those files can be removed.
// Note: it would be wrong to use the result from step 2. as the folder list! The entries removed
// between 1. and 2. are *excluded* folders, and files below them are to be *excluded*, too.
volumeData.Files =
new HashSet<string>(Utility.Utility.GetFilesNotInFolders(changedFiles, volumeData.Folders));
volumeData.Files = [.. Utility.Utility.GetFilesNotInFolders(changedFiles, volumeData.Folders)];
// Record success for volume
volumeData.IsFullScan = false;
@@ -213,26 +224,69 @@ namespace Duplicati.Library.Snapshots
return result;
}
/// <summary>
/// Returns all sources that should be fully scanned
/// </summary>
/// <param name="token">Cancellation token</param>
/// <returns>Filtered sources</returns>
public async IAsyncEnumerable<ISourceProviderEntry> GetFullScanSources([EnumeratorCancellation] CancellationToken token)
{
// Make the method async enumerable
await Task.CompletedTask;
foreach (var volumeData in m_volumeDataDict.Values.Where(x => x.IsFullScan))
{
if (volumeData.Folders != null)
{
foreach (var folderPath in volumeData.Folders)
{
if (token.IsCancellationRequested)
yield break;
var folder = m_snapshot.GetFilesystemEntry(folderPath, true);
if (folder != null)
yield return folder;
else
Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "FolderLookupError", null, "No entry while locating source folder {0}", folderPath);
}
}
if (volumeData.Files != null)
{
foreach (var filePath in volumeData.Files)
{
if (token.IsCancellationRequested)
yield break;
var file = m_snapshot.GetFilesystemEntry(filePath, false);
if (file != null)
yield return file;
else
Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "FolderLookupError", null, "No entry while locating source file {0}", filePath);
}
}
}
}
/// <summary>
/// Filters sources, returning sub-set having been modified since last
/// change, as specified by <c>journalData</c>.
/// </summary>
/// <param name="filter">Filter callback to exclude filtered items</param>
/// <param name="token">Cancellation token</param>
/// <returns>Filtered sources</returns>
public IEnumerable<string> GetModifiedSources(Utility.Utility.EnumerationFilterDelegate filter)
public async IAsyncEnumerable<ISourceProviderEntry> GetModifiedSources(Func<ISourceProviderEntry, ValueTask<bool>> filter, [EnumeratorCancellation] CancellationToken token)
{
// iterate over volumes
foreach (var volumeData in m_volumeDataDict)
foreach (var volumeData in m_volumeDataDict.Values.Where(x => !x.IsFullScan))
{
// prepare cache for includes (value = true) and excludes (value = false, will be populated
// on-demand)
var cache = new Dictionary<string, bool>();
foreach (var source in m_sources)
foreach (var source in m_snapshot.SourceEntries)
{
if (m_token.IsCancellationRequested)
{
break;
}
if (token.IsCancellationRequested)
yield break;
cache[source] = true;
}
@@ -244,17 +298,15 @@ namespace Duplicati.Library.Snapshots
// FilterHandler.EnumerateFilesAndFolders() works top-down, and won't even enumerate child
// folders.
// The sources are needed to stop evaluating parent folders above the specified source folders
if (volumeData.Value.Folders != null)
await foreach (var folder in FilterExcludedFolders(volumeData.Folders, m_snapshot, filter, cache, token).ConfigureAwait(false))
{
foreach (var folder in FilterExcludedFolders(volumeData.Value.Folders, filter, cache).Where(m_snapshot.DirectoryExists))
{
if (m_token.IsCancellationRequested)
{
break;
}
if (token.IsCancellationRequested)
yield break;
yield return folder;
}
if (!m_snapshot.DirectoryExists(folder.Path))
continue;
yield return folder;
}
// The simplified file list also needs to be checked against the exclusion filter, as it
@@ -263,17 +315,15 @@ namespace Duplicati.Library.Snapshots
// Note that the simplified file list may contain entries that have been deleted! They need to
// be kept in the list (unless excluded by the filter) in order for the backup handler to record their
// deletion.
if (volumeData.Value.Files != null)
await foreach (var files in FilterExcludedFiles(volumeData.Files, m_snapshot, filter, cache, token).ConfigureAwait(false))
{
foreach (var files in FilterExcludedFiles(volumeData.Value.Files, filter, cache).Where(m_snapshot.FileExists))
{
if (m_token.IsCancellationRequested)
{
break;
}
if (token.IsCancellationRequested)
yield break;
yield return files;
}
if (!m_snapshot.FileExists(files.Path))
continue;
yield return files;
}
}
}
@@ -283,45 +333,53 @@ namespace Duplicati.Library.Snapshots
/// of its parent folders, is excluded by the <c>filter</c>.
/// </summary>
/// <param name="files">Files to filter</param>
/// <param name="snapshot">Snapshot service</param>
/// <param name="filter">Exclusion filter</param>
/// <param name="cache">Cache of included and excluded files / folders</param>
/// <param name="errorCallback"></param>
/// <param name="token">Cancellation token</param>
/// <returns>Filtered files</returns>
private IEnumerable<string> FilterExcludedFiles(IEnumerable<string> files,
Utility.Utility.EnumerationFilterDelegate filter, IDictionary<string, bool> cache, Utility.Utility.ReportAccessError errorCallback = null)
private static async IAsyncEnumerable<ISourceProviderEntry> FilterExcludedFiles(
IEnumerable<string>? files,
ISnapshotService snapshot,
Func<ISourceProviderEntry, ValueTask<bool>> filter,
IDictionary<string, bool> cache,
[EnumeratorCancellation] CancellationToken token)
{
var result = new List<string>();
if (files == null)
yield break;
foreach (var file in files)
foreach (var filePath in files)
{
if (m_token.IsCancellationRequested)
{
break;
}
var attr = m_snapshot.FileExists(file) ? m_snapshot.GetAttributes(file) : FileAttributes.Normal;
ISourceProviderEntry? file;
try
{
if (!filter(file, file, attr))
file = snapshot.GetFilesystemEntry(filePath, false);
if (file == null)
continue;
if (!IsFolderOrAncestorsExcluded(Utility.Utility.GetParent(file, true), filter, cache))
if (!await filter(file))
continue;
var parentPath = Utility.Utility.GetParent(file.Path, true);
if (!string.IsNullOrWhiteSpace(parentPath))
{
result.Add(file);
var parent = snapshot.GetFilesystemEntry(parentPath, true);
if (parent == null)
continue;
if (await IsFolderOrAncestorsExcluded(parent, snapshot, filter, cache, token))
continue;
}
}
catch (System.Threading.ThreadAbortException)
{
throw;
}
catch (Exception ex)
{
errorCallback?.Invoke(file, file, ex);
filter(file, file, attr | Utility.Utility.ATTRIBUTE_ERROR);
Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "FilterExcludedFilesError", ex, "Error while filtering file: {0}", ex.Message);
continue;
}
}
return result;
if (file != null)
yield return file;
}
}
/// <summary>
@@ -329,41 +387,40 @@ namespace Duplicati.Library.Snapshots
/// of its ancestors, is excluded by the <c>filter</c>.
/// </summary>
/// <param name="folders">Folder to filter</param>
/// <param name="snapshot">Snapshot service</param>
/// <param name="filter">Exclusion filter</param>
/// <param name="cache">Cache of excluded folders (optional)</param>
/// <param name="errorCallback"></param>
/// <param name="cache">Cache of excluded folders</param>
/// <param name="token">Cancellation token</param>
/// <returns>Filtered folders</returns>
private IEnumerable<string> FilterExcludedFolders(IEnumerable<string> folders,
Utility.Utility.EnumerationFilterDelegate filter, IDictionary<string, bool> cache, Utility.Utility.ReportAccessError errorCallback = null)
private static async IAsyncEnumerable<ISourceProviderEntry> FilterExcludedFolders(
IEnumerable<string>? folders,
ISnapshotService snapshot,
Func<ISourceProviderEntry, ValueTask<bool>> filter,
IDictionary<string, bool> cache,
[EnumeratorCancellation] CancellationToken token)
{
var result = new List<string>();
if (folders == null)
yield break;
foreach (var folder in folders)
foreach (var folderPath in folders)
{
if (m_token.IsCancellationRequested)
{
break;
}
ISourceProviderEntry? folder;
try
{
if (!IsFolderOrAncestorsExcluded(folder, filter, cache))
{
result.Add(folder);
}
}
catch (System.Threading.ThreadAbortException)
{
throw;
folder = snapshot.GetFilesystemEntry(folderPath, true);
if (folder == null)
continue;
if (await IsFolderOrAncestorsExcluded(folder, snapshot, filter, cache, token))
continue;
}
catch (Exception ex)
{
errorCallback?.Invoke(folder, folder, ex);
filter(folder, folder, FileAttributes.Directory | Utility.Utility.ATTRIBUTE_ERROR);
Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "FilterExcludedFoldersError", ex, "Error while filtering folder: {0}", ex.Message);
continue;
}
}
return result;
yield return folder;
}
}
/// <summary>
@@ -373,18 +430,23 @@ namespace Duplicati.Library.Snapshots
/// <param name="filter">Filter</param>
/// <param name="cache">Cache of excluded folders (optional)</param>
/// <returns>True if excluded, false otherwise</returns>
private bool IsFolderOrAncestorsExcluded(string folder, Utility.Utility.EnumerationFilterDelegate filter, IDictionary<string, bool> cache)
private static async ValueTask<bool> IsFolderOrAncestorsExcluded(
ISourceProviderEntry inputFolder,
ISnapshotService snapshot,
Func<ISourceProviderEntry, ValueTask<bool>> filter,
IDictionary<string, bool> cache,
CancellationToken token)
{
List<string> parents = null;
List<string>? parents = null;
ISourceProviderEntry? folder = inputFolder;
while (folder != null)
{
if (m_token.IsCancellationRequested)
{
if (token.IsCancellationRequested)
break;
}
// first check cache
if (cache.TryGetValue(folder, out var include))
if (cache.TryGetValue(folder.Path, out var include))
{
if (include)
return false;
@@ -392,20 +454,20 @@ namespace Duplicati.Library.Snapshots
break; // hit!
}
parents ??= []; // create on-demand
// remember folder for cache
if (parents == null)
{
parents = new List<string>(); // create on-demand
}
parents.Add(folder);
parents.Add(folder.Path);
var attr = m_snapshot.DirectoryExists(folder) ? m_snapshot.GetAttributes(folder) : FileAttributes.Directory;
if (!filter(folder, folder, attr))
if (!await filter(folder))
break; // excluded
folder = Utility.Utility.GetParent(folder, true);
var parentPath = Utility.Utility.GetParent(folder.Path, true);
if (string.IsNullOrWhiteSpace(parentPath))
break;
folder = snapshot.GetFilesystemEntry(parentPath, true);
}
if (folder != null)
@@ -422,7 +484,6 @@ namespace Duplicati.Library.Snapshots
/// </summary>
/// <param name="sources">List of sources</param>
/// <returns>Dictionary of volumes, with list of sources as values</returns>
[SupportedOSPlatform("windows")]
private static Dictionary<string, List<string>> SortByVolume(IEnumerable<string> sources)
{
var sourcesByVolume = new Dictionary<string, List<string>>();
@@ -448,7 +509,6 @@ namespace Duplicati.Library.Snapshots
/// </summary>
/// <param name="path"></param>
/// <returns></returns>
[SupportedOSPlatform("windows")]
public bool IsPathEnumerated(string path)
{
// get NTFS volume root
@@ -461,9 +521,12 @@ namespace Duplicati.Library.Snapshots
if (volumeData.IsFullScan)
return true; // do not append from previous set, already scanned
if (volumeData.Files.Contains(path))
if (volumeData.Files != null && volumeData.Files.Contains(path))
return true; // do not append from previous set, already scanned
if (volumeData.Folders == null)
return false;
foreach (var folder in volumeData.Folders)
{
if (m_token.IsCancellationRequested)
@@ -490,23 +553,23 @@ namespace Duplicati.Library.Snapshots
/// <summary>
/// Volume
/// </summary>
public string Volume { get; set; }
public required string Volume { get; set; }
/// <summary>
/// Set of potentially modified files
/// </summary>
public HashSet<string> Files { get; internal set; }
public HashSet<string>? Files { get; internal set; }
/// <summary>
/// Set of folders that are potentially modified, or whose children
/// are potentially modified
/// </summary>
public List<string> Folders { get; internal set; }
public List<string>? Folders { get; internal set; }
/// <summary>
/// Journal data to use for next backup
/// </summary>
public USNJournalDataEntry JournalData { get; internal set; }
public USNJournalDataEntry? JournalData { get; internal set; }
/// <summary>
/// If true, a full scan for this volume was required
@@ -516,6 +579,6 @@ namespace Duplicati.Library.Snapshots
/// <summary>
/// Optional exception message for volume
/// </summary>
public Exception Exception { get; internal set; }
public Exception? Exception { get; internal set; }
}
}
Duplicati/Library/Snapshots/Win32USN.cs → Duplicati/Library/Snapshots/USN/Win32USN.cs
+21 -21
View File
@@ -1,30 +1,30 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Runtime.InteropServices;
using System.Runtime.Versioning;
using Microsoft.Win32.SafeHandles;
namespace Duplicati.Library.Snapshots
namespace Duplicati.Library.Snapshots.USN
{
//The signatures in this file are from http://pinvoke.net
Duplicati/Library/Snapshots/Windows/AlphaVssBackup.cs
+200
View File
@@ -0,0 +1,200 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Alphaleonis.Win32.Vss;
using Duplicati.Library.Common.IO;
namespace Duplicati.Library.Snapshots.Windows;
/// <summary>
/// Implementation of a snapshot, using AlphaVSS
/// </summary>
internal class AlphaVssBackup : ISnapshotProvider
{
/// <summary>
/// The implementation that is being wrapped
/// </summary>
private IVssBackupComponents _components;
/// <summary>
/// Creates a new snapshot implementation
/// </summary>
/// <param name="components">The components to use</param>
private AlphaVssBackup(IVssBackupComponents components)
{
_components = components;
}
/// <inheritdoc/>
public void EnableWriterClasses(Guid[] guids)
=> _components.EnableWriterClasses(guids);
/// <inheritdoc/>
public void DisableWriterClasses(Guid[] guids)
=> _components.DisableWriterClasses(guids);
/// <inheritdoc/>
public void GatherWriterMetadata()
=> _components.GatherWriterMetadata();
/// <inheritdoc/>
public void FreeWriterMetadata()
=> _components.FreeWriterMetadata();
/// <inheritdoc/>
public void VerifyWriters(Guid[] guids)
{
foreach (var writerGUID in guids)
{
if (!_components.WriterMetadata.Any(o => o.WriterId.Equals(writerGUID)))
{
throw new Exception(string.Format("Writer with GUID {0} was not added to VSS writer set.", writerGUID.ToString()));
}
}
}
/// <summary>
/// Stub for returning the snapshot info
/// </summary>
/// <param name="SnapshotDeviceObject">The path where the snapshot is located</param>
private sealed record SnapshotInfo(string SnapshotDeviceObject) : ISnapshotInfo;
/// <inheritdoc/>
public ISnapshotInfo GetSnapshotProperties(Guid shadowId)
=> new SnapshotInfo(_components.GetSnapshotProperties(shadowId).SnapshotDeviceObject);
/// <inheritdoc/>
public void StartSnapshotSet()
=> _components.StartSnapshotSet();
/// <inheritdoc/>
public void PrepareForBackup()
=> _components.PrepareForBackup();
/// <inheritdoc/>
public void DoSnapshotSet()
=> _components.DoSnapshotSet();
/// <inheritdoc/>
public bool IsVolumeSupported(string drive)
=> _components.IsVolumeSupported(drive);
/// <inheritdoc/>
public Guid AddToSnapshotSet(string drive)
=> _components.AddToSnapshotSet(drive);
/// <inheritdoc/>
public void BackupComplete()
=> _components.BackupComplete();
/// <inheritdoc/>
public void DeleteSnapshot(Guid shadowId, bool forceDelete)
=> _components.DeleteSnapshot(shadowId, forceDelete);
/// <inheritdoc/>
public static ISnapshotProvider Create()
=> new AlphaVssBackup(GetVssBackupComponents());
/// <summary>
/// Creates a new AlphaVssBackupComponents instance and adds the context
/// </summary>
/// <returns>The configured instance</returns>
private static IVssBackupComponents GetVssBackupComponents()
{
//Prepare the backup
IVssBackupComponents vssBackupComponents = CreateVssBackupComponents();
vssBackupComponents.InitializeForBackup(null);
vssBackupComponents.SetContext(VssSnapshotContext.Backup);
vssBackupComponents.SetBackupState(false, true, VssBackupType.Full, false);
return vssBackupComponents;
}
/// <summary>
/// Loads the AlphaVSS implementation for the current OS
/// </summary>
/// <returns>The implementation</returns>
private static IVssBackupComponents CreateVssBackupComponents()
{
var vss = VssFactoryProvider.Default.GetVssFactory();
if (vss == null)
throw new InvalidOperationException();
return vss.CreateVssBackupComponents();
}
/// <inheritdoc/>
public IEnumerable<WriterMetaData> ParseWriterMetaData(Guid[] writers)
{
// check if writers got enabled
foreach (var writerGUID in writers)
{
var writer = _components.WriterMetadata.First(o => o.WriterId.Equals(writerGUID));
foreach (var component in writer.Components)
{
yield return new WriterMetaData
{
Guid = writerGUID,
Name = component.ComponentName,
LogicalPath = component.LogicalPath,
Paths = GetPathsFromComponent(component)
};
}
}
}
/// <summary>
/// Returns the paths from the writer metadata
/// </summary>
/// <param name="component">The component to get the paths from</param>
/// <returns>The list of paths</returns>
private List<string> GetPathsFromComponent(IVssWMComponent component)
{
var paths = new List<string>();
foreach (var file in component.Files)
{
if (file.FileSpecification.Contains("*"))
{
if (Directory.Exists(Util.AppendDirSeparator(file.Path)))
paths.Add(Util.AppendDirSeparator(file.Path));
}
else
{
var fileWithSpec = SystemIO.IO_OS.PathCombine(file.Path, file.FileSpecification);
if (File.Exists(fileWithSpec))
paths.Add(fileWithSpec);
}
}
return paths;
}
/// <inheritdoc/>
public void Dispose()
{
_components.Dispose();
}
}
Duplicati/Library/Snapshots/HyperVUtility.cs → Duplicati/Library/Snapshots/Windows/HyperVUtility.cs
+6 -5
View File
@@ -26,7 +26,7 @@ using System.Management;
using System.Runtime.Versioning;
using Duplicati.Library.Common.IO;
namespace Duplicati.Library.Snapshots
namespace Duplicati.Library.Snapshots.Windows
{
public class HyperVGuest : IEquatable<HyperVGuest>
{
@@ -160,8 +160,9 @@ namespace Duplicati.Library.Snapshots
/// Query Hyper-V for all Virtual Machines info
/// </summary>
/// <param name="bIncludePaths">Specify if returned data should contain VM paths</param>
/// <param name="provider">The provider to use for VSS</param>
/// <returns>List of Hyper-V Machines</returns>
public void QueryHyperVGuestsInfo(bool bIncludePaths = false)
public void QueryHyperVGuestsInfo(SnapshotProvider provider, bool bIncludePaths = false)
{
if (!IsHyperVInstalled)
return;
@@ -178,7 +179,7 @@ namespace Duplicati.Library.Snapshots
if (bIncludePaths)
{
foreach (var o in GetAllVMsPathsVSS())
foreach (var o in GetAllVMsPathsVSS(provider))
{
foreach (var mObject in moCollection)
{
@@ -218,9 +219,9 @@ namespace Duplicati.Library.Snapshots
/// For all Hyper-V guests it enumerate all associated paths using VSS data
/// </summary>
/// <returns>A collection of VMs and paths</returns>
private static IEnumerable<WriterMetaData> GetAllVMsPathsVSS()
private static IEnumerable<WriterMetaData> GetAllVMsPathsVSS(SnapshotProvider provider)
{
using (var vssBackupComponents = new VssBackupComponents())
using (var vssBackupComponents = new SnapshotManager(provider))
{
var writerGUIDS = new[] { HyperVWriterGuid };
Duplicati/Library/Snapshots/Windows/ISnapshotProvider.cs
+118
View File
@@ -0,0 +1,118 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
using System;
using System.Collections.Generic;
namespace Duplicati.Library.Snapshots.Windows;
/// <summary>
/// Interface for the snapshot information
/// </summary>
internal interface ISnapshotInfo
{
/// <summary>
/// The path where the snapshot is located
/// </summary>
string SnapshotDeviceObject { get; }
}
/// <summary>
/// Interface for a snapshot provider
/// </summary>
internal interface ISnapshotProvider : IDisposable
{
/// <summary>
/// Explicitly enables the selected writer GUIDs
/// </summary>
/// <param name="guids">The GUIDs to enable</param>
void EnableWriterClasses(Guid[] guids);
/// <summary>
/// Explicitly disable the selected writer GUIDs
/// </summary>
/// <param name="guids">The GUIDs to disable</param>
void DisableWriterClasses(Guid[] guids);
/// <summary>
/// Gather writer metadata prior to the snapshot
/// </summary>
void GatherWriterMetadata();
/// <summary>
/// Release any allocated metadata
/// </summary>
void FreeWriterMetadata();
/// <summary>
/// Verify that the writes are enabled on the snapshot
/// </summary>
/// <param name="guids">The writer GUIDs to verify as enabled</param>
void VerifyWriters(Guid[] guids);
/// <summary>
/// Gets information of a snapshot
/// </summary>
/// <param name="shadowId">The shadow copy id</param>
/// <returns>The snapshot information</returns>
ISnapshotInfo GetSnapshotProperties(Guid shadowId);
/// <summary>
/// Creates a snapshot
/// </summary>
void StartSnapshotSet();
/// <summary>
/// Starts the snapshot as a backup snapshot
/// </summary>
void PrepareForBackup();
/// <summary>
/// Finalizes the snapshot creation
/// </summary>
void DoSnapshotSet();
/// <summary>
/// Checks if the volume is supported by the snapshot
/// </summary>
/// <param name="drive">The drive to check</param>
/// <returns><c>true</c> if the volume is supported; <c>false<c/> otherwise</returns>
bool IsVolumeSupported(string drive);
/// <summary>
/// Adds a drive to the snapshot set
/// </summary>
/// <param name="drive">The drive to add</param>
/// <returns>The shadow id</returns>
Guid AddToSnapshotSet(string drive);
/// <summary>
/// Completes the snapshot after the backup has completed
/// </summary>
void BackupComplete();
/// <summary>
/// Releases all resources held by the snapshot
/// </summary>
/// <param name="shadowId">The snapshot shadow id</param>
/// <param name="forceDelete">Flag to choose force deletion</param>
void DeleteSnapshot(Guid shadowId, bool forceDelete);
/// <summary>
/// Parses write metadata
/// </summary>
/// <param name="writers">The writers to return information from</param>
/// <returns>The writer metadata</returns>
IEnumerable<WriterMetaData> ParseWriterMetaData(Guid[] writers);
}
Duplicati/Library/Snapshots/MSSQLUtility.cs → Duplicati/Library/Snapshots/Windows/MSSQLUtility.cs
+24 -25
View File
@@ -1,31 +1,30 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Common;
using Duplicati.Library.Common.IO;
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.Versioning;
namespace Duplicati.Library.Snapshots
namespace Duplicati.Library.Snapshots.Windows
{
public class MSSQLDB : IEquatable<MSSQLDB>
{
@@ -152,14 +151,14 @@ namespace Duplicati.Library.Snapshots
/// For all MS SQL databases it enumerate all associated paths using VSS data
/// </summary>
/// <returns>A collection of DBs and paths</returns>
public void QueryDBsInfo()
public void QueryDBsInfo(SnapshotProvider provider)
{
if (!IsMSSQLInstalled)
return;
m_DBs.Clear();
using (var vssBackupComponents = new VssBackupComponents())
using (var vssBackupComponents = new SnapshotManager(provider))
{
var writerGUIDS = new [] { MSSQLWriterGuid };
try
Duplicati/Library/Common/IO/VssBackupComponents.cs → Duplicati/Library/Snapshots/Windows/SnapshotManager.cs
+74 -104
View File
@@ -21,13 +21,16 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.Versioning;
using Alphaleonis.Win32.Vss;
using Duplicati.Library.Common.IO;
namespace Duplicati.Library.Common.IO
namespace Duplicati.Library.Snapshots.Windows
{
/// <summary>
/// Metadata from a writer instance
/// </summary>
public class WriterMetaData
{
public string Name { get; set; }
@@ -36,16 +39,21 @@ namespace Duplicati.Library.Common.IO
public List<string> Paths { get; set; }
}
/// <summary>
/// The manager for the Windows snapshot
/// </summary>
[SupportedOSPlatform("windows")]
public class VssBackupComponents : IDisposable
public class SnapshotManager : IDisposable
{
/// <summary>
/// The tag used for logging messages
/// </summary>
public static readonly string LOGTAG = Logging.Log.LogTagFromType<VssBackupComponents>();
private static readonly string LOGTAG = Logging.Log.LogTagFromType<SnapshotManager>();
private IVssBackupComponents _vssBackupComponents;
/// <summary>
/// The snapshot implementation
/// </summary>
private ISnapshotProvider _snapshotProvider;
/// <summary>
/// The list of snapshot ids for each volume, key is the path root, e.g. C:\.
@@ -69,26 +77,39 @@ namespace Duplicati.Library.Common.IO
/// </summary>
private List<DefineDosDevice> _mappedDrives;
public VssBackupComponents()
/// <summary>
/// Creates a new snapshot manager
/// </summary>
/// <param name="provider">The provider to use</param>
public SnapshotManager(SnapshotProvider provider)
{
_vssBackupComponents = VssBackupComponentsHelper.GetVssBackupComponents();
_snapshotProvider = provider switch {
SnapshotProvider.AlphaVSS => AlphaVssBackup.Create(),
SnapshotProvider.Wmic => new WmicVssBackup(),
_ => throw new ArgumentException($"Invalid provider: {provider}")
};
}
/// <summary>
/// Setup writers on the snapshot
/// </summary>
/// <param name="includedWriters">The explicitly included writers</param>
/// <param name="excludedWriters">The explicitly excluded writers</param>
public void SetupWriters(Guid[] includedWriters, Guid[] excludedWriters)
{
if (includedWriters != null && includedWriters.Length > 0)
_vssBackupComponents.EnableWriterClasses(includedWriters);
_snapshotProvider.EnableWriterClasses(includedWriters);
if (excludedWriters != null && excludedWriters.Length > 0)
_vssBackupComponents.DisableWriterClasses(excludedWriters);
_snapshotProvider.DisableWriterClasses(excludedWriters);
try
{
_vssBackupComponents.GatherWriterMetadata();
_snapshotProvider.GatherWriterMetadata();
}
finally
{
_vssBackupComponents.FreeWriterMetadata();
_snapshotProvider.FreeWriterMetadata();
}
if (includedWriters == null)
@@ -97,15 +118,12 @@ namespace Duplicati.Library.Common.IO
}
// check if writers got enabled
foreach (var writerGUID in includedWriters)
{
if (!_vssBackupComponents.WriterMetadata.Any(o => o.WriterId.Equals(writerGUID)))
{
throw new Exception(string.Format("Writer with GUID {0} was not added to VSS writer set.", writerGUID.ToString()));
}
}
_snapshotProvider.VerifyWriters(includedWriters);
}
/// <summary>
/// Maps drives from snapshot paths
/// </summary>
public void MapDrives()
{
_mappedDrives = new List<DefineDosDevice>();
@@ -124,6 +142,9 @@ namespace Duplicati.Library.Common.IO
}
}
/// <summary>
/// Makes a map of the snapshots for easy lookup
/// </summary>
public void MapVolumesToSnapShots()
{
_volumeMap = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
@@ -133,41 +154,17 @@ namespace Duplicati.Library.Common.IO
// Attempting to get attributes on this path will fail, so we need to append a double backslash
// Strangely, the double backslash is not needed when acessing a subfolder, here a single backslash is enough,
// but double backslash works for both cases due to path normalization.
_volumeMap.Add(kvp.Key, Util.AppendDirSeparator(_vssBackupComponents.GetSnapshotProperties(kvp.Value).SnapshotDeviceObject) + "\\");
_volumeMap.Add(kvp.Key, Util.AppendDirSeparator(_snapshotProvider.GetSnapshotProperties(kvp.Value).SnapshotDeviceObject) + "\\");
}
_volumeReverseMap = _volumeMap.ToDictionary(x => x.Value, x => x.Key);
}
public Dictionary<string, string> SnapshotDeviceAndVolumes
{
get
{
return _volumeReverseMap;
}
}
private List<string> GetPathsFromComponent(IVssWMComponent component)
{
var paths = new List<string>();
foreach (var file in component.Files)
{
if (file.FileSpecification.Contains("*"))
{
if (Directory.Exists(Util.AppendDirSeparator(file.Path)))
paths.Add(Util.AppendDirSeparator(file.Path));
}
else
{
var fileWithSpec = SystemIO.IO_OS.PathCombine(file.Path, file.FileSpecification);
if (File.Exists(fileWithSpec))
paths.Add(fileWithSpec);
}
}
return paths;
}
/// <summary>
/// Returns the map of snapshot path and logical path
/// </summary>
public Dictionary<string, string> SnapshotDeviceAndVolumes
=> _volumeReverseMap;
/// <summary>
/// Enumerable to iterate over components of the given writers.
@@ -176,38 +173,30 @@ namespace Duplicati.Library.Common.IO
/// <returns>Anonymous object containing GUID, component name, logical path and paths of the component.</returns>
/// <param name="writers">Writers.</param>
public IEnumerable<WriterMetaData> ParseWriterMetaData(Guid[] writers)
{
// check if writers got enabled
foreach (var writerGUID in writers)
{
var writer = _vssBackupComponents.WriterMetadata.First(o => o.WriterId.Equals(writerGUID));
foreach (var component in writer.Components)
{
yield return new WriterMetaData
{
Guid = writerGUID,
Name = component.ComponentName,
LogicalPath = component.LogicalPath,
Paths = GetPathsFromComponent(component)
};
}
}
}
=> _snapshotProvider.ParseWriterMetaData(writers);
/// <summary>
/// Prepares the snapshot and notifies all writers
/// </summary>
/// <param name="sources">The soruces to include</param>
public void InitShadowVolumes(IEnumerable<string> sources)
{
_vssBackupComponents.StartSnapshotSet();
_snapshotProvider.StartSnapshotSet();
CheckSupportedVolumes(sources);
CheckAndAddSupportedVolumes(sources);
//Make all writers aware that we are going to do the backup
_vssBackupComponents.PrepareForBackup();
_snapshotProvider.PrepareForBackup();
//Create the shadow volumes
_vssBackupComponents.DoSnapshotSet();
_snapshotProvider.DoSnapshotSet();
}
/// <summary>
/// Gets the snapshot path from the local path
/// </summary>
/// <param name="path">The local path</param>
/// <returns>The snapshot path</returns>
public string GetVolumeFromCache(string path)
{
if (!_volumeMap.TryGetValue(path, out var volumePath))
@@ -216,7 +205,11 @@ namespace Duplicati.Library.Common.IO
return volumePath;
}
public void CheckSupportedVolumes(IEnumerable<string> sources)
/// <summary>
/// Checks if the volumes can be added and creates snapshots as needed
/// </summary>
/// <param name="sources">The sources to create snapshots for</param>
private void CheckAndAddSupportedVolumes(IEnumerable<string> sources)
{
//Figure out which volumes are in the set
_volumes = new Dictionary<string, Guid>(StringComparer.OrdinalIgnoreCase);
@@ -225,16 +218,17 @@ namespace Duplicati.Library.Common.IO
var drive = SystemIO.IO_OS.GetPathRoot(s);
if (!_volumes.ContainsKey(drive))
{
if (!_vssBackupComponents.IsVolumeSupported(drive))
if (!_snapshotProvider.IsVolumeSupported(drive))
{
throw new VssVolumeNotSupportedException(drive);
}
_volumes.Add(drive, _vssBackupComponents.AddToSnapshotSet(drive));
_volumes.Add(drive, _snapshotProvider.AddToSnapshotSet(drive));
}
}
}
/// <inheritdoc/>
public void Dispose()
{
try
@@ -256,7 +250,7 @@ namespace Duplicati.Library.Common.IO
try
{
_vssBackupComponents?.BackupComplete();
_snapshotProvider?.BackupComplete();
}
catch (Exception ex)
{
@@ -265,13 +259,13 @@ namespace Duplicati.Library.Common.IO
try
{
if (_vssBackupComponents != null)
if (_snapshotProvider != null)
{
foreach (var g in _volumes.Values)
{
try
{
_vssBackupComponents.DeleteSnapshot(g, false);
_snapshotProvider.DeleteSnapshot(g, false);
}
catch (Exception ex)
{
@@ -285,35 +279,11 @@ namespace Duplicati.Library.Common.IO
Logging.Log.WriteVerboseMessage(LOGTAG, "VSSSnapShotDeleteCleanError", ex, "Failed during VSS esnapshot closing");
}
if (_vssBackupComponents != null)
if (_snapshotProvider != null)
{
_vssBackupComponents.Dispose();
_vssBackupComponents = null;
_snapshotProvider.Dispose();
_snapshotProvider = null;
}
}
}
public static class VssBackupComponentsHelper
{
public static IVssBackupComponents GetVssBackupComponents()
{
//Prepare the backup
IVssBackupComponents vssBackupComponents = CreateVssBackupComponents();
vssBackupComponents.InitializeForBackup(null);
vssBackupComponents.SetContext(VssSnapshotContext.Backup);
vssBackupComponents.SetBackupState(false, true, VssBackupType.Full, false);
return vssBackupComponents;
}
public static IVssBackupComponents CreateVssBackupComponents()
{
var vss = VssFactoryProvider.Default.GetVssFactory();
if (vss == null)
throw new InvalidOperationException();
return vss.CreateVssBackupComponents();
}
}
}
Duplicati/Library/Snapshots/Windows/WmicShadowCopyManager.cs
+240
View File
@@ -0,0 +1,240 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Runtime.Versioning;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
#nullable enable
namespace Duplicati.Library.Snapshots.Windows;
/// <summary>
/// A shadow copy manager using the wmic commandline tool
/// </summary>
[SupportedOSPlatform("windows")]
internal class WmicShadowCopyManager : IDisposable
{
/// <summary>
/// The tag used for logging messages
/// </summary>
private static readonly string LOGTAG = Logging.Log.LogTagFromType<SnapshotManager>();
/// <summary>
/// A single shadow copy
/// </summary>
/// <param name="shadowId">The shadow ID as a string</param>
/// <param name="parsedId">The shadow ID as a GUID</param>
/// <param name="originalDrive">The drive that the snapshot is for</param>
/// <param name="mappedPath">The path that contains the snapshot</param>
public class WmicShadowCopy(string shadowId, Guid parsedId, string originalDrive, string mappedPath) : IDisposable
{
/// <summary>
/// Gets the shadow ID
/// </summary>
public string ShadowID { get; } = shadowId;
/// <summary>
/// Gets the shadow ID
/// </summary>
public Guid ParsedId { get; } = parsedId;
/// <summary>
/// Gets the drive that was originally mapped
/// </summary>
public string OriginalDrive { get; } = originalDrive;
/// <summary>
/// Gets the path where the snapshot is found
/// </summary>
public string MappedPath { get; } = mappedPath;
/// <summary>
/// Flag keeping track of the snapshot deletion state
/// </summary>
private bool _snapshotDeleted;
/// <inheritdoc/>
public void Dispose()
{
DeleteShadowCopy();
}
/// <summary>
/// Deletes the shadow copy
/// </summary>
private void DeleteShadowCopy()
{
if (_snapshotDeleted)
return;
if (!string.IsNullOrEmpty(ShadowID))
{
_snapshotDeleted = true;
Logging.Log.WriteVerboseMessage(LOGTAG, "DeleteShadowCopy", $"Deleting Shadow Copy: {ShadowID}");
DeleteShadow(ShadowID);
}
}
}
/// <summary>
/// The list of the currently registered shadow copies
/// </summary>
private List<WmicShadowCopy> _shadowCopies = new List<WmicShadowCopy>();
/// <summary>
/// Gets the list of the currently registered shadow copies
/// </summary>
public IEnumerable<WmicShadowCopy> ShadowCopies => _shadowCopies;
/// <summary>
/// Creates a new snapshot for the given drive
/// </summary>
/// <param name="drive">The drive to create the snapshot for</param>
/// <returns>The created snapshot</returns>
public WmicShadowCopy Add(string drive)
{
var shadowId = CreateShadowCopy(drive);
if (string.IsNullOrEmpty(shadowId))
throw new InvalidOperationException("Failed to create shadow copy");
var shadowPath = GetShadowPath(shadowId);
if (string.IsNullOrEmpty(shadowPath))
{
DeleteShadow(shadowId);
throw new InvalidOperationException("Failed to get shadow copy path");
}
var snapshot = new WmicShadowCopy(shadowId, Guid.Parse(shadowId), drive, shadowPath);
_shadowCopies.Add(snapshot);
return snapshot;
}
/// <inheritdoc />
public void Dispose()
{
foreach (var shadow in ShadowCopies)
{
shadow.Dispose();
}
_shadowCopies.Clear();
}
/// <summary>
/// Creates a shadow copy
/// </summary>
/// <param name="drive">The drive to create the snapshot for</param>
/// <returns>The shadow id</returns>
private static string? CreateShadowCopy(string drive)
{
string output = ExecuteCommand("wmic", $"shadowcopy call create Volume='{drive}'", 10000);
// Extract ShadowID using regex
Match match = Regex.Match(output, @"ShadowID\s*=\s*""({[0-9A-F\-]+})""");
if (match.Success)
{
return match.Groups[1].Value;
}
Logging.Log.WriteErrorMessage(LOGTAG, "ShadowCopyFailed", null, "Failed to create shadow copy for {0}: {1}", drive, output);
return null;
}
/// <summary>
/// Gets the path where the shadow copy is mounted
/// </summary>
/// <param name="shadowId">The shadow copy id</param>
/// <returns>The path where the copy is mounted</returns>
private static string? GetShadowPath(string shadowId)
{
string output = ExecuteCommand("wmic", "shadowcopy get ID, DeviceObject", 5000);
// Extract DeviceObject corresponding to the ShadowID
string pattern = $@"(\\\\\?\\GLOBALROOT\\Device\\HarddiskVolumeShadowCopy\d+)\s+{shadowId}";
Match match = Regex.Match(output, pattern);
if (match.Success)
{
return match.Groups[1].Value;
}
Logging.Log.WriteErrorMessage(LOGTAG, "ShadowCopyFailed", null, "Failed to get shadow copy path for {0}: {1}", shadowId, output);
return null;
}
/// <summary>
/// Returns the drives that are vss enabled
/// </summary>
/// <returns></returns>
public static HashSet<string> GetVssCapableDrivesViaVssadmin()
{
var vssDrives = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
try
{
var output = ExecuteCommand("vssadmin", "list volumes", 5000);
// Regex to match drive letters in output
var matches = Regex.Matches(output, @"Volume\s+path:\s*([A-Z]:)\\?\s", RegexOptions.IgnoreCase);
foreach (Match match in matches)
vssDrives.Add(match.Groups[1].Value.Substring(0, 1));
}
catch (Exception ex)
{
Logging.Log.WriteErrorMessage(LOGTAG, "ShadowCopyListFailed", ex, "Failed to list volumes");
}
return vssDrives;
}
/// <summary>
/// Delete a shadow copy
/// </summary>
/// <param name="shadowId">The shadow id</param>
private static void DeleteShadow(string shadowId)
=> ExecuteCommand("wmic", $"shadowcopy where ID=\"{shadowId}\" delete", 5000);
/// <summary>
/// Executes a command
/// </summary>
/// <param name="fileName">The binary to execute</param>
/// <param name="arguments">The arguments to use</param>
/// <param name="timeoutMs">The timeout</param>
/// <returns>The output of the command</returns>
private static string ExecuteCommand(string fileName, string arguments, int timeoutMs)
{
try
{
using (var process = new Process())
{
process.StartInfo.FileName = fileName;
process.StartInfo.Arguments = arguments;
process.StartInfo.RedirectStandardOutput = true;
process.StartInfo.RedirectStandardError = true;
process.StartInfo.UseShellExecute = false;
process.StartInfo.CreateNoWindow = true;
process.Start();
var outputTask = process.StandardOutput.ReadToEndAsync();
var errorTask = process.StandardError.ReadToEndAsync();
if (!process.WaitForExit(timeoutMs))
{
process.Kill();
throw new TimeoutException($"Command '{fileName} {arguments}' timed out after {timeoutMs}ms.");
}
var output = outputTask.Result;
var error = errorTask.Result;
if (!string.IsNullOrWhiteSpace(error))
Logging.Log.WriteWarningMessage(LOGTAG, "ShadowCopyFailed", null, "Failed to execute command: {0} {1}: {2}", fileName, arguments, error);
return output;
}
}
catch (Exception ex)
{
Logging.Log.WriteErrorMessage(LOGTAG, "ShadowCopyFailed", ex, "Failed to execute command: {0} {1}", fileName, arguments);
return string.Empty;
}
}
}
Duplicati/Library/Snapshots/Windows/WmicVssBackup.cs
+131
View File
@@ -0,0 +1,131 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.Versioning;
namespace Duplicati.Library.Snapshots.Windows;
/// <summary>
/// Snapshot provider based on WMIC
/// </summary>
[SupportedOSPlatform("windows")]
internal class WmicVssBackup : ISnapshotProvider
{
/// <summary>
/// The manager that contains the actual interface to WMIC
/// </summary>
private WmicShadowCopyManager _manager = new WmicShadowCopyManager();
/// <summary>
/// Gets the VSS enabled drives
/// </summary>
private Lazy<HashSet<string>> _vssEnabledDrives = new Lazy<HashSet<string>>(() => WmicShadowCopyManager.GetVssCapableDrivesViaVssadmin());
/// <inheritdoc/>
public Guid AddToSnapshotSet(string drive)
=> _manager.Add(drive).ParsedId;
/// <inheritdoc/>
public void BackupComplete()
{
}
/// <inheritdoc/>
public void DeleteSnapshot(Guid shadowId, bool forceDelete)
{
foreach (var x in _manager.ShadowCopies.Where(x => x.ParsedId == shadowId))
x.Dispose();
}
/// <inheritdoc/>
public void DisableWriterClasses(Guid[] guids)
{
}
/// <inheritdoc/>
public void Dispose()
{
_manager.Dispose();
}
/// <inheritdoc/>
public void DoSnapshotSet()
{
}
/// <inheritdoc/>
public void EnableWriterClasses(Guid[] guids)
{
}
/// <inheritdoc/>
public void FreeWriterMetadata()
{
}
/// <inheritdoc/>
public void GatherWriterMetadata()
{
}
/// <summary>
/// Stub for returning the snapshot info
/// </summary>
/// <param name="SnapshotDeviceObject">The path where the snapshot is located</param>
private sealed record SnapshotInfo (string SnapshotDeviceObject) : ISnapshotInfo;
/// <inheritdoc/>
public ISnapshotInfo GetSnapshotProperties(Guid shadowId)
=> new SnapshotInfo(_manager.ShadowCopies.First(x => x.ParsedId == shadowId).MappedPath);
/// <inheritdoc/>
public bool IsVolumeSupported(string drive)
{
if (string.IsNullOrWhiteSpace(drive))
return false;
return _vssEnabledDrives.Value.Contains(drive.Substring(0, 1));
}
/// <inheritdoc/>
public IEnumerable<WriterMetaData> ParseWriterMetaData(Guid[] writers)
=> Enumerable.Empty<WriterMetaData>();
/// <inheritdoc/>
public void PrepareForBackup()
{
}
/// <inheritdoc/>
public void StartSnapshotSet()
{
}
/// <inheritdoc/>
public void VerifyWriters(Guid[] guids)
{
}
}
Duplicati/Library/Snapshots/WindowsSnapshot.cs
+45 -38
View File
@@ -19,13 +19,14 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.Versioning;
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.Snapshots.Windows;
namespace Duplicati.Library.Snapshots
{
@@ -52,20 +53,28 @@ namespace Duplicati.Library.Snapshots
/// <summary>
/// The main reference to the backup controller
/// </summary>
private readonly VssBackupComponents _vssBackupComponents;
private readonly SnapshotManager _snapshotManager;
/// <summary>
/// The source folders included in the snapshot
/// </summary>
private readonly IReadOnlyList<string> _sourceEntries;
/// <summary>
/// Constructs a new backup snapshot, using all the required disks
/// </summary>
/// <param name="sources">Sources to determine which volumes to include in snapshot</param>
/// <param name="options">A set of commandline options</param>
public WindowsSnapshot(IEnumerable<string> sources, IDictionary<string, string> options)
/// <param name="followSymlinks">A flag indicating if symlinks should be followed</param>
public WindowsSnapshot(IEnumerable<string> sources, IDictionary<string, string> options, bool followSymlinks)
: base(followSymlinks)
{
// For Windows, ensure we don't store paths with extended device path prefixes (i.e., @"\\?\" or @"\\?\UNC\")
sources = sources.Select(SystemIOWindows.RemoveExtendedDevicePathPrefix);
_sourceEntries = sources.Select(SystemIOWindows.RemoveExtendedDevicePathPrefix).ToList();
try
{
_vssBackupComponents = new VssBackupComponents();
var provider = Utility.Utility.ParseEnumOption(options.AsReadOnly(), "snapshot-provider", SnapshotProvider.AlphaVSS);
_snapshotManager = new SnapshotManager(provider);
// Default to exclude the System State writer
var excludedWriters = new Guid[] { new Guid("{e8132975-6f93-4464-a53e-1050253ae220}") };
@@ -77,16 +86,16 @@ namespace Duplicati.Library.Snapshots
.Select(x => new Guid(x))
.ToArray();
}
_vssBackupComponents.SetupWriters(null, excludedWriters);
_snapshotManager.SetupWriters(null, excludedWriters);
_vssBackupComponents.InitShadowVolumes(sources);
_snapshotManager.InitShadowVolumes(_sourceEntries);
_vssBackupComponents.MapVolumesToSnapShots();
_snapshotManager.MapVolumesToSnapShots();
//If we should map the drives, we do that now and update the volumeMap
if (Utility.Utility.ParseBoolOption(options.AsReadOnly(), "vss-use-mapping"))
{
_vssBackupComponents.MapDrives();
_snapshotManager.MapDrives();
}
}
catch (Exception ex1)
@@ -108,6 +117,26 @@ namespace Duplicati.Library.Snapshots
}
}
/// <summary>
/// Gets the source folders
/// </summary>
public override IEnumerable<string> SourceEntries => _sourceEntries;
/// <summary>
/// Enumerates the root source files and folders
/// </summary>
/// <returns>The source files and folders</returns>
public override IEnumerable<ISourceProviderEntry> EnumerateFilesystemEntries()
{
foreach (var folder in _sourceEntries)
{
if (folder.EndsWith(Path.DirectorySeparatorChar) || DirectoryExists(folder))
yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(folder), true, true);
else
yield return new SnapshotSourceFileEntry(this, folder, false, true);
}
}
#region Private functions
/// <summary>
@@ -163,18 +192,6 @@ namespace Duplicati.Library.Snapshots
#region ISnapshotService Members
/// <summary>
/// Enumerates all files and folders in the snapshot, restricted to sources
/// </summary>
/// <param name="sources">Sources to enumerate</param>
/// <param name="callback">The callback to invoke with each found path</param>
/// <param name="errorCallback">The callback used to report errors</param>
public override IEnumerable<string> EnumerateFilesAndFolders(IEnumerable<string> sources, Utility.Utility.EnumerationFilterDelegate callback, Utility.Utility.ReportAccessError errorCallback)
{
// For Windows, ensure we don't store paths with extended device path prefixes (i.e., @"\\?\" or @"\\?\UNC\")
return base.EnumerateFilesAndFolders(sources.Select(SystemIOWindows.RemoveExtendedDevicePathPrefix), callback, errorCallback);
}
/// <summary>
/// Gets the last write time of a given file in UTC
/// </summary>
@@ -246,23 +263,16 @@ namespace Duplicati.Library.Snapshots
/// <returns>The metadata for the given file or folder</returns>
/// <param name="localPath">The file or folder to examine</param>
/// <param name="isSymlink">A flag indicating if the target is a symlink</param>
/// <param name="followSymlink">A flag indicating if a symlink should be followed</param>
public override Dictionary<string, string> GetMetadata(string localPath, bool isSymlink, bool followSymlink)
public override Dictionary<string, string> GetMetadata(string localPath, bool isSymlink)
{
return IO_WIN.GetMetadata(ConvertToSnapshotPath(localPath), isSymlink, followSymlink);
return SystemIO.IO_OS.GetMetadata(ConvertToSnapshotPath(localPath), isSymlink, FollowSymlinks);
}
/// <inheritdoc />
public override bool IsBlockDevice(string localPath)
{
return false;
}
public override bool IsBlockDevice(string localPath) => false;
/// <inheritdoc />
public override string HardlinkTargetID(string localPath)
{
return null;
}
public override string HardlinkTargetID(string localPath) => null;
/// <inheritdoc />
public override string ConvertToLocalPath(string snapshotPath)
@@ -270,7 +280,7 @@ namespace Duplicati.Library.Snapshots
if (!Path.IsPathRooted(snapshotPath))
throw new InvalidOperationException();
foreach (var kvp in _vssBackupComponents.SnapshotDeviceAndVolumes)
foreach (var kvp in _snapshotManager.SnapshotDeviceAndVolumes)
{
if (snapshotPath.StartsWith(kvp.Key, Utility.Utility.ClientFilenameStringComparison))
return IO_WIN.PathCombine(kvp.Value, snapshotPath.Substring(kvp.Key.Length));
@@ -289,7 +299,7 @@ namespace Duplicati.Library.Snapshots
throw new InvalidOperationException();
var root = IO_WIN.GetPathRoot(localPath);
var volumePath = _vssBackupComponents.GetVolumeFromCache(root);
var volumePath = _snapshotManager.GetVolumeFromCache(root);
// Note that using a simple Path.Combine() for the following code
// can result in invalid snapshot paths; e.g., if localPath is
@@ -317,9 +327,6 @@ namespace Duplicati.Library.Snapshots
return IO_WIN.DirectoryExists(ConvertToSnapshotPath(localFolderPath));
}
/// <inheritdoc />
public override bool IsSnapshot => true;
#endregion
/// <inheritdoc />
@@ -327,7 +334,7 @@ namespace Duplicati.Library.Snapshots
{
if (disposing)
{
_vssBackupComponents?.Dispose();
_snapshotManager?.Dispose();
}
base.Dispose(disposing);
Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs
+345
View File
@@ -0,0 +1,345 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System.IO.Pipelines;
using System.Runtime.CompilerServices;
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
namespace Duplicati.Library.SourceProvider;
/// <summary>
/// Creates a new backend source entry
/// </summary>
/// <param name="parent">The parent backend</param>
/// <param name="path">The path of the entry</param>
/// <param name="isFolder">True if the entry is a folder</param>
/// <param name="isRootEntry">True if the entry is a meta entry</param>
/// <param name="createdUtc">The creation time of the entry</param>
/// <param name="lastModificationUtc">The last modification time of the entry</param>
/// <param name="size">The size of the entry</param>
public class BackendSourceFileEntry(BackendSourceProvider parent, string path, bool isFolder, bool isRootEntry, DateTime createdUtc, DateTime lastModificationUtc, long size)
: ISourceProviderEntry
{
/// <summary>
/// The log tag for this instance
/// </summary>
private static string LOGTAG = Logging.Log.LogTagFromType<BackendSourceFileEntry>();
/// <inheritdoc/>
public bool IsFolder => isFolder;
/// <inheritdoc/>
public bool IsMetaEntry => false;
/// <inheritdoc/>
public bool IsRootEntry => isRootEntry;
/// <inheritdoc/>
public DateTime CreatedUtc => createdUtc;
/// <inheritdoc/>
public DateTime LastModificationUtc => lastModificationUtc;
/// <inheritdoc/>
public string Path => SystemIO.IO_OS.PathCombine(parent?.MountedPath, NormalizePathToLocalSystem(path));
/// <inheritdoc/>
public long Size => size;
/// <inheritdoc/>
public bool IsSymlink => false;
/// <inheritdoc/>
public string? SymlinkTarget => null;
/// <inheritdoc/>
public FileAttributes Attributes => IsFolder
? FileAttributes.Directory
: FileAttributes.Normal;
/// <inheritdoc/>
public Dictionary<string, string> MinorMetadata => new Dictionary<string, string>();
/// <inheritdoc/>
public bool IsBlockDevice => false;
/// <inheritdoc/>
public bool IsCharacterDevice => false;
/// <inheritdoc/>
public bool IsAlternateStream => false;
/// <inheritdoc/>
public string? HardlinkTargetId => null;
/// <summary>
/// An async enumerator for the entries
/// </summary>
private IAsyncEnumerator<ISourceProviderEntry>? preparedEnumerator = null;
/// <summary>
/// A flag to indicate if the prepared enumerator has any entries
/// </summary>
private bool preparedEnumeratorAny = false;
/// <summary>
/// Prepares the enumerator for this entry
/// </summary>
/// <param name="cancellationToken">The cancellation token</param>
/// <returns>The prepared enumerator</returns>
public async Task PrepareEnumerator(CancellationToken cancellationToken)
{
if (!isFolder || !isRootEntry)
throw new InvalidOperationException("PrepareEnumerator can only be called on root folders");
if (preparedEnumerator != null)
throw new InvalidOperationException("PrepareEnumerator can only be called once");
var result = EnumerateInternal(cancellationToken).GetAsyncEnumerator(cancellationToken);
var prev = Interlocked.Exchange(ref preparedEnumerator, result);
if (prev != null)
throw new InvalidOperationException("PrepareEnumerator can only be called once");
// Advance to the first entry, so we are sure it does not throw exceptions
preparedEnumeratorAny = await result.MoveNextAsync().ConfigureAwait(false);
}
/// <inheritdoc/>
public async IAsyncEnumerable<ISourceProviderEntry> Enumerate([EnumeratorCancellation] CancellationToken cancellationToken)
{
if (!isFolder)
throw new InvalidOperationException("Enumerate can only be called on folders");
if (isRootEntry)
{
// If we have a prepared enumerator, consume and use it
var enumerator = Interlocked.Exchange(ref preparedEnumerator, null);
if (enumerator != null)
{
if (preparedEnumeratorAny)
{
// It has already been advanced, so return the current value
yield return enumerator.Current;
while (await enumerator.MoveNextAsync())
yield return enumerator.Current;
}
yield break;
}
}
// Otherwise, enumerate the entries
await foreach (var entry in EnumerateInternal(cancellationToken).ConfigureAwait(false))
yield return entry;
}
private IAsyncEnumerable<ISourceProviderEntry> EnumerateInternal(CancellationToken cancellationToken)
=> parent.WrappedBackend.ListAsync(NormalizePathTo(path, '/'), cancellationToken)
// Remove the current and parent folder entries
.Where(x => !string.IsNullOrWhiteSpace(x.Name) && x.Name != "." && x.Name != "..")
// Remove sub-folder entries
.Where(x => !x.Name[0..^1].Contains('\\') && !x.Name[0..^1].Contains('/'))
// Convert to source file entries
.Select(x =>
{
var localPath = SystemIO.IO_OS.PathCombine(path, NormalizePathToLocalSystem(x.Name));
if (x.IsFolder)
localPath = Util.AppendDirSeparator(localPath);
return new BackendSourceFileEntry(
parent,
localPath,
x.IsFolder,
false,
x.Created,
x.LastModification,
x.Size
);
});
/// <inheritdoc/>
public async Task<bool> FileExists(string path, CancellationToken cancellationToken)
{
if (!isFolder)
throw new InvalidOperationException("FileExists cannot be called on folders");
try
{
var entry = await parent.WrappedBackend.GetEntryAsync(NormalizePathTo(path, '/'), cancellationToken);
return entry != null && !entry.IsFolder;
}
catch (FileNotFoundException)
{
return false;
}
}
/// <inheritdoc/>
public Task<Stream?> OpenMetadataRead(CancellationToken cancellationToken)
=> Task.FromResult<Stream?>(null);
/// <summary>
/// Helper class for reporting th length of a stream
/// </summary>
/// <param name="stream">The stream to wrap</param>
/// <param name="reportedLength">The length to report</param>
private class LengthReportingStream(Stream stream, long reportedLength) : OverrideableStream(stream)
{
/// <summary>
/// Track the position
/// </summary>
private long position = 0;
/// <inheritdoc/>
public override long Length => reportedLength;
/// <inheritdoc/>
public override long Position
{
get => position;
set => position = base.Position = value;
}
/// <inheritdoc/>
public override void Write(byte[] buffer, int offset, int count)
{
base.Write(buffer, offset, count);
position += count;
}
/// <inheritdoc/>
public override int Read(byte[] buffer, int offset, int count)
{
var read = base.Read(buffer, offset, count);
position += read;
return read;
}
/// <inheritdoc/>
public override async Task<int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
{
var read = await base.ReadAsync(buffer, offset, count, cancellationToken);
position += read;
return read;
}
/// <inheritdoc/>
public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
{
await base.WriteAsync(buffer, offset, count, cancellationToken);
position += count;
}
}
/// <inheritdoc/>
public async Task<Stream> OpenRead(CancellationToken cancellationToken)
{
if (isFolder)
throw new InvalidOperationException("OpenRead can only be called on files");
if (parent.WrappedBackend is IStreamingBackend streamingBackend)
{
var pipe = new Pipe();
// Start writing data to the pipe asynchronously
_ = Task.Run(async () =>
{
try
{
await streamingBackend.GetAsync(NormalizePathTo(path, '/'), pipe.Writer.AsStream(), cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
await pipe.Writer.CompleteAsync(ex).ConfigureAwait(false);
}
finally
{
await pipe.Writer.CompleteAsync().ConfigureAwait(false);
}
})
.ContinueWith(t =>
{
if (t.IsFaulted)
Logging.Log.WriteWarningMessage(LOGTAG, "ErrorPipingStream", t.Exception, "Error piping stream for {0}", this.Path);
});
// Return the readable stream so the caller can read data as it's produced
return new LengthReportingStream(pipe.Reader.AsStream(), size);
}
else
{
TempFile? tempFile = null;
TempFileStream? file = null;
try
{
tempFile = new TempFile();
await parent.WrappedBackend.GetAsync(path, tempFile, cancellationToken);
file = TempFileStream.Create(tempFile);
return file;
}
catch
{
file?.Dispose();
tempFile?.Dispose();
throw;
}
}
}
/// <summary>
/// Normalizes the path, turning backslashes into forward slashes,
/// or vice versa, depending on the platform
/// </summary>
/// <param name="path">The path to normalize</param>
/// <returns>The normalized path</returns>
public static string NormalizePathToLocalSystem(string path)
=> NormalizePathTo(path, System.IO.Path.DirectorySeparatorChar);
/// <summary>
/// Normalizes the path, turning backslashes into forward slashes,
/// or vice versa, depending on the platform
/// </summary>
/// <param name="path">The path to normalize</param>
/// <returns>The normalized path</returns>
public static string NormalizePathTo(string path, char separator)
{
if (string.IsNullOrEmpty(path))
return path;
return path
.Replace('/', separator)
.Replace('\\', separator);
}
/// <summary>
/// Creates a new backend source entry from a file entry
/// </summary>
/// <param name="parent">The parent backend</param>
/// <param name="entry">The file entry</param>
/// <param name="prefix">The prefix to add to the path</param>
/// <returns>The new backend source entry</returns>
public static BackendSourceFileEntry FromFileEntry(BackendSourceProvider parent, string prefix, IFileEntry entry)
=> new BackendSourceFileEntry(parent, SystemIO.IO_OS.PathCombine(prefix, entry.Name), entry.IsFolder, false, entry.Created, entry.LastModification, entry.Size);
}
Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs
+99
View File
@@ -0,0 +1,99 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Interface;
namespace Duplicati.Library.SourceProvider;
/// <summary>
/// A source provider that wraps a backend
/// </summary>
/// <param name="backend">The backend to wrap</param>
/// <param name="mountedPath">The path to mount the backend</param>
public class BackendSourceProvider(IFolderEnabledBackend backend, string mountedPath) : ISourceProvider, ISourceProviderModule
{
/// <summary>
/// The wrapped backend
/// </summary>
public IFolderEnabledBackend WrappedBackend => backend;
/// <summary>
/// The path where the provider is logically mounted
/// </summary>
public string MountedPath => mountedPath;
/// <inheritdoc/>
public string Key => backend.ProtocolKey;
/// <inheritdoc/>
public string DisplayName => backend.DisplayName;
/// <inheritdoc/>
public IList<ICommandLineArgument> SupportedCommands => backend.SupportedCommands;
/// <summary>
/// The prepared root entry, if any
/// </summary>
private BackendSourceFileEntry? preparedRoot;
/// <summary>
/// Flags if the provider has been initialized
/// </summary>
private int isInitialized = 0;
/// <summary>
/// Creates a root entry
/// </summary>
/// <returns>The root entry</returns>
private BackendSourceFileEntry CreateRoot()
=> new BackendSourceFileEntry(this, "", true, true, new DateTime(0), new DateTime(0), 0);
/// <inheritdoc/>
public async Task Initialize(CancellationToken cancellationToken)
{
// Only allow a single intiiialization call
if (Interlocked.Exchange(ref isInitialized, 1) != 0)
return;
// Prepare the root entry
var root = CreateRoot();
await root.PrepareEnumerator(cancellationToken).ConfigureAwait(false);
preparedRoot = root;
}
/// <inheritdoc/>
public IAsyncEnumerable<ISourceProviderEntry> Enumerate(CancellationToken cancellationToken)
=> new[] { Interlocked.Exchange(ref preparedRoot, null) ?? CreateRoot() }.ToAsyncEnumerable();
/// <inheritdoc/>
public async Task<ISourceProviderEntry?> GetEntry(string path, bool isFolder, CancellationToken cancellationToken)
{
var entry = await backend.GetEntryAsync(BackendSourceFileEntry.NormalizePathTo(path, '/'), cancellationToken).ConfigureAwait(false);
return entry == null ? null : BackendSourceFileEntry.FromFileEntry(this, path, entry);
}
/// <inheritdoc/>
public void Dispose()
{
WrappedBackend.Dispose();
GC.SuppressFinalize(this);
}
}
Duplicati/Library/SourceProvider/Builtin/Combiner.cs
+102
View File
@@ -0,0 +1,102 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System.Runtime.CompilerServices;
using Duplicati.Library.Interface;
namespace Duplicati.Library.SourceProvider;
/// <summary>
/// Helper class that wraps multiple source providers into a single source provider
/// </summary>
public class Combiner(IEnumerable<ISourceProvider> providers) : ISourceProvider
{
/// <inheritdoc/>
public string MountedPath => string.Empty;
/// <summary>
/// The providers to combine
/// </summary>
private readonly List<ISourceProvider> providers = providers.SelectMany(x => x is Combiner c ? c.providers.AsEnumerable() : [x]).ToList();
/// <inheritdoc/>
public Task Initialize(CancellationToken cancellationToken)
=> Task.WhenAll(providers.Select(x => x.Initialize(cancellationToken)));
/// <inheritdoc />
public async IAsyncEnumerable<ISourceProviderEntry> Enumerate([EnumeratorCancellation] CancellationToken cancellationToken)
{
foreach (var provider in providers)
{
await foreach (var entry in provider.Enumerate(cancellationToken).ConfigureAwait(false))
yield return entry;
}
}
/// <summary>
/// Provides access to the combined providers
/// </summary>
public IEnumerable<ISourceProvider> Providers => providers;
/// <summary>
/// Gets a filesystem entry for a given path
/// </summary>
/// <param name="path">The path to get</param>
/// <param name="isFolder">True if the path is a folder</param>
/// <param name="cancellationToken">The cancellation token</param>
/// <returns>The filesystem entry</returns>
public async Task<ISourceProviderEntry?> GetEntry(string path, bool isFolder, CancellationToken cancellationToken)
{
foreach (var provider in providers)
if (!string.IsNullOrWhiteSpace(MountedPath) && provider.MountedPath.StartsWith(path))
{
var subpath = provider.MountedPath.Substring(path.Length);
return await provider.GetEntry(subpath, isFolder, cancellationToken).ConfigureAwait(false);
}
foreach (var provider in providers.Where(x => string.IsNullOrWhiteSpace(x.MountedPath)))
{
var res = await provider.GetEntry(path, isFolder, cancellationToken).ConfigureAwait(false);
if (res != null)
return res;
}
return null;
}
/// <summary>
/// Creates a combined source provider from a list of providers
/// </summary>
/// <param name="providers">The providers to combine</param>
/// <returns>The combined provider</returns>
public static ISourceProvider Combine(IEnumerable<ISourceProvider> providers)
{
var providerTotal = providers.SelectMany(x => x is Combiner c ? c.providers.AsEnumerable() : [x]).ToList();
return providerTotal.Count == 1 ? providerTotal[0] : new Combiner(providers);
}
/// <inheritdoc/>
public void Dispose()
{
foreach (var provider in providers)
provider.Dispose();
}
}
Duplicati/Library/SourceProvider/Builtin/Duplicati.Library.SourceProvider.Builtin.csproj
+31
View File
@@ -0,0 +1,31 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<OutputType>Library</OutputType>
<RootNamespace>Duplicati.Library.SourceProvider</RootNamespace>
<Copyright>Copyright © 2024 Team Duplicati, MIT license</Copyright>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<RootNamespace>Duplicati.Library.SourceProvider</RootNamespace>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\..\Interface\Duplicati.Library.Interface.csproj" />
<ProjectReference Include="..\..\Localization\Duplicati.Library.Localization.csproj" />
<ProjectReference Include="..\..\Utility\Duplicati.Library.Utility.csproj" />
<ProjectReference Include="..\..\Snapshots\Duplicati.Library.Snapshots.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.DotNet.Analyzers.Compatibility" Version="0.2.12-alpha">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
</ItemGroup>
<ItemGroup>
<PackageReference Include="System.IO.Pipelines" Version="9.0.2" />
</ItemGroup>
</Project>
Duplicati/Library/SourceProvider/Builtin/LocalFileSource.cs
+58
View File
@@ -0,0 +1,58 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Interface;
using Duplicati.Library.Snapshots;
namespace Duplicati.Library.SourceProvider;
/// <summary>
/// A source provider that wraps a file system through a snapshot service
/// </summary>
/// <param name="snapshotService">The snapshot service to use</param>
public class LocalFileSource(ISnapshotService snapshotService) : ISourceProvider
{
/// <inheritdoc/>
public string MountedPath => string.Empty;
/// <inheritdoc/>
public Task Initialize(CancellationToken cancellationToken) => Task.CompletedTask;
/// <inheritdoc/>
public IAsyncEnumerable<ISourceProviderEntry> Enumerate(CancellationToken cancellationToken)
=> snapshotService.EnumerateFilesystemEntries().ToAsyncEnumerable();
/// <summary>
/// The snapshot service being used
/// </summary>
public ISnapshotService SnapshotService => snapshotService;
/// <inheritdoc/>
public Task<ISourceProviderEntry?> GetEntry(string path, bool isFolder, CancellationToken cancellationToken)
=> Task.FromResult(snapshotService.GetFilesystemEntry(path, isFolder));
/// <inheritdoc/>
public void Dispose()
{
snapshotService.Dispose();
GC.SuppressFinalize(this);
}
}
Duplicati/Library/SourceProvider/Builtin/Strings.cs
+29
View File
@@ -0,0 +1,29 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Localization.Short;
namespace Duplicati.Library.SourceProvider.Strings
{
internal static class File
{
public static string Description { get { return LC.L(@"This backend can read and write data to an file based backend. Allowed formats are ""file://hostname/folder"" and ""file://username:password@hostname/folder"". You may supply UNC paths (e.g.: ""file://\\server\folder"") or local paths (e.g.: (win) ""file://c:\folder"", (linux) ""file:///usr/pub/files""). Use the prefix ""vss://"" or ""lvm://"" to create snapshot based folders."); } }
public static string DisplayName { get { return LC.L(@"Local folder or drive"); } }
}
}
Duplicati/Library/SourceProviders/Duplicati.Library.SourceProviders.csproj
+14
View File
@@ -0,0 +1,14 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<Description>Central reference to all source providers</Description>
<RootNamespace>Duplicati.Library.Backend</RootNamespace>
<Copyright>Copyright © 2024 Team Duplicati, MIT license</Copyright>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\Interface\Duplicati.Library.Interface.csproj" />
</ItemGroup>
</Project>
Duplicati/Library/SourceProviders/SourceProviderModules.cs
+45
View File
@@ -0,0 +1,45 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Interface;
namespace Duplicati.Library.SourceProviders;
/// <summary>
/// Shared list of statically linked source-provider modules
/// </summary>
public static class SourceProviderModules
{
/// <summary>
/// The list of all built-in source-provider modules
/// </summary>
public static IReadOnlyList<ISourceProviderModule> BuiltInSourceProviderModules => SupportedSourceProviders;
/// <summary>
/// Calculate list once and cache it
/// </summary>
private static readonly IReadOnlyList<ISourceProviderModule> SupportedSourceProviders = new ISourceProviderModule[] {
// ... No providers currently implemented
}
.Where(x => x != null)
.ToList();
}
Duplicati/Library/Utility/OverrideableStream.cs
+67 -69
View File
@@ -1,28 +1,27 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
using System.Threading.Tasks;
namespace Duplicati.Library.Utility
{
@@ -33,79 +32,78 @@ namespace Duplicati.Library.Utility
/// </summary>
public class OverrideableStream : Stream
{
protected System.IO.Stream m_basestream;
/// <summary>
/// The base stream that is wrapped
/// </summary>
protected Stream m_basestream;
/// <summary>
/// Creates a new <see cref="OverrideableStream"/> instance
/// </summary>
/// <param name="basestream">The stream to wrap</param>
/// <exception cref="ArgumentNullException"></exception>
public OverrideableStream(Stream basestream)
{
if (basestream == null)
throw new ArgumentNullException(nameof(basestream));
m_basestream = basestream;
m_basestream = basestream ?? throw new ArgumentNullException(nameof(basestream));
}
public override bool CanRead
{
get { return m_basestream.CanRead; }
}
/// <inheritdoc/>
public override bool CanRead => m_basestream.CanRead;
public override bool CanSeek
{
get { return m_basestream.CanSeek; }
}
/// <inheritdoc/>
public override bool CanSeek => m_basestream.CanSeek;
public override bool CanWrite
{
get { return m_basestream.CanWrite; }
}
/// <inheritdoc/>
public override bool CanWrite => m_basestream.CanWrite;
public override void Flush()
{
m_basestream.Flush();
}
/// <inheritdoc/>
public override void Flush() => m_basestream.Flush();
public override long Length
{
get { return m_basestream.Length; }
}
/// <inheritdoc/>
public override long Length => m_basestream.Length;
/// <inheritdoc/>
public override long Position
{
get
{
return m_basestream.Position;
}
set
{
m_basestream.Position = value;
}
get => m_basestream.Position;
set => m_basestream.Position = value;
}
/// <inheritdoc/>
public override int Read(byte[] buffer, int offset, int count)
{
return m_basestream.Read(buffer, offset, count);
}
=> m_basestream.Read(buffer, offset, count);
/// <inheritdoc/>
public override long Seek(long offset, System.IO.SeekOrigin origin)
{
return m_basestream.Seek(offset, origin);
}
=> m_basestream.Seek(offset, origin);
/// <inheritdoc/>
public override void SetLength(long value)
{
m_basestream.SetLength(value);
}
=> m_basestream.SetLength(value);
/// <inheritdoc/>
public override void Write(byte[] buffer, int offset, int count)
{
m_basestream.Write(buffer, offset, count);
}
=> m_basestream.Write(buffer, offset, count);
/// <inheritdoc/>
public override Task<int> ReadAsync(byte[] buffer, int offset, int count, System.Threading.CancellationToken cancellationToken)
=> m_basestream.ReadAsync(buffer, offset, count, cancellationToken);
/// <inheritdoc/>
public override Task WriteAsync(byte[] buffer, int offset, int count, System.Threading.CancellationToken cancellationToken)
=> m_basestream.WriteAsync(buffer, offset, count, cancellationToken);
/// <inheritdoc/>
public override Task FlushAsync(System.Threading.CancellationToken cancellationToken)
=> m_basestream.FlushAsync(cancellationToken);
/// <inheritdoc/>
protected override void Dispose(bool disposing)
{
if (m_basestream != null)
m_basestream.Dispose();
m_basestream?.Dispose();
m_basestream = null;
base.Dispose(disposing);
}
}
}
Duplicati/Library/Utility/TempFileStream.cs
+97
View File
@@ -0,0 +1,97 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
using System.IO;
using System.Threading;
using System.Threading.Tasks;
namespace Duplicati.Library.Utility;
/// <summary>
/// A stream that wraps a temporary file.
/// The temporary file is automatically deleted when the stream is disposed.
/// </summary>
public class TempFileStream : StreamUtil.WrappingAsyncStream
{
/// <summary>
/// The temporary file instance.
/// </summary>
private readonly TempFile tempFile;
/// <summary>
/// The stream for the temporary file.
/// </summary>
private readonly Stream stream;
/// <summary>
/// The path to the temporary file.
/// </summary>
public string Path => tempFile.Name;
/// <summary>
/// Creates a new TempFileStream instance.
/// </summary>
/// <param name="tempFile">The temporary file instance.</param>
/// <param name="stream">The stream for the temporary file.</param>
private TempFileStream(TempFile tempFile, Stream stream)
: base(stream)
{
this.tempFile = tempFile;
this.stream = stream;
}
/// <summary>
/// Creates a new TempFileStream instance.
/// </summary>
/// <param name="tempFile">The temporary file instance.</param>
/// <returns>The new TempFileStream instance.</returns>
public static TempFileStream Create(TempFile tempFile)
{
return new TempFileStream(tempFile, File.Open(tempFile.Name, FileMode.Open, FileAccess.ReadWrite, FileShare.None));
}
/// <summary>
/// Creates a new TempFileStream instance.
/// </summary>
/// <returns>The new TempFileStream instance.</returns>
public static TempFileStream Create()
{
var tempFile = new TempFile();
return new TempFileStream(tempFile, File.Open(tempFile.Name, FileMode.Open, FileAccess.ReadWrite, FileShare.None));
}
/// <inheritdoc/>
protected override Task<int> ReadImplAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
=> stream.ReadAsync(buffer, offset, count, cancellationToken);
/// <inheritdoc/>
protected override Task WriteImplAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
=> stream.WriteAsync(buffer, offset, count, cancellationToken);
/// <inheritdoc/>
protected override void Dispose(bool disposing)
{
base.Dispose(disposing);
stream.Dispose();
tempFile.Dispose();
}
}
Duplicati/Library/Utility/Utility.cs
+60 -50
View File
@@ -246,7 +246,7 @@ namespace Duplicati.Library.Utility
/// <returns>A list of the full filenames and foldernames. Foldernames ends with the directoryseparator char</returns>
public static IEnumerable<string> EnumerateFileSystemEntries(string basepath)
{
return EnumerateFileSystemEntries(basepath, (rootpath, path, attributes) => true, SystemIO.IO_OS.GetDirectories, Directory.GetFiles, null);
return EnumerateFileSystemEntries(basepath, SystemIO.IO_OS.GetDirectories, Directory.GetFiles, null);
}
/// <summary>
@@ -276,13 +276,12 @@ namespace Duplicati.Library.Utility
/// The search is recursive.
/// </summary>
/// <param name="rootpath">The folder to look in</param>
/// <param name="callback">The function to call with the filenames</param>
/// <param name="folderList">A function to call that lists all folders in the supplied folder</param>
/// <param name="fileList">A function to call that lists all files in the supplied folder</param>
/// <param name="attributeReader">A function to call that obtains the attributes for an element, set to null to avoid reading attributes</param>
/// <param name="errorCallback">An optional function to call with error messages.</param>
/// <returns>A list of the full filenames</returns>
public static IEnumerable<string> EnumerateFileSystemEntries(string rootpath, EnumerationFilterDelegate callback, FileSystemInteraction folderList, FileSystemInteraction fileList, ExtractFileAttributes? attributeReader, ReportAccessError? errorCallback = null)
public static IEnumerable<string> EnumerateFileSystemEntries(string rootpath, FileSystemInteraction folderList, FileSystemInteraction fileList, ExtractFileAttributes? attributeReader, ReportAccessError? errorCallback = null)
{
var lst = new Stack<string>();
@@ -292,8 +291,7 @@ namespace Duplicati.Library.Utility
try
{
var attr = attributeReader?.Invoke(rootpath) ?? FileAttributes.Directory;
if (callback(rootpath, rootpath, attr))
lst.Push(rootpath);
lst.Push(rootpath);
}
catch (System.Threading.ThreadAbortException)
{
@@ -302,7 +300,6 @@ namespace Duplicati.Library.Utility
catch (Exception ex)
{
errorCallback?.Invoke(rootpath, rootpath, ex);
callback(rootpath, rootpath, FileAttributes.Directory | ATTRIBUTE_ERROR);
}
while (lst.Count > 0)
@@ -319,8 +316,7 @@ namespace Duplicati.Library.Utility
try
{
var attr = attributeReader?.Invoke(sf) ?? FileAttributes.Directory;
if (callback(rootpath, sf, attr))
lst.Push(sf);
lst.Push(sf);
}
catch (System.Threading.ThreadAbortException)
{
@@ -329,7 +325,6 @@ namespace Duplicati.Library.Utility
catch (Exception ex)
{
errorCallback?.Invoke(rootpath, sf, ex);
callback(rootpath, sf, FileAttributes.Directory | ATTRIBUTE_ERROR);
}
}
}
@@ -340,7 +335,6 @@ namespace Duplicati.Library.Utility
catch (Exception ex)
{
errorCallback?.Invoke(rootpath, f, ex);
callback(rootpath, f, FileAttributes.Directory | ATTRIBUTE_ERROR);
}
string[]? files = null;
@@ -357,54 +351,18 @@ namespace Duplicati.Library.Utility
catch (Exception ex)
{
errorCallback?.Invoke(rootpath, f, ex);
callback(rootpath, f, FileAttributes.Directory | ATTRIBUTE_ERROR);
}
}
if (files != null)
{
foreach (var s in files)
{
try
{
var attr = attributeReader?.Invoke(s) ?? FileAttributes.Normal;
if (!callback(rootpath, s, attr))
continue;
}
catch (System.Threading.ThreadAbortException)
{
throw;
}
catch (Exception ex)
{
errorCallback?.Invoke(rootpath, s, ex);
callback(rootpath, s, ATTRIBUTE_ERROR);
continue;
}
yield return s;
}
}
}
}
else
{
try
{
var attr = attributeReader?.Invoke(rootpath) ?? FileAttributes.Normal;
if (!callback(rootpath, rootpath, attr))
yield break;
}
catch (System.Threading.ThreadAbortException)
{
throw;
}
catch (Exception ex)
{
errorCallback?.Invoke(rootpath, rootpath, ex);
callback(rootpath, rootpath, ATTRIBUTE_ERROR);
yield break;
}
yield return rootpath;
}
}
@@ -1023,7 +981,7 @@ namespace Duplicati.Library.Utility
/// <param name="collection">The collection to remove duplicate items from.</param>
/// <param name="duplicateItems">The duplicate items in <paramref name="collection"/>.</param>
/// <returns>The unique items from <paramref name="collection"/>.</returns>
public static ISet<T> GetUniqueItems<T>(IEnumerable<T> collection, out ISet<T> duplicateItems)
public static IList<T> GetUniqueItems<T>(IEnumerable<T> collection, out ISet<T> duplicateItems)
{
return GetUniqueItems(collection, EqualityComparer<T>.Default, out duplicateItems);
}
@@ -1036,18 +994,21 @@ namespace Duplicati.Library.Utility
/// <param name="comparer">The <see cref="System.Collections.Generic.IEqualityComparer{T}"/> implementation to use when comparing values in the collection.</param>
/// <param name="duplicateItems">The duplicate items in <paramref name="collection"/>.</param>
/// <returns>The unique items from <paramref name="collection"/>.</returns>
public static ISet<T> GetUniqueItems<T>(IEnumerable<T> collection, IEqualityComparer<T> comparer, out ISet<T> duplicateItems)
public static IList<T> GetUniqueItems<T>(IEnumerable<T> collection, IEqualityComparer<T> comparer, out ISet<T> duplicateItems)
{
var uniqueItems = new HashSet<T>(comparer);
var results = new List<T>();
duplicateItems = new HashSet<T>(comparer);
foreach (var item in collection)
{
if (!uniqueItems.Add(item))
if (uniqueItems.Add(item))
results.Add(item);
else
duplicateItems.Add(item);
}
return uniqueItems;
return results;
}
// <summary>
@@ -1591,5 +1552,54 @@ namespace Duplicati.Library.Utility
foreach (var iex in FlattenException(ex.InnerException))
yield return iex;
}
/// <summary>
/// Guesses the URL scheme and returns it
/// </summary>
/// <param name="url">The URL to guess the scheme for</param>
/// <returns>The guessed scheme, or null if no scheme was found</returns>
public static string? GuessScheme(string url)
{
if (string.IsNullOrWhiteSpace(url))
return null;
var idx = url.IndexOf("://");
if (idx < 0 && idx < 15 && idx + "://".Length < url.Length)
return null;
return url.Substring(0, idx);
}
/// <summary>
/// Returns a url that is safe to display, by removing any credentials
/// </summary>
/// <param name="url">The url to sanitize</param>
/// <returns>The sanitized url</returns>
public static string GetUrlWithoutCredentials(string url)
{
// Assumed safe part of the url to show
const int maxShown = 25;
if (string.IsNullOrWhiteSpace(url))
return url;
// Use a reportable url without credentials
var sepIndex = Math.Max(0, url.IndexOf('|')) + 1;
var length = url.Length - sepIndex;
var shown = Math.Min(length, maxShown);
var hidden = length - maxShown;
var sanitizedUrl = $"{url[sepIndex..(sepIndex + shown)]}{new string('*', hidden)}";
// If we can parse it, this result is better
try
{
var uri = new Uri(url[sepIndex..]);
sanitizedUrl = new Uri($"{uri.Scheme}://{uri.Host}").SetPath(uri.Path).ToString();
}
catch
{
}
return sanitizedUrl;
}
}
}
Duplicati/UnitTest/ImportExportTests.cs
+51 -39
View File
@@ -1,28 +1,29 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Collections.Generic;
using System.IO;
using Duplicati.Library.RestAPI;
using Duplicati.Library.SQLiteHelper;
using Duplicati.Server;
using Duplicati.Server.Database;
using Duplicati.Server.Serializable;
@@ -126,46 +127,57 @@ namespace Duplicati.UnitTest
[Category("ImportExport")]
public void RoundTrip()
{
Dictionary<string, string> metadata = new Dictionary<string, string> { { "SourceFilesCount", "1" } };
Dictionary<string, string> advancedOptions = new Dictionary<string, string> { { "server-datafolder", this.serverDatafolder } };
var metadata = new Dictionary<string, string> { { "SourceFilesCount", "1" } };
var advancedOptions = new Dictionary<string, string> { { "server-datafolder", this.serverDatafolder } };
// Mock the setup enough to get the import/export working
var serviceCollection = new ServiceCollection();
serviceCollection.AddSingleton<INotificationUpdateService, NotificationUpdateService>();
serviceCollection.AddSingleton(new EventPollNotify());
FIXMEGlobal.Provider = new DefaultServiceProviderFactory().CreateServiceProvider(serviceCollection);
FIXMEGlobal.DataFolder = this.serverDatafolder;
using (Program.DataConnection = Program.GetDatabaseConnection(advancedOptions, true))
var dbpath = Path.Combine(this.serverDatafolder, Library.AutoUpdater.DataFolderManager.SERVER_DATABASE_FILENAME);
if (File.Exists(dbpath))
File.Delete(dbpath);
var con = SQLiteLoader.LoadConnection();
SQLiteLoader.OpenDatabase(con, dbpath, null);
DatabaseUpgrader.UpgradeDatabase(con, dbpath, typeof(Duplicati.Library.RestAPI.Database.DatabaseConnectionSchemaMarker));
using (var connection = new Connection(con, true, null))
{
// Unencrypted file, don't import metadata.
string unencryptedWithoutMetadata = Path.Combine(this.serverDatafolder, Path.GetRandomFileName());
File.WriteAllBytes(unencryptedWithoutMetadata, BackupImportExportHandler.ExportToJSON(Program.DataConnection, this.CreateBackup("unencrypted without metadata", "user", "password", metadata), null));
BackupImportExportHandler.ImportBackup(Program.DataConnection, unencryptedWithoutMetadata, false, () => null);
Assert.AreEqual(1, Program.DataConnection.Backups.Length);
Assert.AreEqual(0, Program.DataConnection.Backups[0].Metadata.Count);
File.WriteAllBytes(unencryptedWithoutMetadata, BackupImportExportHandler.ExportToJSON(connection, this.CreateBackup("unencrypted without metadata", "user", "password", metadata), null));
BackupImportExportHandler.ImportBackup(connection, unencryptedWithoutMetadata, false, () => null);
Assert.AreEqual(1, connection.Backups.Length);
Assert.AreEqual(0, connection.Backups[0].Metadata.Count);
// Unencrypted file, import metadata.
string unencryptedWithMetadata = Path.Combine(this.serverDatafolder, Path.GetRandomFileName());
File.WriteAllBytes(unencryptedWithMetadata, BackupImportExportHandler.ExportToJSON(Program.DataConnection, this.CreateBackup("unencrypted with metadata", "user", "password", metadata), null));
BackupImportExportHandler.ImportBackup(Program.DataConnection, unencryptedWithMetadata, true, () => null);
Assert.AreEqual(2, Program.DataConnection.Backups.Length);
Assert.AreEqual(metadata.Count, Program.DataConnection.Backups[1].Metadata.Count);
File.WriteAllBytes(unencryptedWithMetadata, BackupImportExportHandler.ExportToJSON(connection, this.CreateBackup("unencrypted with metadata", "user", "password", metadata), null));
BackupImportExportHandler.ImportBackup(connection, unencryptedWithMetadata, true, () => null);
Assert.AreEqual(2, connection.Backups.Length);
Assert.AreEqual(metadata.Count, connection.Backups[1].Metadata.Count);
// Encrypted file, don't import metadata.
string encryptedWithoutMetadata = Path.Combine(this.serverDatafolder, Path.GetRandomFileName());
string passphrase = "abcde";
File.WriteAllBytes(encryptedWithoutMetadata, BackupImportExportHandler.ExportToJSON(Program.DataConnection, this.CreateBackup("encrypted without metadata", "user", "password", metadata), passphrase));
BackupImportExportHandler.ImportBackup(Program.DataConnection, encryptedWithoutMetadata, false, () => passphrase);
Assert.AreEqual(3, Program.DataConnection.Backups.Length);
Assert.AreEqual(0, Program.DataConnection.Backups[2].Metadata.Count);
File.WriteAllBytes(encryptedWithoutMetadata, BackupImportExportHandler.ExportToJSON(connection, this.CreateBackup("encrypted without metadata", "user", "password", metadata), passphrase));
BackupImportExportHandler.ImportBackup(connection, encryptedWithoutMetadata, false, () => passphrase);
Assert.AreEqual(3, connection.Backups.Length);
Assert.AreEqual(0, connection.Backups[2].Metadata.Count);
// Encrypted file, import metadata.
string encryptedWithMetadata = Path.Combine(this.serverDatafolder, Path.GetRandomFileName());
File.WriteAllBytes(encryptedWithMetadata, BackupImportExportHandler.ExportToJSON(Program.DataConnection, this.CreateBackup("encrypted with metadata", "user", "password", metadata), passphrase));
BackupImportExportHandler.ImportBackup(Program.DataConnection, encryptedWithMetadata, true, () => passphrase);
Assert.AreEqual(4, Program.DataConnection.Backups.Length);
Assert.AreEqual(metadata.Count, Program.DataConnection.Backups[3].Metadata.Count);
File.WriteAllBytes(encryptedWithMetadata, BackupImportExportHandler.ExportToJSON(connection, this.CreateBackup("encrypted with metadata", "user", "password", metadata), passphrase));
BackupImportExportHandler.ImportBackup(connection, encryptedWithMetadata, true, () => passphrase);
Assert.AreEqual(4, connection.Backups.Length);
Assert.AreEqual(metadata.Count, connection.Backups[3].Metadata.Count);
// Encrypted file, incorrect passphrase.
Assert.Throws(Is.InstanceOf<Exception>(), () => BackupImportExportHandler.ImportBackup(Program.DataConnection, encryptedWithMetadata, true, () => passphrase + " "));
Assert.Throws(Is.InstanceOf<Exception>(), () => BackupImportExportHandler.ImportBackup(connection, encryptedWithMetadata, true, () => passphrase + " "));
}
}
}
Duplicati/UnitTest/SVNCheckoutsTest.cs
+7 -7
View File
@@ -407,6 +407,13 @@ namespace Duplicati.UnitTest
restoredname = System.IO.Path.Combine(System.IO.Path.Combine(tempfolder, System.IO.Path.GetFileName(rootfolder.Split(System.IO.Path.PathSeparator)[int.Parse(s.Substring(0, six))])), s.Substring(six + 1));
}
if (!System.IO.File.Exists(sourcename))
{
Log.WriteErrorMessage(LOGTAG, "PartialRestoreMissingFile", null, "Partial restore, missing SOURCE file: {0}", sourcename);
BasicSetupHelper.ProgressWriteLine("Partial restore missing file: " + sourcename);
throw new Exception("Unittest is broken");
}
if (!System.IO.File.Exists(restoredname))
{
Log.WriteErrorMessage(LOGTAG, "PartialRestoreMissingFile", null, "Partial restore missing file: {0}", restoredname);
@@ -414,13 +421,6 @@ namespace Duplicati.UnitTest
}
else
{
if (!System.IO.File.Exists(sourcename))
{
Log.WriteErrorMessage(LOGTAG, "PartialRestoreMissingFile", null, "Partial restore missing file: {0}", sourcename);
BasicSetupHelper.ProgressWriteLine("Partial restore missing file: " + sourcename);
throw new Exception("Unittest is broken");
}
TestUtils.AssertFilesAreEqual(sourcename, restoredname, verifymetadata, $"Partial restore file differs: {s}");
}
}
Duplicati/UnitTest/TestUtils.cs
+19 -43
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
@@ -24,25 +24,17 @@ using System.IO;
using System.Collections.Generic;
using Duplicati.Library.Utility;
using System.Linq;
using Duplicati.Library.Logging;
using System.Reflection;
using System.Threading;
using System.Threading.Tasks;
using Duplicati.Library.Common.IO;
using NUnit.Framework;
using System.Runtime.InteropServices;
using Duplicati.Library.Common;
using Duplicati.Library.Interface;
namespace Duplicati.UnitTest
{
public static class TestUtils
{
/// <summary>
/// The log tag
/// </summary>
private static readonly string LOGTAG = Library.Logging.Log.LogTagFromType(typeof(TestUtils));
public static Dictionary<string, string> DefaultOptions
{
get
@@ -161,22 +153,6 @@ namespace Duplicati.UnitTest
Console.WriteLine("Encountered additional {0} timestamp errors!", timestampfailures);
}
/// <summary>
/// Returns the index of a given string, using the file system case sensitivity
/// </summary>
/// <returns>The index of the entry or -1 if no entry was found</returns>
/// <param name='lst'>The list to search</param>
/// <param name='m'>The string to find</param>
private static int IndexOf(List<string> lst, string m)
{
StringComparison sc = Duplicati.Library.Utility.Utility.ClientFilenameStringComparison;
for (int i = 0; i < lst.Count; i++)
if (lst[i].Equals(m, sc))
return i;
return -1;
}
/// <summary>
/// Asserts that the two directory trees are equivalent; i.e.,
/// that they they contain the same directories and files, recursively.
Duplicati/UnitTest/UtilityTests.cs
+20 -21
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Utility;
@@ -349,8 +349,7 @@ namespace Duplicati.UnitTest
string[] duplicateItems = { "A", "c" };
// Test with default comparer.
ISet<string> actualDuplicateItems;
ISet<string> actualUniqueItems = Utility.GetUniqueItems(collection, out actualDuplicateItems);
var actualUniqueItems = Utility.GetUniqueItems(collection, out var actualDuplicateItems);
CollectionAssert.AreEquivalent(uniqueItems, actualUniqueItems);
CollectionAssert.AreEquivalent(duplicateItems, actualDuplicateItems);
Duplicati/WebserverCore/Endpoints/V1/Filesystem.cs
+4 -4
View File
@@ -254,11 +254,11 @@ public class Filesystem : IEndpointV1
{
var attr = SystemIO.IO_OS.GetFileAttributes(s);
var isSymlink = SystemIO.IO_OS.IsSymlink(s, attr);
var isFolder = (attr & FileAttributes.Directory) != 0;
var isFolder = attr.HasFlag(FileAttributes.Directory);
var isFile = !isFolder;
var isHidden = (attr & FileAttributes.Hidden) != 0;
bool isSystem = (attr & FileAttributes.System) != 0;
bool isTemporary = (attr & FileAttributes.Temporary) != 0;
var isHidden = attr.HasFlag(FileAttributes.Hidden);
bool isSystem = attr.HasFlag(FileAttributes.System);
bool isTemporary = attr.HasFlag(FileAttributes.Temporary);
long fileSize = -1;
var accessible = isFile || canAccess(s);
Duplicati/WebserverCore/Endpoints/V1/HyperV.cs
+3 -2
View File
@@ -21,6 +21,7 @@
using System.Runtime.Versioning;
using System.Security.Principal;
using Duplicati.Library.Snapshots;
using Duplicati.Library.Snapshots.Windows;
using Duplicati.WebserverCore.Abstractions;
using Duplicati.WebserverCore.Exceptions;
using Microsoft.AspNetCore.Mvc;
@@ -58,7 +59,7 @@ public class HyperV : IEndpointV1
{
if (string.IsNullOrEmpty(key))
{
hypervUtility.QueryHyperVGuestsInfo();
hypervUtility.QueryHyperVGuestsInfo(SnapshotProvider.AlphaVSS);
return hypervUtility.Guests.Select(x => new Dto.TreeNodeDto()
{
id = x.ID.ToString(),
@@ -77,7 +78,7 @@ public class HyperV : IEndpointV1
}
else
{
hypervUtility.QueryHyperVGuestsInfo(true);
hypervUtility.QueryHyperVGuestsInfo(SnapshotProvider.AlphaVSS, true);
var foundVMs = hypervUtility.Guests.FindAll(x => x.ID.Equals(new Guid(key)));
if (foundVMs.Count != 1)
Duplicati/WebserverCore/Endpoints/V1/MSSql.cs
+2 -1
View File
@@ -21,6 +21,7 @@
using System.Runtime.Versioning;
using System.Security.Principal;
using Duplicati.Library.Snapshots;
using Duplicati.Library.Snapshots.Windows;
using Duplicati.WebserverCore.Abstractions;
using Duplicati.WebserverCore.Exceptions;
using Microsoft.AspNetCore.Mvc;
@@ -56,7 +57,7 @@ public class MSSql : IEndpointV1
try
{
mssqlUtility.QueryDBsInfo();
mssqlUtility.QueryDBsInfo(SnapshotProvider.AlphaVSS);
if (string.IsNullOrEmpty(key))
{
Executables/net8/Duplicati.CommandLine.SourceTool/Duplicati.CommandLine.SourceTool.csproj
+22
View File
@@ -0,0 +1,22 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net8.0</TargetFramework>
<Description>SyncTool for Duplicati</Description>
<Copyright>Copyright © 2025 Team Duplicati, MIT license</Copyright>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\..\..\Duplicati\Library\Crashlog\Duplicati.Library.Crashlog.csproj" />
<ProjectReference Include="..\..\..\Duplicati\CommandLine\SourceTool\Duplicati.CommandLine.SourceTool.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.DotNet.Analyzers.Compatibility" Version="0.2.12-alpha">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
</ItemGroup>
</Project>
Executables/net8/Duplicati.CommandLine.SourceTool/Program.cs
+31
View File
@@ -0,0 +1,31 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Crashlog;
namespace Duplicati.CommandLine.SourceTool.Net8
{
// Wrapper class to keep code independent
public static class Program
{
public static int Main(string[] args)
=> CrashlogHelper.WrapWithCrashLog(() => SourceTool.Program.Main(args).ConfigureAwait(false).GetAwaiter().GetResult());
}
}
ReleaseBuilder/Build/Command.cs
+1
View File
@@ -87,6 +87,7 @@ public static partial class Command
{ "Duplicati.CommandLine.ServerUtil", "duplicati-server-util" },
{ "Duplicati.CommandLine.SecretTool", "duplicati-secret-tool" },
{ "Duplicati.CommandLine.SyncTool", "duplicati-sync-tool" },
{ "Duplicati.CommandLine.SourceTool", "duplicati-source-tool" },
{ "Duplicati.Service", "duplicati-service" },
{ "Duplicati.Agent", "duplicati-agent" },
{ "Duplicati.CommandLine", "duplicati-cli" },

Some files were not shown because too many files have changed in this diff Show More