From 8c935a7344df2678cd8af638e59af86e375719e7 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sun, 9 Feb 2025 16:14:22 +0100 Subject: [PATCH 01/28] This creates a new ISourceProvider interface for describing a data source. With this interface it is possible to wrap the current support for snapshots and USN and simplify the enumeration. This makes a number of assumptions into fields such as `IsMetaEntry` and `IsRootEntry` that removes code that was checking for various properties. This first commit has support for VSS and USN as well as "naked" files, keeping feature parity. There is also an experimental support for loading a backend-as-a-source. The support for this is currently limited as the backends need to be extended to support a folder concept. --- Duplicati.sln | 15 + .../Duplicati.Library.Backend.File.csproj | 1 + Duplicati/Library/Backend/File/FileBackend.cs | 138 ++++++-- .../Library/Backend/SSHv2/SSHv2Backend.cs | 42 +-- .../Library/Common/IO/VssBackupComponents.cs | 40 ++- .../Duplicati.Library.DynamicLoader.csproj | 1 + .../DynamicLoader/SourceProviderLoader.cs | 180 ++++++++++ .../Interface/IFolderEnabledBackend.cs | 53 +++ .../Library/Interface/ISourceFileEntry.cs | 126 +++++++ .../Library/Interface/ISourceProvider.cs | 51 +++ .../Interface/ISourceProviderModule.cs | 39 +++ .../Main/Duplicati.Library.Main.csproj | 2 + .../Library/Main/Operation/Backup/Channels.cs | 3 +- .../Operation/Backup/CountFilesHandler.cs | 58 ++-- .../Operation/Backup/FileBlockProcessor.cs | 67 ++-- .../Backup/FileEnumerationProcess.cs | 325 +++++++++++------- .../Operation/Backup/FilePreFilterProcess.cs | 66 ++-- .../Operation/Backup/MetadataGenerator.cs | 59 ++-- .../Operation/Backup/MetadataPreProcess.cs | 95 ++--- .../Library/Main/Operation/BackupHandler.cs | 143 ++++++-- .../Main/Operation/TestFilterHandler.cs | 86 ++--- .../Library/Modules/Builtin/ReportHelper.cs | 2 +- .../Modules/Builtin/SendJabberMessage.cs | 3 +- .../Duplicati.Library.Snapshots.csproj | 4 +- .../Library/Snapshots/ISnapshotService.cs | 102 +++--- Duplicati/Library/Snapshots/LinuxSnapshot.cs | 103 ++++-- .../Library/Snapshots/NoSnapshotLinux.cs | 74 ++-- .../Library/Snapshots/NoSnapshotWindows.cs | 134 ++++---- Duplicati/Library/Snapshots/Program.cs | 2 +- .../Snapshots/RecursiveFilterHelper.cs | 229 ++++++++++++ Duplicati/Library/Snapshots/SnapshotBase.cs | 89 +++-- .../Snapshots/SnapshotSourceFileEntry.cs | 143 ++++++++ .../Library/Snapshots/SnapshotUtility.cs | 92 ++--- .../Library/Snapshots/UsnJournalService.cs | 227 +----------- .../Library/Snapshots/WindowsSnapshot.cs | 93 ++--- .../Builtin/BackendSourceProvider.cs | 211 ++++++++++++ .../SourceProvider/Builtin/Combiner.cs | 82 +++++ ...cati.Library.SourceProvider.Builtin.csproj | 27 ++ .../Library/SourceProvider/Builtin/File.cs | 52 +++ .../Library/SourceProvider/Builtin/Strings.cs | 29 ++ .../Duplicati.Library.SourceProviders.csproj | 14 + .../SourceProviders/SourceProviderModules.cs | 45 +++ Duplicati/Library/Utility/MapStream.cs | 176 ++++++++++ Duplicati/Library/Utility/TempFileStream.cs | 87 +++++ Duplicati/Library/Utility/Utility.cs | 65 +--- .../WebserverCore/Endpoints/V1/Filesystem.cs | 8 +- 46 files changed, 2702 insertions(+), 981 deletions(-) create mode 100644 Duplicati/Library/DynamicLoader/SourceProviderLoader.cs create mode 100644 Duplicati/Library/Interface/IFolderEnabledBackend.cs create mode 100644 Duplicati/Library/Interface/ISourceFileEntry.cs create mode 100644 Duplicati/Library/Interface/ISourceProvider.cs create mode 100644 Duplicati/Library/Interface/ISourceProviderModule.cs create mode 100644 Duplicati/Library/Snapshots/RecursiveFilterHelper.cs create mode 100644 Duplicati/Library/Snapshots/SnapshotSourceFileEntry.cs create mode 100644 Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs create mode 100644 Duplicati/Library/SourceProvider/Builtin/Combiner.cs create mode 100644 Duplicati/Library/SourceProvider/Builtin/Duplicati.Library.SourceProvider.Builtin.csproj create mode 100644 Duplicati/Library/SourceProvider/Builtin/File.cs create mode 100644 Duplicati/Library/SourceProvider/Builtin/Strings.cs create mode 100644 Duplicati/Library/SourceProviders/Duplicati.Library.SourceProviders.csproj create mode 100644 Duplicati/Library/SourceProviders/SourceProviderModules.cs create mode 100644 Duplicati/Library/Utility/MapStream.cs create mode 100644 Duplicati/Library/Utility/TempFileStream.cs diff --git a/Duplicati.sln b/Duplicati.sln index 879a03397..eaf3aee27 100644 --- a/Duplicati.sln +++ b/Duplicati.sln @@ -166,6 +166,12 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.Crashlog" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.Backend.CIFS", "Duplicati\Library\Backend\CIFS\Duplicati.Library.Backend.CIFS.csproj", "{836E0557-B40C-4DC7-9A2A-5C062F9ACC6B}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.SourceProviders", "Duplicati\Library\SourceProviders\Duplicati.Library.SourceProviders.csproj", "{8A2C7A9F-3EC8-4DE5-A9FA-9E4BCF955EF3}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SourceProviders", "SourceProviders", "{424CEF73-4984-430E-9C8B-E61CC0F22074}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.SourceProvider.Builtin", "Duplicati\Library\SourceProvider\Builtin\Duplicati.Library.SourceProvider.Builtin.csproj", "{6D9742A7-6F32-4571-8276-79716168B525}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -472,6 +478,14 @@ Global {836E0557-B40C-4DC7-9A2A-5C062F9ACC6B}.Debug|Any CPU.Build.0 = Debug|Any CPU {836E0557-B40C-4DC7-9A2A-5C062F9ACC6B}.Release|Any CPU.ActiveCfg = Release|Any CPU {836E0557-B40C-4DC7-9A2A-5C062F9ACC6B}.Release|Any CPU.Build.0 = Release|Any CPU + {8A2C7A9F-3EC8-4DE5-A9FA-9E4BCF955EF3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8A2C7A9F-3EC8-4DE5-A9FA-9E4BCF955EF3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8A2C7A9F-3EC8-4DE5-A9FA-9E4BCF955EF3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8A2C7A9F-3EC8-4DE5-A9FA-9E4BCF955EF3}.Release|Any CPU.Build.0 = Release|Any CPU + {6D9742A7-6F32-4571-8276-79716168B525}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6D9742A7-6F32-4571-8276-79716168B525}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6D9742A7-6F32-4571-8276-79716168B525}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6D9742A7-6F32-4571-8276-79716168B525}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -532,6 +546,7 @@ Global {CA1A0EE8-DFFF-42AF-B15D-EF538357A2DE} = {E1A9B303-F281-45C5-A4F6-CADD9DE3F3C4} {8ACA2736-3C69-4FA5-BCF9-5EDF50CAF332} = {566EBBDA-19A4-4056-A615-D901D57D2439} {836E0557-B40C-4DC7-9A2A-5C062F9ACC6B} = {E1A9B303-F281-45C5-A4F6-CADD9DE3F3C4} + {6D9742A7-6F32-4571-8276-79716168B525} = {424CEF73-4984-430E-9C8B-E61CC0F22074} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {8B40BAFE-D862-4397-9495-8F5EAF5CE80C} diff --git a/Duplicati/Library/Backend/File/Duplicati.Library.Backend.File.csproj b/Duplicati/Library/Backend/File/Duplicati.Library.Backend.File.csproj index f8a99e4c3..60e2917a7 100644 --- a/Duplicati/Library/Backend/File/Duplicati.Library.Backend.File.csproj +++ b/Duplicati/Library/Backend/File/Duplicati.Library.Backend.File.csproj @@ -11,6 +11,7 @@ + diff --git a/Duplicati/Library/Backend/File/FileBackend.cs b/Duplicati/Library/Backend/File/FileBackend.cs index 89dbb44fc..de049627b 100644 --- a/Duplicati/Library/Backend/File/FileBackend.cs +++ b/Duplicati/Library/Backend/File/FileBackend.cs @@ -1,29 +1,32 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; +using Duplicati.Library.Snapshots; using System; using System.Collections.Generic; using System.IO; +using System.Linq; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Versioning; using System.Threading; @@ -33,7 +36,7 @@ namespace Duplicati.Library.Backend { // ReSharper disable once UnusedMember.Global // This class is instantiated dynamically in the BackendLoader. - public class File : IBackend, IStreamingBackend, IQuotaEnabledBackend, IRenameEnabledBackend + public class File : IBackend, IStreamingBackend, IQuotaEnabledBackend, IRenameEnabledBackend, IFolderEnabledBackend { private const string OPTION_DESTINATION_MARKER = "alternate-destination-marker"; private const string OPTION_ALTERNATE_PATHS = "alternate-target-paths"; @@ -452,16 +455,107 @@ namespace Duplicati.Library.Backend long? sourceStreamLength = sourceStream == null ? null : Utility.Utility.GetStreamLength(sourceStream, out isStreamPostion); if (sourceStreamLength.HasValue && targetFileInfo.Length != sourceStreamLength.Value) - throw new FileMissingException($"Target file size ({targetFileInfo.Length:n0}) is different from the source length ({sourceStreamLength.Value:n0}){(isStreamPostion ? " - ending stream position)" : "")}. Target: {targetFilePath}"); + throw new FileMissingException($"Target file size ({targetFileInfo.Length:n0}) is different from the source length ({sourceStreamLength.Value:n0}){(isStreamPostion ? " - ending stream position" : "")}. Target: {targetFilePath}"); if (expectedLength.HasValue && targetFileInfo.Length != expectedLength.Value) throw new FileMissingException($"Target file size ({targetFileInfo.Length:n0}) is different from the expected length ({expectedLength.Value:n0}). Target: {targetFilePath}"); } catch { - try { System.IO.File.Delete(targetFilePath); } catch { } + try { System.IO.File.Delete(targetFilePath); } + catch { } throw; } } + + /// + public async IAsyncEnumerable ListAsync(string path, [EnumeratorCancellation] CancellationToken cancellationToken) + { + PreAuthenticate(); + + var remotePath = string.IsNullOrWhiteSpace(path) ? m_path : GetRemoteName(path); + + if (!systemIO.DirectoryExists(remotePath)) + throw new FolderMissingException(Strings.FileBackend.FolderMissingError(m_path)); + + var service = SnapshotUtility.CreateNoSnapshot([remotePath], false, false); + + await foreach (var folderEntry in systemIO.EnumerateDirectories(remotePath).ToAsyncEnumerable().WithCancellation(cancellationToken).ConfigureAwait(false)) + yield return new SourceFileEntry(service, folderEntry, true); + + await foreach (var fileEntry in systemIO.EnumerateFiles(remotePath).ToAsyncEnumerable().WithCancellation(cancellationToken).ConfigureAwait(false)) + yield return new SourceFileEntry(service, fileEntry, false); + } + + /// + public Task GetEntryAsync(string path, CancellationToken cancellationToken) + { + var service = SnapshotUtility.CreateNoSnapshot([path], false, false); + var isFolder = service.DirectoryExists(path); + return Task.FromResult(new SourceFileEntry(service, path, isFolder)); + } + + private class SourceFileEntry(ISnapshotService service, string path, bool isFolder) : ISourceFileEntry + { + /// + public bool IsFolder => isFolder; + + /// + public bool IsMetaEntry => false; + /// + public bool IsRootEntry => false; + + /// + public DateTime CreatedUtc => service.GetCreationTimeUtc(path); + + /// + public DateTime LastModificationUtc => service.GetLastWriteTimeUtc(path); + + /// + public string Path => path; + + /// + public long Size => isFolder ? -1 : service.GetFileSize(path); + + /// + public bool IsSymlink => service.IsSymlink(path, Attributes); + + /// + public string SymlinkTarget => service.GetSymlinkTarget(path); + + /// + public FileAttributes Attributes => service.GetAttributes(path); + + /// + public Dictionary MinorMetadata => service.GetMetadata(path, IsSymlink); + + /// + public bool IsBlockDevice => service.IsBlockDevice(path); + + /// + public bool IsCharacterDevice => false; + + /// + public bool IsAlternateStream => false; + + /// + public string HardlinkTargetId => service.HardlinkTargetID(path); + + /// + public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) + => service.EnumerateFilesystemEntries(this).ToAsyncEnumerable(); + + /// + public Task FileExists(string filename, CancellationToken cancellationToken) + => Task.FromResult(service.FileExists(systemIO.PathCombine(Path, filename))); + + /// + public Task OpenMetadataRead(CancellationToken cancellationToken) + => Task.FromResult(new MemoryStream(System.Text.Json.JsonSerializer.SerializeToUtf8Bytes(MinorMetadata))); + + /// + public Task OpenRead(CancellationToken cancellationToken) + => service.OpenReadAsync(path, cancellationToken); + } } } diff --git a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs index 073217e2b..05a1bfb7d 100644 --- a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs +++ b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. @@ -396,7 +396,7 @@ namespace Duplicati.Library.Backend { if (!filename.StartsWith(KEYFILE_URI, StringComparison.OrdinalIgnoreCase)) { - return String.IsNullOrEmpty(password) + return string.IsNullOrEmpty(password) ? new PrivateKeyFile(filename) : new PrivateKeyFile(filename, password); } @@ -409,7 +409,7 @@ namespace Duplicati.Library.Backend ms.Position = 0; - return String.IsNullOrEmpty(password) ? new PrivateKeyFile(ms) : new PrivateKeyFile(ms, password); + return string.IsNullOrEmpty(password) ? new PrivateKeyFile(ms) : new PrivateKeyFile(ms, password); } } catch (Exception ex) diff --git a/Duplicati/Library/Common/IO/VssBackupComponents.cs b/Duplicati/Library/Common/IO/VssBackupComponents.cs index 8b87aa044..d2524c140 100644 --- a/Duplicati/Library/Common/IO/VssBackupComponents.cs +++ b/Duplicati/Library/Common/IO/VssBackupComponents.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -147,6 +147,8 @@ namespace Duplicati.Library.Common.IO } } + public IEnumerable SnapshotLocalPaths => _volumeMap.Keys; + private List GetPathsFromComponent(IVssWMComponent component) { var paths = new List(); diff --git a/Duplicati/Library/DynamicLoader/Duplicati.Library.DynamicLoader.csproj b/Duplicati/Library/DynamicLoader/Duplicati.Library.DynamicLoader.csproj index 512e0f909..0ed1b1e42 100644 --- a/Duplicati/Library/DynamicLoader/Duplicati.Library.DynamicLoader.csproj +++ b/Duplicati/Library/DynamicLoader/Duplicati.Library.DynamicLoader.csproj @@ -17,6 +17,7 @@ + diff --git a/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs b/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs new file mode 100644 index 000000000..5b38ac617 --- /dev/null +++ b/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs @@ -0,0 +1,180 @@ +// Copyright (C) 2024, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using System; +using System.Collections.Generic; +using Duplicati.Library.Interface; +using System.Linq; +using System.Runtime.ExceptionServices; +using Duplicati.Library.SourceProviders; + +namespace Duplicati.Library.DynamicLoader +{ + /// + /// Loads all SourceProviders dynamically and exposes a list of those + /// + public class SourceProviderLoader + { + /// + /// Implementation overrides specific to SourceProviders + /// + private class SourceProviderLoaderSub : DynamicLoader + { + /// + /// Returns the protocol key + /// + /// The item to load the key for + /// The protocol key + protected override string GetInterfaceKey(ISourceProviderModule item) + { + return item.Key; + } + + /// + /// Returns the subfolders searched for SourceProviders + /// + protected override string[] Subfolders => ["SourceProviders"]; + + /// + /// The built-in modules + /// + protected override IEnumerable BuiltInModules => SourceProviderModules.BuiltInSourceProviderModules; + + /// + /// Instanciates a specific SourceProvider, given the url and options + /// + /// The url to create the instance for + /// The options to pass to the instance constructor + /// The instanciated SourceProvider or null if the url is not supported + public ISourceProviderModule GetSourceProvider(string url, Dictionary options) + { + var uri = new Utility.Uri(url); + + LoadInterfaces(); + + var newOpts = new Dictionary(options); + foreach (var key in uri.QueryParameters.AllKeys) + newOpts[key] = uri.QueryParameters[key]; + + lock (m_lock) + { + try + { + if (m_interfaces.ContainsKey(uri.Scheme)) + return (ISourceProviderModule)Activator.CreateInstance(m_interfaces[uri.Scheme].GetType(), url, newOpts); + } + catch (System.Reflection.TargetInvocationException tex) + { + if (tex.InnerException != null) + { + // Unwrap exceptions for nicer display. The ExceptionDispatchInfo class allows us to + // rethrow an exception without changing the stack trace. + ExceptionDispatchInfo.Capture(tex.InnerException).Throw(); + } + + throw; + } + + return null; + } + } + + /// + /// Gets the supported commands for a certain url + /// + /// The url to find commands for + /// The supported commands or null if the url scheme was not supported + public IReadOnlyList GetSupportedCommands(string url) + { + var uri = new Utility.Uri(url); + + LoadInterfaces(); + + // TODO: The loading logic is replicated in the "GetSourceProvider" method, should be refactored + lock (m_lock) + { + ISourceProviderModule b; + if (m_interfaces.TryGetValue(uri.Scheme, out b) && b != null) + return GetSupportedCommandsCached(b).ToList(); + else if (uri.Scheme.EndsWith("s", StringComparison.Ordinal)) + { + var tmpscheme = uri.Scheme.Substring(0, uri.Scheme.Length - 1); + if (m_interfaces.TryGetValue(tmpscheme, out b) && b != null) + return GetSupportedCommandsCached(b).ToList(); + } + + return null; + } + } + } + + /// + /// The static instance used to access SourceProvider information + /// + private static readonly SourceProviderLoaderSub _SourceProviderLoader = new SourceProviderLoaderSub(); + + #region Public static API + + /// + /// Gets a list of loaded SourceProviders, the instances can be used to extract interface information, not used to interact with the SourceProvider. + /// + public static ISourceProviderModule[] SourceProviders { get { return _SourceProviderLoader.Interfaces; } } + + /// + /// Gets a list of keys supported + /// + public static string[] Keys { get { return _SourceProviderLoader.Keys; } } + + /// + /// Gets the supported commands for a given SourceProvider + /// + /// The url to find the commands for + /// The supported commands or null if the url is not supported + public static IReadOnlyList GetSupportedCommands(string url) + { + if (string.IsNullOrEmpty(url)) + throw new ArgumentNullException(nameof(url)); + + return _SourceProviderLoader.GetSupportedCommands(url); + } + + /// + /// Instanciates a specific SourceProvider, given the url and options + /// + /// The url to create the instance for + /// The options to pass to the instance constructor + /// The instanciated SourceProvider or null if the url is not supported + public static ISourceProviderModule GetSourceProvider(string url, Dictionary options) + { + return _SourceProviderLoader.GetSourceProvider(url, options); + } + #endregion + + /// + /// Adds a SourceProvider to the loader + /// + /// The SourceProvider to add + public static void AddSourceProvider(ISourceProviderModule SourceProvider) + { + _SourceProviderLoader.AddModule(SourceProvider); + } + } +} diff --git a/Duplicati/Library/Interface/IFolderEnabledBackend.cs b/Duplicati/Library/Interface/IFolderEnabledBackend.cs new file mode 100644 index 000000000..2ccd3f44c --- /dev/null +++ b/Duplicati/Library/Interface/IFolderEnabledBackend.cs @@ -0,0 +1,53 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace Duplicati.Library.Interface; + +/// +/// An interface a backend may implement if it supports streaming operations. +/// Backends that implement this interface can be throttled and correctly shows +/// the progressbar when transferring data. +/// +public interface IFolderEnabledBackend : IBackend +{ + /// + /// Enumerates a list of files and folders found on the remote location + /// + /// The path to list, empty or null for the root folder + /// The cancellation token + /// The list of files + IAsyncEnumerable ListAsync(string? path, CancellationToken cancellationToken); + + /// + /// Gets a single file or folder entry + /// + /// The path to get + /// The cancellation token + /// The file or folder entry + Task GetEntryAsync(string path, CancellationToken cancellationToken); +} + diff --git a/Duplicati/Library/Interface/ISourceFileEntry.cs b/Duplicati/Library/Interface/ISourceFileEntry.cs new file mode 100644 index 000000000..0f41fb690 --- /dev/null +++ b/Duplicati/Library/Interface/ISourceFileEntry.cs @@ -0,0 +1,126 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +#nullable enable + +namespace Duplicati.Library.Interface; + +/// +/// Interface for an instance of a file +/// +public interface ISourceFileEntry +{ + /// + /// True if the entry represents a folder, false otherwise + /// + bool IsFolder { get; } + /// + /// True if the entry is a meta entry, false otherwise + /// + bool IsMetaEntry { get; } + /// + /// True if the entry is the root entry, false otherwise + /// + bool IsRootEntry { get; } + /// + /// The creation time of the file or folder + /// + DateTime CreatedUtc { get; } + /// + /// The time the file or folder was last modified + /// + DateTime LastModificationUtc { get; } + /// + /// The name of the file or folder + /// + string Path { get; } + /// + /// The size of the file or folder + /// + long Size { get; } + /// + /// True if the file is a symlink, false otherwise + /// + bool IsSymlink { get; } + /// + /// The target of a symlink, if the file is a symlink + /// + string? SymlinkTarget { get; } + /// + /// The entry attributes + /// + FileAttributes Attributes { get; } + /// + /// The minor metadata of the file, should be less than 1KB + /// + Dictionary MinorMetadata { get; } + /// + /// True if the file is a block device, false otherwise + /// + bool IsBlockDevice { get; } + /// + /// True if the file is a character device, false otherwise + /// + bool IsCharacterDevice { get; } + /// + /// True if the files is an alternate stream, false otherwise + /// + bool IsAlternateStream { get; } + /// + /// The hardlink target id, if the file is a hardlink + /// + string? HardlinkTargetId { get; } + + /// + /// Opens the file for reading + /// + /// A token that can be used to cancel the operation + /// A stream that can be read + Task OpenRead(CancellationToken cancellationToken); + + /// + /// Opens the metadata for reading + /// + /// A token that can be used to cancel the operation + /// A stream that can be read + Task OpenMetadataRead(CancellationToken cancellationToken); + + /// + /// Checks if the specified file exists + /// + /// The file to check for existence + /// A token that can be used to cancel the operation + /// true if the file exists, false otherwise + Task FileExists(string filename, CancellationToken cancellationToken); + + /// + /// Returns the contents of the object, if it is a folder + /// + /// A token that can be used to cancel the operation + /// The contents of the folder + IAsyncEnumerable Enumerate(CancellationToken cancellationToken); +} diff --git a/Duplicati/Library/Interface/ISourceProvider.cs b/Duplicati/Library/Interface/ISourceProvider.cs new file mode 100644 index 000000000..4b5725739 --- /dev/null +++ b/Duplicati/Library/Interface/ISourceProvider.cs @@ -0,0 +1,51 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace Duplicati.Library.Interface; + +/// +/// Implements a data source that can be backed up +/// +public interface ISourceProvider : IDisposable +{ + /// + /// Gets the root entries + /// + /// The cancellation token + /// An enumerable of file entries + IAsyncEnumerable Enumerate(CancellationToken cancellationToken); + + /// + /// Gets a specific entry + /// + /// The path to use + /// True if the path is a folder + /// The cancellation token + /// The file entry + Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken); +} diff --git a/Duplicati/Library/Interface/ISourceProviderModule.cs b/Duplicati/Library/Interface/ISourceProviderModule.cs new file mode 100644 index 000000000..469db1fb0 --- /dev/null +++ b/Duplicati/Library/Interface/ISourceProviderModule.cs @@ -0,0 +1,39 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +namespace Duplicati.Library.Interface; + +/// +/// Implements a data source that can be backed up +/// +public interface ISourceProviderModule : ISourceProvider, IDynamicModule +{ + /// + /// Gets the module key + /// + string Key { get; } + /// + /// Gets the display name of the module + /// + string DisplayName { get; } +} diff --git a/Duplicati/Library/Main/Duplicati.Library.Main.csproj b/Duplicati/Library/Main/Duplicati.Library.Main.csproj index dd7f67065..031c46fd1 100644 --- a/Duplicati/Library/Main/Duplicati.Library.Main.csproj +++ b/Duplicati/Library/Main/Duplicati.Library.Main.csproj @@ -19,6 +19,7 @@ + @@ -47,6 +48,7 @@ + diff --git a/Duplicati/Library/Main/Operation/Backup/Channels.cs b/Duplicati/Library/Main/Operation/Backup/Channels.cs index 6a6576f67..b5948e13e 100644 --- a/Duplicati/Library/Main/Operation/Backup/Channels.cs +++ b/Duplicati/Library/Main/Operation/Backup/Channels.cs @@ -20,6 +20,7 @@ // DEALINGS IN THE SOFTWARE. using CoCoL; +using Duplicati.Library.Interface; namespace Duplicati.Library.Main.Operation.Backup { @@ -51,7 +52,7 @@ namespace Duplicati.Library.Main.Operation.Backup /// /// When enumerating the source folders, all discovered paths are sent by the to the /// - public readonly IChannel SourcePaths = ChannelManager.CreateChannel(); + public readonly IChannel SourcePaths = ChannelManager.CreateChannel(); /// /// All progress events are communicated over this channel, to ensure that parallel progress is reported as a if it was sequential /// diff --git a/Duplicati/Library/Main/Operation/Backup/CountFilesHandler.cs b/Duplicati/Library/Main/Operation/Backup/CountFilesHandler.cs index 83f46b264..ffefa0abc 100644 --- a/Duplicati/Library/Main/Operation/Backup/CountFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/Backup/CountFilesHandler.cs @@ -1,42 +1,40 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System.Collections.Generic; using System.Threading.Tasks; using Duplicati.Library.Utility; -using Duplicati.Library.Snapshots; using CoCoL; -using Duplicati.Library.Common.IO; +using Duplicati.Library.Interface; +using Duplicati.Library.Snapshots; namespace Duplicati.Library.Main.Operation.Backup { internal static class CountFilesHandler { public static async Task Run( - IEnumerable sources, - ISnapshotService snapshot, + ISourceProvider sources, UsnJournalService journalService, BackupResults result, Options options, - IFilter sourcefilter, IFilter filter, HashSet blacklistPaths, Common.ITaskReader taskreader, @@ -47,9 +45,8 @@ namespace Duplicati.Library.Main.Operation.Backup using (Logging.Log.StartIsolatingScope(true)) { Channels channels = new(); - var enumeratorTask = Backup.FileEnumerationProcess.Run( - channels, sources, snapshot, journalService, - options.FileAttributeFilter, sourcefilter, filter, + var enumeratorTask = FileEnumerationProcess.Run( + channels, sources, journalService, options.FileAttributeFilter, filter, options.SymlinkPolicy, options.HardlinkPolicy, options.ExcludeEmptyFolders, options.IgnoreFilenames, blacklistPaths, options.ChangedFilelist, taskreader, null, token); @@ -68,14 +65,15 @@ namespace Duplicati.Library.Main.Operation.Backup { while (await taskreader.ProgressRendevouz() && !token.IsCancellationRequested) { - var path = await self.Input.ReadAsync(); + var entry = await self.Input.ReadAsync(); + if (entry.IsFolder) + continue; count++; try { - if (SystemIO.IO_OS.FileExists(path)) - size += snapshot.GetFileSize(path); + size += entry.Size; } catch { diff --git a/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs b/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs index 7dcfead03..b5b5eeb11 100644 --- a/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs +++ b/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs @@ -1,29 +1,28 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using CoCoL; using Duplicati.Library.Main.Operation.Common; using System.Threading.Tasks; -using System.Threading; namespace Duplicati.Library.Main.Operation.Backup { @@ -37,7 +36,7 @@ namespace Duplicati.Library.Main.Operation.Backup /// private static readonly string FILELOGTAG = Logging.Log.LogTagFromType(typeof(FileBlockProcessor)) + ".FileEntry"; - public static Task Run(Channels channels, Snapshots.ISnapshotService snapshot, Options options, BackupDatabase database, BackupStatsCollector stats, ITaskReader taskreader) + public static Task Run(Channels channels, Options options, BackupDatabase database, BackupStatsCollector stats, ITaskReader taskreader) { return AutomationExtensions.RunTask( new @@ -57,7 +56,7 @@ namespace Duplicati.Library.Main.Operation.Backup try { - var hint = options.GetCompressionHintFromFilename(e.Path); + var hint = options.GetCompressionHintFromFilename(e.Entry.Path); var oldHash = e.OldId < 0 ? null : await database.GetFileHashAsync(e.OldId); StreamProcessResult filestreamdata; @@ -73,15 +72,15 @@ namespace Duplicati.Library.Main.Operation.Backup if (res.Item1) return res.Item2; - Logging.Log.WriteWarningMessage(FILELOGTAG, "UnexpectedMetadataLookup", null, "Metadata was reported as not changed, but still requires being added?\nHash: {0}, Length: {1}, ID: {2}, Path: {3}", e.MetaHashAndSize.FileHash, e.MetaHashAndSize.Blob.Length, res.Item2, e.Path); + Logging.Log.WriteWarningMessage(FILELOGTAG, "UnexpectedMetadataLookup", null, "Metadata was reported as not changed, but still requires being added?\nHash: {0}, Length: {1}, ID: {2}, Path: {3}", e.MetaHashAndSize.FileHash, e.MetaHashAndSize.Blob.Length, res.Item2, e.Entry.Path); e.MetadataChanged = true; } - return (await MetadataPreProcess.AddMetadataToOutputAsync(e.Path, e.MetaHashAndSize, database, self.StreamBlockChannel)).Item2; + return (await MetadataPreProcess.AddMetadataToOutputAsync(e.Entry.Path, e.MetaHashAndSize, database, self.StreamBlockChannel)).Item2; }); - using (var fs = snapshot.OpenRead(e.Path)) - filestreamdata = await StreamBlock.ProcessStream(self.StreamBlockChannel, e.Path, fs, false, hint); + using (var fs = await e.Entry.OpenRead(taskreader.ProgressToken)) + filestreamdata = await StreamBlock.ProcessStream(self.StreamBlockChannel, e.Entry.Path, fs, false, hint); await stats.AddOpenedFile(filestreamdata.Streamlength); @@ -92,36 +91,36 @@ namespace Duplicati.Library.Main.Operation.Backup if (oldHash != filekey) { if (oldHash == null) - Logging.Log.WriteVerboseMessage(FILELOGTAG, "NewFile", "New file {0}", e.Path); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "NewFile", "New file {0}", e.Entry.Path); else - Logging.Log.WriteVerboseMessage(FILELOGTAG, "ChangedFile", "File has changed {0}", e.Path); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "ChangedFile", "File has changed {0}", e.Entry.Path); if (e.OldId < 0) { await stats.AddAddedFile(filesize); if (options.Dryrun) - Logging.Log.WriteVerboseMessage(FILELOGTAG, "WouldAddNewFile", "Would add new file {0}, size {1}", e.Path, Library.Utility.Utility.FormatSizeString(filesize)); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "WouldAddNewFile", "Would add new file {0}, size {1}", e.Entry.Path, Library.Utility.Utility.FormatSizeString(filesize)); } else { await stats.AddModifiedFile(filesize); if (options.Dryrun) - Logging.Log.WriteVerboseMessage(FILELOGTAG, "WouldAddChangedFile", "Would add changed file {0}, size {1}", e.Path, Library.Utility.Utility.FormatSizeString(filesize)); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "WouldAddChangedFile", "Would add changed file {0}, size {1}", e.Entry.Path, Library.Utility.Utility.FormatSizeString(filesize)); } await database.AddFileAsync(e.PathPrefixID, e.Filename, e.LastWrite, filestreamdata.Blocksetid, metadataid); } else if (e.MetadataChanged) { - Logging.Log.WriteVerboseMessage(FILELOGTAG, "FileMetadataChanged", "File has only metadata changes {0}", e.Path); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "FileMetadataChanged", "File has only metadata changes {0}", e.Entry.Path); await database.AddFileAsync(e.PathPrefixID, e.Filename, e.LastWrite, filestreamdata.Blocksetid, metadataid); } else /*if (e.OldId >= 0)*/ { // When we write the file to output, update the last modified time - Logging.Log.WriteVerboseMessage(FILELOGTAG, "NoFileChanges", "File has not changed {0}", e.Path); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "NoFileChanges", "File has not changed {0}", e.Entry.Path); try { @@ -129,7 +128,7 @@ namespace Duplicati.Library.Main.Operation.Backup } catch (Exception ex) { - Logging.Log.WriteWarningMessage(FILELOGTAG, "FailedToAddFile", ex, "Failed while attempting to add unmodified file to database: {0}", e.Path); + Logging.Log.WriteWarningMessage(FILELOGTAG, "FailedToAddFile", ex, "Failed while attempting to add unmodified file to database: {0}", e.Entry.Path); } } } @@ -138,7 +137,7 @@ namespace Duplicati.Library.Main.Operation.Backup if (ex.IsRetiredException()) return; else - Logging.Log.WriteWarningMessage(FILELOGTAG, "PathProcessingFailed", ex, "Failed to process path: {0}", e.Path); + Logging.Log.WriteWarningMessage(FILELOGTAG, "PathProcessingFailed", ex, "Failed to process path: {0}", e.Entry.Path); } } }); diff --git a/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs b/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs index 7b7478a82..7283da48e 100644 --- a/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs @@ -27,9 +27,10 @@ using System.Collections.Generic; using System.Linq; using System.Threading; using Duplicati.Library.Main.Operation.Common; -using Duplicati.Library.Snapshots; -using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; +using System.Runtime.CompilerServices; +using Duplicati.Library.Snapshots; +using Duplicati.Library.SourceProvider; namespace Duplicati.Library.Main.Operation.Backup { @@ -47,18 +48,16 @@ namespace Duplicati.Library.Main.Operation.Backup public static Task Run( Channels channels, - IEnumerable sources, - ISnapshotService snapshot, + ISourceProvider sourceProvider, UsnJournalService journalService, - FileAttributes fileAttributes, - Library.Utility.IFilter sourcefilter, + FileAttributes fileAttributeFilter, Library.Utility.IFilter emitfilter, Options.SymlinkStrategy symlinkPolicy, Options.HardlinkStrategy hardlinkPolicy, bool excludeemptyfolders, string[] ignorenames, HashSet blacklistPaths, - string[] changedfilelist, + IEnumerable changedfilelist, ITaskReader taskreader, Action onStopRequested, CancellationToken token) @@ -73,9 +72,19 @@ namespace Duplicati.Library.Main.Operation.Backup { if (!token.IsCancellationRequested) { + // The hardlink map tracks the hardlink targets we have seen + // and avoid multiple processing of the same contents var hardlinkmap = new Dictionary(); - var mixinqueue = new Queue(); - Duplicati.Library.Utility.IFilter enumeratefilter = emitfilter; + + // The mixin queue is used to store symlinks that should be processed + // The symlinks are emitted during the enumeration process when they are found + var mixinqueue = new Queue(); + + // The enumeration filter is used to determine what paths to + // recurse into. If the emit filter only has includes, + // the enumeration filter will also include all folders, + // as nothing will match otherwise + var enumeratefilter = emitfilter; Library.Utility.FilterExpression.AnalyzeFilters(emitfilter, out var includes, out var excludes); if (includes && !excludes) @@ -86,58 +95,59 @@ namespace Duplicati.Library.Main.Operation.Backup ignorenames = null; // If we have a specific list, use that instead of enumerating the filesystem - IEnumerable worklist; - if (changedfilelist != null && changedfilelist.Length > 0) + async IAsyncEnumerable ExpandSources(IEnumerable list) { - worklist = changedfilelist.Where(x => - { - var fa = FileAttributes.Normal; - try - { - fa = snapshot.GetAttributes(x); - } - catch - { - } + foreach (var s in list) + yield return await sourceProvider.GetEntry(s, false, token).ConfigureAwait(false); + } - if (token.IsCancellationRequested) - { - return false; - } + IAsyncEnumerable worklist; + if (changedfilelist != null && changedfilelist.Any()) + { + worklist = ExpandSources(changedfilelist).WhereAwait(x => + SourceFileEntryFilter(x, blacklistPaths, hardlinkPolicy, symlinkPolicy, hardlinkmap, fileAttributeFilter, enumeratefilter, ignorenames, mixinqueue, token)); + } + else if (journalService != null) + { + if (!OperatingSystem.IsWindows()) + throw new NotSupportedException("USN is only supported on Windows"); - return AttributeFilter(x, fa, snapshot, sourcefilter, blacklistPaths, hardlinkPolicy, symlinkPolicy, hardlinkmap, fileAttributes, enumeratefilter, ignorenames, mixinqueue); - }); + var fileProviders = (sourceProvider is Combiner c ? c.Providers.AsEnumerable() : [sourceProvider]) + .OfType() + .ToList(); + + if (fileProviders.Count <= 0) + throw new InvalidOperationException("No file providers found, but USN was enabled?"); + if (fileProviders.Count > 1) + throw new InvalidOperationException("Multiple file providers found, but USN only supports one"); + + worklist = RecursiveFilterHelper.GetModifiedSources( + ExpandSources(journalService.VolumeDataList.SelectMany(x => x.Folders.Concat(x.Files))), + fileProviders.First().SnapshotService, + x => SourceFileEntryFilter(x, blacklistPaths, hardlinkPolicy, symlinkPolicy, hardlinkmap, fileAttributeFilter, enumeratefilter, ignorenames, mixinqueue, token), + token + ); } else { - Library.Utility.Utility.EnumerationFilterDelegate attributeFilter = (root, path, attr) => - AttributeFilter(path, attr, snapshot, sourcefilter, blacklistPaths, hardlinkPolicy, symlinkPolicy, hardlinkmap, fileAttributes, enumeratefilter, ignorenames, mixinqueue); - - if (journalService != null) - { - if (!OperatingSystem.IsWindows()) - throw new UserInformationException("USN journal is only supported on Windows", "USNJournalNotSupported"); - // filter sources using USN journal, to obtain a sub-set of files / folders that may have been modified - sources = journalService.GetModifiedSources(attributeFilter); - } - - worklist = snapshot.EnumerateFilesAndFolders(sources, attributeFilter, (rootpath, errorpath, ex) => - { - Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "FileAccessError", ex, "Error reported while accessing file: {0}", errorpath); - }).Distinct(Library.Utility.Utility.IsFSCaseSensitive ? StringComparer.Ordinal : StringComparer.OrdinalIgnoreCase); + worklist = RecurseEntries(sourceProvider.Enumerate(token), + x => SourceFileEntryFilter(x, blacklistPaths, hardlinkPolicy, symlinkPolicy, hardlinkmap, fileAttributeFilter, enumeratefilter, ignorenames, mixinqueue, token), + token + ); } if (token.IsCancellationRequested) - { return; - } - var source = ExpandWorkList(worklist, mixinqueue, emitfilter, enumeratefilter); + var source = ExpandWorkList(worklist, mixinqueue, emitfilter, enumeratefilter, token); + // TODO: There was a call to DistinctBy here, but this would cause all paths to be stored in memory + //.DistinctBy(x => x.Path, Library.Utility.Utility.IsFSCaseSensitive ? StringComparer.Ordinal : StringComparer.OrdinalIgnoreCase); + if (excludeemptyfolders) - source = ExcludeEmptyFolders(source); + source = ExcludeEmptyFolders(source, token); // Process each path, and dequeue the mixins with symlinks as we go - foreach (var s in source) + await foreach (var s in source.WithCancellation(token).ConfigureAwait(false)) { #if DEBUG // For testing purposes, we need exact control @@ -146,7 +156,7 @@ namespace Duplicati.Library.Main.Operation.Backup // if the process is the real file enumeration process // because the counter processe does not have a callback if (onStopRequested != null) - taskreader.TestMethodCallback?.Invoke(s); + taskreader.TestMethodCallback?.Invoke(s.Path); #endif // Stop if requested if (token.IsCancellationRequested || !await taskreader.ProgressRendevouz().ConfigureAwait(false)) @@ -167,9 +177,9 @@ namespace Duplicati.Library.Main.Operation.Backup private class DirectoryStackEntry { /// - /// The path for the folder + /// The item being tracked /// - public string Path; + public ISourceFileEntry Item; /// /// A flag indicating if any items are found in this folder /// @@ -182,17 +192,17 @@ namespace Duplicati.Library.Main.Operation.Backup /// /// The list without empty folders. /// The list with potential empty folders. - private static IEnumerable ExcludeEmptyFolders(IEnumerable source) + private static async IAsyncEnumerable ExcludeEmptyFolders(IAsyncEnumerable source, [EnumeratorCancellation] CancellationToken cancellationToken) { var pathstack = new Stack(); - foreach (var s in source) + await foreach (var s in source.WithCancellation(cancellationToken).ConfigureAwait(false)) { // Keep track of directories - var isDirectory = s[s.Length - 1] == System.IO.Path.DirectorySeparatorChar; + var isDirectory = s.Path[s.Path.Length - 1] == System.IO.Path.DirectorySeparatorChar; if (isDirectory) { - while (pathstack.Count > 0 && !s.StartsWith(pathstack.Peek().Path, Library.Utility.Utility.ClientFilenameStringComparison)) + while (pathstack.Count > 0 && !s.Path.StartsWith(pathstack.Peek().Item.Path, Library.Utility.Utility.ClientFilenameStringComparison)) { var e = pathstack.Pop(); if (e.AnyEntries || pathstack.Count == 0) @@ -201,15 +211,15 @@ namespace Duplicati.Library.Main.Operation.Backup if (pathstack.Count > 0) pathstack.Peek().AnyEntries = true; - yield return e.Path; + yield return e.Item; } else - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingEmptyFolder", "Excluding empty folder {0}", e.Path); + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingEmptyFolder", "Excluding empty folder {0}", e.Item); } - if (pathstack.Count == 0 || s.StartsWith(pathstack.Peek().Path, Library.Utility.Utility.ClientFilenameStringComparison)) + if (pathstack.Count == 0 || s.Path.StartsWith(pathstack.Peek().Item.Path, Library.Utility.Utility.ClientFilenameStringComparison)) { - pathstack.Push(new DirectoryStackEntry() { Path = s }); + pathstack.Push(new DirectoryStackEntry() { Item = s }); continue; } } @@ -231,7 +241,46 @@ namespace Duplicati.Library.Main.Operation.Backup if (pathstack.Count > 0) pathstack.Peek().AnyEntries = true; - yield return e.Path; + yield return e.Item; + } + } + } + + /// + /// Performs recursive traversal of the sources + /// + /// The entries to recurse + /// The filter to apply + /// + private static async IAsyncEnumerable RecurseEntries(IAsyncEnumerable entries, Func> filter, [EnumeratorCancellation] CancellationToken cancellationToken) + { + var work = new Stack(); + + await foreach (var e in entries.WithCancellation(cancellationToken).ConfigureAwait(false)) + if (await filter(e).ConfigureAwait(false)) + work.Push(e); + + while (work.Count > 0) + { + var e = work.Pop(); + + // Process meta entry contents, but don't emit them for processing + if (!e.IsMetaEntry) + yield return e; + + if (e.IsFolder) + { + try + { + // We only filter new items, as we assume the input is already filtered + await foreach (var r in e.Enumerate(cancellationToken).ConfigureAwait(false)) + if (await filter(r).ConfigureAwait(false)) + work.Push(r); + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingErrorEnumerate", ex, "Failed to enumerate path: {0}", e.Path); + } } } } @@ -244,16 +293,18 @@ namespace Duplicati.Library.Main.Operation.Backup /// The mix in queue. /// The emitfilter. /// The enumeratefilter. - private static IEnumerable ExpandWorkList(IEnumerable worklist, Queue mixinqueue, Library.Utility.IFilter emitfilter, Library.Utility.IFilter enumeratefilter) + private static async IAsyncEnumerable ExpandWorkList(IAsyncEnumerable worklist, Queue mixinqueue, Library.Utility.IFilter emitfilter, Library.Utility.IFilter enumeratefilter, [EnumeratorCancellation] CancellationToken cancellationToken) { // Process each path, and dequeue the mixins with symlinks as we go - foreach (var s in worklist) + await foreach (var s in worklist.WithCancellation(cancellationToken).ConfigureAwait(false)) { while (mixinqueue.Count > 0) yield return mixinqueue.Dequeue(); - Library.Utility.IFilter m; - if (emitfilter != enumeratefilter && !Library.Utility.FilterExpression.Matches(emitfilter, s, out m)) + // If there are only includes in the filter, check if the item is in the original filter + // Since the enumerate filter also includes all folders, we need to ensure we do not emit + // any entries that are filtered explicitly by the user + if (emitfilter != enumeratefilter && !Library.Utility.FilterExpression.Matches(emitfilter, s.Path, out var _)) continue; yield return s; @@ -265,65 +316,92 @@ namespace Duplicati.Library.Main.Operation.Backup } /// - /// Plugin filter for enumerating a list of files. + /// Performs a pre-filter on the source entry to see if it should be included in the backup /// - /// True if the path should be returned, false otherwise. - /// The current path. - /// The file or folder attributes. - /// The snapshot service. - /// The source filter. + /// The entry to evaluate. /// The blacklist paths. - /// The hardlink policy. - /// The symlink policy. - /// The hardlink map. - /// The file attributes to exclude. - /// The enumerate filter. - /// The ignore names. - /// The mixin queue. - private static bool AttributeFilter(string path, FileAttributes attributes, Snapshots.ISnapshotService snapshot, Library.Utility.IFilter sourcefilter, HashSet blacklistPaths, Options.HardlinkStrategy hardlinkPolicy, Options.SymlinkStrategy symlinkPolicy, Dictionary hardlinkmap, FileAttributes fileAttributes, Duplicati.Library.Utility.IFilter enumeratefilter, string[] ignorenames, Queue mixinqueue) + /// True if the path should be returned, false otherwise. + private static bool PreFilterSourceEntry(ISourceFileEntry entry, HashSet blacklistPaths) { + // Don't filter meta stuff + if (entry.IsMetaEntry) + return true; + // Exclude any blacklisted paths - if (blacklistPaths.Contains(path)) + if (blacklistPaths.Contains(entry.Path)) { - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingBlacklistedPath", "Excluding blacklisted path: {0}", path); + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingBlacklistedPath", "Excluding blacklisted path: {0}", entry.Path); return false; } // Exclude block devices try { - if (snapshot.IsBlockDevice(path)) + if (entry.IsBlockDevice) { - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingBlockDevice", "Excluding block device: {0}", path); + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingBlockDevice", "Excluding block device: {0}", entry.Path); return false; } } catch (Exception ex) { - Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingErrorBlockDevice", ex, "Failed to process path: {0}", path); + Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingErrorBlockDevice", ex, "Failed to process path: {0}", entry.Path); return false; } - // Check if we explicitly include this entry - Duplicati.Library.Utility.IFilter sourcematch; - bool sourcematches; - if (sourcefilter.Matches(path, out sourcematches, out sourcematch) && sourcematches) + // Exclude character devices + try { - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "IncludingSourcePath", "Including source path: {0}", path); - return true; + if (entry.IsCharacterDevice) + { + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingCharacterDevice", "Excluding character device: {0}", entry.Path); + return false; + } } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingErrorCharacterDevice", ex, "Failed to process path: {0}", entry.Path); + return false; + } + + return true; + } + + /// + /// Evaluates a single entry for inclusion in the backup + /// + /// The current entry. + /// The snapshot service. + /// The blacklist paths. + /// The hardlink policy. + /// The symlink policy. + /// The hardlink map. + /// The file attributes to exclude. + /// The enumerate filter. + /// The ignore names. + /// The mixin queue. + /// True if the path should be returned, false otherwise. + private static async ValueTask SourceFileEntryFilter(ISourceFileEntry entry, HashSet blacklistPaths, Options.HardlinkStrategy hardlinkPolicy, Options.SymlinkStrategy symlinkPolicy, Dictionary hardlinkmap, FileAttributes fileAttributeFilter, Duplicati.Library.Utility.IFilter enumeratefilter, string[] ignorenames, Queue mixinqueue, CancellationToken cancellationToken) + { + // Do the course pre-filtering first + if (!PreFilterSourceEntry(entry, blacklistPaths)) + return false; + + // Never exclude the root entries + if (entry.IsRootEntry) + return true; // If we have a hardlink strategy, obey it if (hardlinkPolicy != Options.HardlinkStrategy.All) { try { - var id = snapshot.HardlinkTargetID(path); + var id = entry.HardlinkTargetId; if (id != null) { if (hardlinkPolicy == Options.HardlinkStrategy.None) { - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingHardlinkByPolicy", "Excluding hardlink: {0} ({1})", path, id); + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingHardlinkByPolicy", "Excluding hardlink: {0} ({1})", entry.Path, id); return false; } else if (hardlinkPolicy == Options.HardlinkStrategy.First) @@ -331,99 +409,116 @@ namespace Duplicati.Library.Main.Operation.Backup string prevPath; if (hardlinkmap.TryGetValue(id, out prevPath)) { - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingDuplicateHardlink", "Excluding hardlink ({1}) for: {0}, previous hardlink: {2}", path, id, prevPath); + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingDuplicateHardlink", "Excluding hardlink ({1}) for: {0}, previous hardlink: {2}", entry.Path, id, prevPath); return false; } else { - hardlinkmap.Add(id, path); + hardlinkmap.Add(id, entry.Path); } } } } catch (Exception ex) { - Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingErrorHardLink", ex, "Failed to process path: {0}", path); + Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingErrorHardLink", ex, "Failed to process path: {0}", entry.Path); return false; } } - if (ignorenames != null && (attributes & FileAttributes.Directory) != 0) + // Check if there is an ignore marker file + if (ignorenames != null && entry.IsFolder) { try { foreach (var n in ignorenames) { - var ignorepath = SystemIO.IO_OS.PathCombine(path, n); - if (snapshot.FileExists(ignorepath)) + if (await entry.FileExists(n, cancellationToken).ConfigureAwait(false)) { - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingPathDueToIgnoreFile", "Excluding path because ignore file was found: {0}", ignorepath); + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingPathDueToIgnoreFile", "Excluding path because ignore file {0} was found in: {1}", n, entry.Path); return false; } } } catch (Exception ex) { - Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingErrorIgnoreFile", ex, "Failed to process path: {0}", path); + Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingErrorIgnoreFile", ex, "Failed to process path: {0}", entry.Path); } } - // If we exclude files based on attributes, filter that - if ((fileAttributes & attributes) != 0) + // Setup some basic processing attributes + var attributes = entry.IsFolder + ? FileAttributes.Directory + : FileAttributes.Normal; + + try { - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingPathFromAttributes", "Excluding path due to attribute filter: {0}", path); + attributes = entry.Attributes; + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingErrorAttributes", ex, "Failed to process path, using default attributes: {0}", entry.Path); + } + + // If we exclude files based on attributes, filter that + if ((fileAttributeFilter & attributes) != 0) + { + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingPathFromAttributes", "Excluding path due to attribute filter: {0}", entry.Path); return false; } // Then check if the filename is not explicitly excluded by a filter - Library.Utility.IFilter match; var filtermatch = false; - if (!Library.Utility.FilterExpression.Matches(enumeratefilter, path, out match)) + if (!Library.Utility.FilterExpression.Matches(enumeratefilter, entry.Path, out var match)) { - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingPathFromFilter", "Excluding path due to filter: {0} => {1}", path, match == null ? "null" : match.ToString()); + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingPathFromFilter", "Excluding path due to filter: {0} => {1}", entry.Path, match == null ? "null" : match.ToString()); return false; } else if (match != null) { filtermatch = true; - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "IncludingPathFromFilter", "Including path due to filter: {0} => {1}", path, match.ToString()); + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "IncludingPathFromFilter", "Including path due to filter: {0} => {1}", entry.Path, match.ToString()); } // If the file is a symlink, apply special handling - var isSymlink = snapshot.IsSymlink(path, attributes); string symlinkTarget = null; - if (isSymlink) - try { symlinkTarget = snapshot.GetSymlinkTarget(path); } - catch (Exception ex) { Logging.Log.WriteExplicitMessage(FILTER_LOGTAG, "SymlinkTargetReadError", ex, "Failed to read symlink target for path: {0}", path); } + try + { + symlinkTarget = entry.SymlinkTarget; + } + catch (Exception ex) + { + Logging.Log.WriteExplicitMessage(FILTER_LOGTAG, "SymlinkTargetReadError", ex, "Failed to read symlink target for path: {0}", entry.Path); + } - if (isSymlink) + if (symlinkTarget != null) { if (!string.IsNullOrWhiteSpace(symlinkTarget)) { if (symlinkPolicy == Options.SymlinkStrategy.Ignore) { - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludeSymlink", "Excluding symlink: {0}", path); + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludeSymlink", "Excluding symlink: {0}", entry.Path); return false; } if (symlinkPolicy == Options.SymlinkStrategy.Store) { - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "StoreSymlink", "Storing symlink: {0}", path); + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "StoreSymlink", "Storing symlink: {0}", entry.Path); // We return false because we do not want to recurse into the path, // but we add the symlink to the mixin so we process the symlink itself - mixinqueue.Enqueue(path); + mixinqueue.Enqueue(entry); return false; } } else { - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "FollowingEmptySymlink", "Treating empty symlink as regular path {0}", path); + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "FollowingEmptySymlink", "Treating empty symlink as regular path {0}", entry.Path); } } if (!filtermatch) - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "IncludingPath", "Including path as no filters matched: {0}", path); + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "IncludingPath", "Including path as no filters matched: {0}", entry.Path); // All the way through, yes! return true; diff --git a/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs b/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs index 4276fdefc..f65398e91 100644 --- a/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -40,7 +40,7 @@ namespace Duplicati.Library.Main.Operation.Backup /// private static readonly string FILELOGTAG = Logging.Log.LogTagFromType(typeof(FilePreFilterProcess)) + ".FileEntry"; - public static Task Run(Channels channels, Snapshots.ISnapshotService snapshot, Options options, BackupStatsCollector stats, BackupDatabase database) + public static Task Run(Channels channels, Options options, BackupStatsCollector stats, BackupDatabase database) { return AutomationExtensions.RunTask( new @@ -73,11 +73,11 @@ namespace Duplicati.Library.Main.Operation.Backup long filestatsize = -1; try { - filestatsize = snapshot.GetFileSize(e.Path); + filestatsize = e.Entry.Size; } - catch(Exception ex) + catch (Exception ex) { - Logging.Log.WriteExplicitMessage(FILELOGTAG, "FailedToReadSize", ex, "Failed to read size of file: {0}", e.Path); + Logging.Log.WriteExplicitMessage(FILELOGTAG, "FailedToReadSize", ex, "Failed to read size of file: {0}", e.Entry.Path); } await stats.AddExaminedFile(filestatsize); @@ -86,8 +86,8 @@ namespace Duplicati.Library.Main.Operation.Backup var tooLargeFile = SKIPFILESLARGERTHAN != 0 && filestatsize >= 0 && filestatsize > SKIPFILESLARGERTHAN; if (tooLargeFile) { - Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckTooLarge", "Skipped checking file, because the size exceeds limit {0}", e.Path); - await self.ProgressChannel.WriteAsync(new ProgressEvent() { Filepath = e.Path, Length = filestatsize, Type = EventType.FileSkipped }); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckTooLarge", "Skipped checking file, because the size exceeds limit {0}", e.Entry.Path); + await self.ProgressChannel.WriteAsync(new ProgressEvent() { Filepath = e.Entry.Path, Length = filestatsize, Type = EventType.FileSkipped }); continue; } @@ -102,7 +102,7 @@ namespace Duplicati.Library.Main.Operation.Backup // and we only check for timestamp changes if (CHECKFILETIMEONLY && !timestampChanged && !isNewFile) { - Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoTimestampChange", "Skipped checking file, because timestamp was not updated {0}", e.Path); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoTimestampChange", "Skipped checking file, because timestamp was not updated {0}", e.Entry.Path); try { await database.AddUnmodifiedAsync(e.OldId, e.LastWrite); @@ -111,9 +111,9 @@ namespace Duplicati.Library.Main.Operation.Backup { if (ex.IsRetiredException()) throw; - Logging.Log.WriteWarningMessage(FILELOGTAG, "FailedToAddFile", ex, "Failed while attempting to add unmodified file to database: {0}", e.Path); + Logging.Log.WriteWarningMessage(FILELOGTAG, "FailedToAddFile", ex, "Failed while attempting to add unmodified file to database: {0}", e.Entry.Path); } - await self.ProgressChannel.WriteAsync(new ProgressEvent() { Filepath = e.Path, Length = filestatsize, Type = EventType.FileSkipped }); + await self.ProgressChannel.WriteAsync(new ProgressEvent() { Filepath = e.Entry.Path, Length = filestatsize, Type = EventType.FileSkipped }); continue; } @@ -130,28 +130,28 @@ namespace Duplicati.Library.Main.Operation.Backup } // Compute current metadata - e.MetaHashAndSize = SKIPMETADATA ? EMPTY_METADATA : Utility.WrapMetadata(MetadataGenerator.GenerateMetadata(e.Path, e.Attributes, options, snapshot), options); + e.MetaHashAndSize = SKIPMETADATA ? EMPTY_METADATA : Utility.WrapMetadata(MetadataGenerator.GenerateMetadata(e.Entry, e.Attributes, options), options); e.MetadataChanged = !SKIPMETADATA && (e.MetaHashAndSize.Blob.Length != e.OldMetaSize || e.MetaHashAndSize.FileHash != e.OldMetaHash); // Check if the file is new, or something indicates a change var filesizeChanged = filestatsize < 0 || e.LastFileSize < 0 || filestatsize != e.LastFileSize; if (isNewFile || timestampChanged || filesizeChanged || e.MetadataChanged) { - Logging.Log.WriteVerboseMessage(FILELOGTAG, "CheckFileForChanges", "Checking file for changes {0}, new: {1}, timestamp changed: {2}, size changed: {3}, metadatachanged: {4}, {5} vs {6}", e.Path, isNewFile, timestampChanged, filesizeChanged, e.MetadataChanged, e.LastWrite, e.OldModified); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "CheckFileForChanges", "Checking file for changes {0}, new: {1}, timestamp changed: {2}, size changed: {3}, metadatachanged: {4}, {5} vs {6}", e.Entry.Path, isNewFile, timestampChanged, filesizeChanged, e.MetadataChanged, e.LastWrite, e.OldModified); await self.Output.WriteAsync(e); } else { - Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoMetadataChange", "Skipped checking file, because no metadata was updated {0}", e.Path); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoMetadataChange", "Skipped checking file, because no metadata was updated {0}", e.Entry.Path); try { await database.AddUnmodifiedAsync(e.OldId, e.LastWrite); } catch (Exception ex) { - Logging.Log.WriteWarningMessage(FILELOGTAG, "FailedToAddFile", ex, "Failed while attempting to add unmodified file to database: {0}", e.Path); + Logging.Log.WriteWarningMessage(FILELOGTAG, "FailedToAddFile", ex, "Failed while attempting to add unmodified file to database: {0}", e.Entry.Path); } - await self.ProgressChannel.WriteAsync(new ProgressEvent() { Filepath = e.Path, Length = filestatsize, Type = EventType.FileSkipped }); + await self.ProgressChannel.WriteAsync(new ProgressEvent() { Filepath = e.Entry.Path, Length = filestatsize, Type = EventType.FileSkipped }); } } }); diff --git a/Duplicati/Library/Main/Operation/Backup/MetadataGenerator.cs b/Duplicati/Library/Main/Operation/Backup/MetadataGenerator.cs index adb32616c..02c6f75c2 100644 --- a/Duplicati/Library/Main/Operation/Backup/MetadataGenerator.cs +++ b/Duplicati/Library/Main/Operation/Backup/MetadataGenerator.cs @@ -1,30 +1,27 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; -using System.Threading.Tasks; using System.Collections.Generic; -using CoCoL; -using Duplicati.Library.Main.Operation.Common; -using Duplicati.Library.Snapshots; +using Duplicati.Library.Interface; namespace Duplicati.Library.Main.Operation.Backup { @@ -35,7 +32,7 @@ namespace Duplicati.Library.Main.Operation.Backup { private static readonly string METALOGTAG = Logging.Log.LogTagFromType(typeof(MetadataGenerator)) + ".Metadata"; - public static Dictionary GenerateMetadata(string path, System.IO.FileAttributes attributes, Options options, Snapshots.ISnapshotService snapshot) + public static Dictionary GenerateMetadata(ISourceFileEntry entry, System.IO.FileAttributes attributes, Options options) { try { @@ -43,7 +40,7 @@ namespace Duplicati.Library.Main.Operation.Backup if (!options.SkipMetadata) { - metadata = snapshot.GetMetadata(path, snapshot.IsSymlink(path, attributes), options.SymlinkPolicy == Options.SymlinkStrategy.Follow); + metadata = entry.MinorMetadata; if (metadata == null) metadata = new Dictionary(); @@ -54,11 +51,11 @@ namespace Duplicati.Library.Main.Operation.Backup { try { - metadata["CoreLastWritetime"] = snapshot.GetLastWriteTimeUtc(path).Ticks.ToString(); + metadata["CoreLastWritetime"] = entry.LastModificationUtc.Ticks.ToString(); } catch (Exception ex) { - Logging.Log.WriteWarningMessage(METALOGTAG, "TimestampReadFailed", ex, "Failed to read timestamp on \"{0}\"", path); + Logging.Log.WriteWarningMessage(METALOGTAG, "TimestampReadFailed", ex, "Failed to read timestamp on \"{0}\"", entry.Path); } } @@ -66,11 +63,11 @@ namespace Duplicati.Library.Main.Operation.Backup { try { - metadata["CoreCreatetime"] = snapshot.GetCreationTimeUtc(path).Ticks.ToString(); + metadata["CoreCreatetime"] = entry.CreatedUtc.Ticks.ToString(); } catch (Exception ex) { - Logging.Log.WriteWarningMessage(METALOGTAG, "TimestampReadFailed", ex, "Failed to read timestamp on \"{0}\"", path); + Logging.Log.WriteWarningMessage(METALOGTAG, "TimestampReadFailed", ex, "Failed to read timestamp on \"{0}\"", entry.Path); } } } @@ -81,9 +78,9 @@ namespace Duplicati.Library.Main.Operation.Backup return metadata; } - catch(Exception ex) + catch (Exception ex) { - Logging.Log.WriteWarningMessage(METALOGTAG, "MetadataProcessFailed", ex, "Failed to process metadata for \"{0}\", storing empty metadata", path); + Logging.Log.WriteWarningMessage(METALOGTAG, "MetadataProcessFailed", ex, "Failed to process metadata for \"{0}\", storing empty metadata", entry.Path); return new Dictionary(); } } diff --git a/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs b/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs index b8554d45f..1642d1b8a 100644 --- a/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -24,11 +24,8 @@ using CoCoL; using System.Threading.Tasks; using System.IO; using System.Collections.Generic; -using System.Security.Principal; -using System.Threading; using Duplicati.Library.Interface; using Duplicati.Library.Main.Operation.Common; -using Duplicati.Library.Snapshots; namespace Duplicati.Library.Main.Operation.Backup { @@ -43,7 +40,7 @@ namespace Duplicati.Library.Main.Operation.Backup public class FileEntry { // From input - public string Path; + public ISourceFileEntry Entry; // Split public long PathPrefixID; @@ -65,7 +62,7 @@ namespace Duplicati.Library.Main.Operation.Backup public bool MetadataChanged; } - public static Task Run(Channels channels, Snapshots.ISnapshotService snapshot, Options options, BackupDatabase database, long lastfilesetid, ITaskReader taskReader) + public static Task Run(Channels channels, Options options, BackupDatabase database, long lastfilesetid, ITaskReader taskReader) { return AutomationExtensions.RunTask(new { @@ -84,37 +81,40 @@ namespace Duplicati.Library.Main.Operation.Backup while (true) { - var path = await self.Input.ReadAsync(); + var entry = await self.Input.ReadAsync(); // We ignore the stop signal, but not the pause and terminate await taskReader.ProgressRendevouz().ConfigureAwait(false); var lastwrite = new DateTime(0, DateTimeKind.Utc); - var attributes = default(FileAttributes); + var attributes = entry.IsFolder + ? FileAttributes.Directory + : FileAttributes.Normal; + try { - lastwrite = snapshot.GetLastWriteTimeUtc(path); + lastwrite = entry.LastModificationUtc; } catch (Exception ex) { - Logging.Log.WriteWarningMessage(FILELOGTAG, "TimestampReadFailed", ex, "Failed to read timestamp on \"{0}\"", path); + Logging.Log.WriteWarningMessage(FILELOGTAG, "TimestampReadFailed", ex, "Failed to read timestamp on \"{0}\"", entry.Path); } try { - attributes = snapshot.GetAttributes(path); + attributes = entry.Attributes; } catch (Exception ex) { - Logging.Log.WriteVerboseMessage(FILELOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", path, ex.Message); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", entry.Path, ex.Message); } // If we only have metadata, stop here - if (await ProcessMetadata(path, attributes, lastwrite, options, snapshot, emptymetadata, database, self.StreamBlockChannel).ConfigureAwait(false)) + if (await ProcessMetadata(entry, attributes, lastwrite, options, emptymetadata, database, self.StreamBlockChannel).ConfigureAwait(false)) { try { - var split = Database.LocalDatabase.SplitIntoPrefixAndName(path); + var split = Database.LocalDatabase.SplitIntoPrefixAndName(entry.Path); long prefixid; if (string.Equals(prevprefix.Key, split.Key, StringComparison.Ordinal)) @@ -131,7 +131,7 @@ namespace Duplicati.Library.Main.Operation.Backup await self.Output.WriteAsync(new FileEntry { OldId = tmp.Item1, - Path = path, + Entry = entry, PathPrefixID = prefixid, Filename = split.Value, Attributes = attributes, @@ -148,7 +148,7 @@ namespace Duplicati.Library.Main.Operation.Backup await self.Output.WriteAsync(new FileEntry { OldId = res == null ? -1 : res.id, - Path = path, + Entry = entry, PathPrefixID = prefixid, Filename = split.Value, Attributes = attributes, @@ -166,7 +166,7 @@ namespace Duplicati.Library.Main.Operation.Backup continue; Logging.Log.WriteWarningMessage(FILELOGTAG, "ProcessingMetadataFailed", ex, - "Failed to process entry, path: {0}", path); + "Failed to process entry, path: {0}", entry.Path); } } } @@ -179,9 +179,9 @@ namespace Duplicati.Library.Main.Operation.Backup /// Processes the metadata for the given path. /// /// True if the path should be submitted to more analysis, false if there is nothing else to do - private static async Task ProcessMetadata(string path, FileAttributes attributes, DateTime lastwrite, Options options, Snapshots.ISnapshotService snapshot, IMetahash emptymetadata, BackupDatabase database, IWriteChannel streamblockchannel) + private static async Task ProcessMetadata(ISourceFileEntry entry, FileAttributes attributes, DateTime lastwrite, Options options, IMetahash emptymetadata, BackupDatabase database, IWriteChannel streamblockchannel) { - if (snapshot.IsSymlink(path, attributes)) + if (entry.IsSymlink) { // Not all reparse points are symlinks. // For example, on Windows 10 Fall Creator's Update, the OneDrive folder (and all subfolders) @@ -190,57 +190,58 @@ namespace Duplicati.Library.Main.Operation.Backup string symlinkTarget = null; try { - symlinkTarget = snapshot.GetSymlinkTarget(path); + symlinkTarget = entry.SymlinkTarget; } catch (Exception ex) { - Logging.Log.WriteExplicitMessage(FILELOGTAG, "SymlinkTargetReadFailure", ex, "Failed to read symlink target for path: {0}", path); + Logging.Log.WriteExplicitMessage(FILELOGTAG, "SymlinkTargetReadFailure", ex, "Failed to read symlink target for path: {0}", entry.Path); } if (!string.IsNullOrWhiteSpace(symlinkTarget)) { if (options.SymlinkPolicy == Options.SymlinkStrategy.Ignore) { - Logging.Log.WriteVerboseMessage(FILELOGTAG, "IgnoreSymlink", "Ignoring symlink {0}", path); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "IgnoreSymlink", "Ignoring symlink {0}", entry.Path); return false; } if (options.SymlinkPolicy == Options.SymlinkStrategy.Store) { - var metadata = MetadataGenerator.GenerateMetadata(path, attributes, options, snapshot); + var metadata = MetadataGenerator.GenerateMetadata(entry, attributes, options); if (!metadata.ContainsKey("CoreSymlinkTarget")) metadata["CoreSymlinkTarget"] = symlinkTarget; var metahash = Utility.WrapMetadata(metadata, options); - await AddSymlinkToOutputAsync(path, DateTime.UtcNow, metahash, database, streamblockchannel).ConfigureAwait(false); + await AddSymlinkToOutputAsync(entry.Path, DateTime.UtcNow, metahash, database, streamblockchannel).ConfigureAwait(false); - Logging.Log.WriteVerboseMessage(FILELOGTAG, "StoreSymlink", "Stored symlink {0}", path); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "StoreSymlink", "Stored symlink {0}", entry.Path); // Don't process further return false; } } else { - Logging.Log.WriteVerboseMessage(FILELOGTAG, "FollowingEmptySymlink", "Treating empty symlink as regular path {0}", path); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "FollowingEmptySymlink", "Treating empty symlink as regular path {0}", entry.Path); } } + if ((attributes & FileAttributes.Directory) == FileAttributes.Directory) { IMetahash metahash; if (!options.SkipMetadata) { - metahash = Utility.WrapMetadata(MetadataGenerator.GenerateMetadata(path, attributes, options, snapshot), options); + metahash = Utility.WrapMetadata(MetadataGenerator.GenerateMetadata(entry, attributes, options), options); } else { metahash = emptymetadata; } - Logging.Log.WriteVerboseMessage(FILELOGTAG, "AddDirectory", "Adding directory {0}", path); - await AddFolderToOutputAsync(path, lastwrite, metahash, database, streamblockchannel).ConfigureAwait(false); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "AddDirectory", "Adding directory {0}", entry.Path); + await AddFolderToOutputAsync(entry.Path, lastwrite, metahash, database, streamblockchannel).ConfigureAwait(false); return false; } diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 258119897..4c3171f6c 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -34,6 +34,8 @@ using Duplicati.Library.Common.IO; using Duplicati.Library.Logging; using Duplicati.Library.Main.Operation.Common; using System.Data; +using Duplicati.Library.DynamicLoader; +using Duplicati.Library.SourceProvider; namespace Duplicati.Library.Main.Operation { @@ -74,53 +76,120 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException(Strings.Common.PassphraseChangeUnsupported, "PassphraseChangeUnsupported"); } - public static Snapshots.ISnapshotService GetSnapshot(string[] sources, Options options) + /// + /// Gets a single source provider for the given sources + /// + /// The sources to get providers for + /// The options to use + /// The source providers + public static ISourceProvider GetSourceProvider(IEnumerable sources, Options options) + => Combiner.Combine(GetSourceProviders(sources, options)); + + /// + /// Gets a snapshot service for the given sources + /// + /// The sources to get the snapshot for + /// The options to use + /// The source provider + private static ISnapshotService GetFileSnapshotService(IEnumerable sources, Options options) { try { if (options.SnapShotStrategy != Options.OptimizationStrategy.Off) - return Duplicati.Library.Snapshots.SnapshotUtility.CreateSnapshot(sources, options.RawOptions); + return SnapshotUtility.CreateSnapshot(sources, options.RawOptions, options.SymlinkPolicy == Options.SymlinkStrategy.Follow); } catch (Exception ex) { if (options.SnapShotStrategy == Options.OptimizationStrategy.Required) throw new UserInformationException(Strings.Common.SnapshotFailedError(ex.Message), "SnapshotFailed", ex); else if (options.SnapShotStrategy == Options.OptimizationStrategy.On) - Logging.Log.WriteWarningMessage(LOGTAG, "SnapshotFailed", ex, Strings.Common.SnapshotFailedError(ex.Message)); + Log.WriteWarningMessage(LOGTAG, "SnapshotFailed", ex, Strings.Common.SnapshotFailedError(ex.Message)); else if (options.SnapShotStrategy == Options.OptimizationStrategy.Auto) - Logging.Log.WriteInformationMessage(LOGTAG, "SnapshotFailed", Strings.Common.SnapshotFailedError(ex.Message)); + Log.WriteInformationMessage(LOGTAG, "SnapshotFailed", Strings.Common.SnapshotFailedError(ex.Message)); } - if (OperatingSystem.IsMacOS() || OperatingSystem.IsLinux()) - { - return new NoSnapshotLinux(options.IgnoreAdvisoryLocking); - } - else if (OperatingSystem.IsWindows()) - { - return new NoSnapshotWindows(); - } - else - { - throw new NotSupportedException("Unsupported Operating System"); - } + return SnapshotUtility.CreateNoSnapshot(sources, options.IgnoreAdvisoryLocking, options.SymlinkPolicy == Options.SymlinkStrategy.Follow); } /// - /// Create instance of USN journal service + /// Gets all source providers for the given sources /// - /// - /// - /// - /// - /// - private UsnJournalService GetJournalService(IEnumerable sources, ISnapshotService snapshot, IFilter filter, long lastfilesetid) + /// The sources to get providers for + /// The options to use + /// The filter to use + /// The last fileset id + /// The source providers + private static List GetSourceProviders(IEnumerable sources, Options options) + { + // Group the sources by their type, so we can combine all snapshot paths into a single snapshot + var sourceTypes = sources.GroupBy(x => Library.Utility.Utility.GuessScheme(x) ?? "file", StringComparer.OrdinalIgnoreCase); + + // To avoid leaking snapshot instances, we create all instances first and then dispose them if an exception occurs + // The number of instances is expected to be low, so the memory overhead is acceptable + var results = new List(); + try + { + foreach (var entry in sourceTypes) + { + if ("file".Equals(entry.Key, StringComparison.OrdinalIgnoreCase)) + results.Add(new SourceProvider.File(GetFileSnapshotService(entry, options))); + else if ("vss".Equals(entry.Key, StringComparison.OrdinalIgnoreCase) || "lvm".Equals(entry.Key, StringComparison.OrdinalIgnoreCase)) + results.Add(new SourceProvider.File(SnapshotUtility.CreateSnapshot(entry, options.RawOptions, options.SymlinkPolicy == Options.SymlinkStrategy.Follow))); + else + { + foreach (var url in entry) + { + // Source providers are preferred over backends + var provider = SourceProviderLoader.GetSourceProvider(url, options.RawOptions); + if (provider != null) + { + results.Add(provider); + continue; + } + + // See if we can implement the same with a backend + var backend = BackendLoader.GetBackend(url, options.RawOptions); + if (backend != null) + { + results.Add(new SourceProvider.BackendSourceProvider(backend, url)); + continue; + } + + throw new UserInformationException(string.Format("The source \"{0}\" is not supported", url), "SourceNotSupported"); + } + } + } + } + catch + { + foreach (var provider in results) + (provider as IDisposable)?.Dispose(); + + throw; + } + + return results; + } + + public UsnJournalService GetJournalService(ISourceProvider provider, IFilter filter, long lastfilesetid) { if (m_options.UsnStrategy == Options.OptimizationStrategy.Off) return null; if (!OperatingSystem.IsWindows()) throw new UserInformationException("USN journal is only supported on Windows", "UsnJournalNotSupported"); + var providers = (provider is Combiner c ? c.Providers : [provider]) + .Select((x, i) => new { Provider = x, Index = i }) + .Where(x => x.Provider is SourceProvider.File) + .ToList(); + + if (providers.Count == 0) + return null; + if (providers.Count > 1) + throw new UserInformationException("Multiple USN journal services are not supported", "MultipleUSNJournalServices"); + var fileProvider = providers.First().Provider as SourceProvider.File; + var journalData = m_database.GetChangeJournalData(lastfilesetid); - var service = new UsnJournalService(sources, snapshot, filter, m_options.FileAttributeFilter, m_options.SkipFilesLargerThan, + var service = new UsnJournalService(fileProvider.SnapshotService, filter, m_options.FileAttributeFilter, m_options.SkipFilesLargerThan, journalData, cancellationTokenSource.Token); foreach (var volumeData in service.VolumeDataList) @@ -288,7 +357,7 @@ namespace Duplicati.Library.Main.Operation /// /// Performs the bulk of work by starting all relevant processes /// - private static async Task RunMainOperation(Backup.Channels channels, IEnumerable sources, Snapshots.ISnapshotService snapshot, UsnJournalService journalService, Backup.BackupDatabase database, IBackendManager backendManager, Backup.BackupStatsCollector stats, Options options, IFilter sourcefilter, IFilter filter, BackupResults result, ITaskReader taskreader, long filesetid, long lastfilesetid) + private static async Task RunMainOperation(Backup.Channels channels, ISourceProvider source, UsnJournalService journalService, Backup.BackupDatabase database, IBackendManager backendManager, Backup.BackupStatsCollector stats, Options options, IFilter sourcefilter, IFilter filter, BackupResults result, ITaskReader taskreader, long filesetid, long lastfilesetid) { using (new Logging.Timer(LOGTAG, "BackupMainOperation", "BackupMainOperation")) { @@ -299,15 +368,15 @@ namespace Duplicati.Library.Main.Operation new[] { Backup.DataBlockProcessor.Run(channels, database, backendManager, options, taskreader), - Backup.FileBlockProcessor.Run(channels, snapshot, options, database, stats, taskreader), + Backup.FileBlockProcessor.Run(channels, options, database, stats, taskreader), Backup.StreamBlockSplitter.Run(channels, options, database, taskreader), - Backup.FileEnumerationProcess.Run(channels, sources, snapshot, journalService, - options.FileAttributeFilter, sourcefilter, filter, options.SymlinkPolicy, + Backup.FileEnumerationProcess.Run(channels, source, journalService, + options.FileAttributeFilter, filter, options.SymlinkPolicy, options.HardlinkPolicy, options.ExcludeEmptyFolders, options.IgnoreFilenames, GetBlacklistedPaths(options), options.ChangedFilelist, taskreader, () => result.PartialBackup = true, CancellationToken.None), - Backup.FilePreFilterProcess.Run(channels, snapshot, options, stats, database), - Backup.MetadataPreProcess.Run(channels, snapshot, options, database, lastfilesetid, taskreader), + Backup.FilePreFilterProcess.Run(channels, options, stats, database), + Backup.MetadataPreProcess.Run(channels, options, database, lastfilesetid, taskreader), Backup.SpillCollectorProcess.Run(channels, options, database, backendManager, taskreader), Backup.ProgressHandler.Run(channels, result) } @@ -324,7 +393,7 @@ namespace Duplicati.Library.Main.Operation // Spawn additional file processors .Concat( Enumerable.Range(0, options.ConcurrencyFileprocessors - 1).Select(x => - Backup.FileBlockProcessor.Run(channels, snapshot, options, database, stats, taskreader) + Backup.FileBlockProcessor.Run(channels, options, database, stats, taskreader) ) ) ); @@ -351,9 +420,8 @@ namespace Duplicati.Library.Main.Operation return true; if (fileSize >= 0) - { stats.AddExaminedFile(fileSize); - } + return false; }); @@ -528,7 +596,7 @@ namespace Duplicati.Library.Main.Operation { long filesetid; using var counterToken = CancellationTokenSource.CreateLinkedTokenSource(m_taskReader.ProgressToken); - using (var snapshot = GetSnapshot(sources, m_options)) + using (var source = GetSourceProvider(sources, m_options)) { try { @@ -558,8 +626,7 @@ namespace Duplicati.Library.Main.Operation var filesetvolumeid = await db.RegisterRemoteVolumeAsync(filesetvolume.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary); filesetid = await db.CreateFilesetAsync(filesetvolumeid, VolumeBase.ParseFilename(filesetvolume.RemoteFilename).Time); - // create USN-based scanner if enabled - var journalService = GetJournalService(sources, snapshot, filter, lastfilesetid); + var journalService = GetJournalService(source, filter, lastfilesetid); // Start parallel scan, or use the database if (m_options.DisableFileScanner) @@ -569,13 +636,13 @@ namespace Duplicati.Library.Main.Operation } else { - parallelScanner = Backup.CountFilesHandler.Run(sources, snapshot, journalService, m_result, m_options, m_sourceFilter, m_filter, GetBlacklistedPaths(), m_result.TaskControl, counterToken.Token); + parallelScanner = Backup.CountFilesHandler.Run(source, journalService, m_result, m_options, m_filter, GetBlacklistedPaths(), m_result.TaskControl, counterToken.Token); } // Run the backup operation if (await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await RunMainOperation(channels, sources, snapshot, journalService, db, backendManager, stats, m_options, m_sourceFilter, m_filter, m_result, m_result.TaskControl, filesetid, lastfilesetid).ConfigureAwait(false); + await RunMainOperation(channels, source, journalService, db, backendManager, stats, m_options, m_sourceFilter, m_filter, m_result, m_result.TaskControl, filesetid, lastfilesetid).ConfigureAwait(false); } } finally diff --git a/Duplicati/Library/Main/Operation/TestFilterHandler.cs b/Duplicati/Library/Main/Operation/TestFilterHandler.cs index c2f96ffba..4a06bbc89 100644 --- a/Duplicati/Library/Main/Operation/TestFilterHandler.cs +++ b/Duplicati/Library/Main/Operation/TestFilterHandler.cs @@ -1,27 +1,26 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.IO; -using Duplicati.Library.Snapshots; using CoCoL; using System.Threading.Tasks; @@ -45,14 +44,13 @@ namespace Duplicati.Library.Main.Operation public async Task RunAsync(string[] sources, Library.Utility.IFilter filter) { - var sourcefilter = new Library.Utility.FilterExpression(sources, true); var stopToken = m_result.TaskControl.ProgressToken; - using (var snapshot = BackupHandler.GetSnapshot(sources, m_options)) + using (var provider = BackupHandler.GetSourceProvider(sources, m_options)) { Backup.Channels channels = new(); - var source = Backup.FileEnumerationProcess.Run(channels, sources, snapshot, null, - m_options.FileAttributeFilter, sourcefilter, filter, m_options.SymlinkPolicy, + var source = Backup.FileEnumerationProcess.Run(channels, provider, null, + m_options.FileAttributeFilter, filter, m_options.SymlinkPolicy, m_options.HardlinkPolicy, m_options.ExcludeEmptyFolders, m_options.IgnoreFilenames, BackupHandler.GetBlacklistedPaths(m_options), null, m_result.TaskControl, null, stopToken); @@ -64,20 +62,20 @@ namespace Duplicati.Library.Main.Operation { while (await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - var path = await self.source.ReadAsync(); - var fa = FileAttributes.Normal; - try { fa = snapshot.GetAttributes(path); } - catch (Exception ex) { Logging.Log.WriteVerboseMessage(LOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", path, ex.Message); } + var entry = await self.source.ReadAsync(); + var fa = entry.IsFolder + ? FileAttributes.Directory + : FileAttributes.Normal; + + try { fa = entry.Attributes; } + catch (Exception ex) { Logging.Log.WriteVerboseMessage(LOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", entry.Path, ex.Message); } // Analyze symlinks - var isSymlink = snapshot.IsSymlink(path, fa); string symlinkTarget = null; + try { symlinkTarget = entry.SymlinkTarget; } + catch (Exception ex) { Logging.Log.WriteExplicitMessage(LOGTAG, "SymlinkTargetReadFailure", ex, "Failed to read symlink target for path: {0}", entry.Path); } - if (isSymlink) - try { symlinkTarget = snapshot.GetSymlinkTarget(path); } - catch (Exception ex) { Logging.Log.WriteExplicitMessage(LOGTAG, "SymlinkTargetReadFailure", ex, "Failed to read symlink target for path: {0}", path); } - - if (isSymlink && m_options.SymlinkPolicy == Options.SymlinkStrategy.Store && !string.IsNullOrWhiteSpace(symlinkTarget)) + if (m_options.SymlinkPolicy == Options.SymlinkStrategy.Store && !string.IsNullOrWhiteSpace(symlinkTarget)) { // Skip stored symlinks continue; @@ -86,33 +84,37 @@ namespace Duplicati.Library.Main.Operation // Go for the symlink target, as we know we follow symlinks if (!string.IsNullOrWhiteSpace(symlinkTarget)) { - path = symlinkTarget; + entry = await provider.GetEntry(symlinkTarget, false, stopToken).ConfigureAwait(false); fa = FileAttributes.Normal; - try { fa = snapshot.GetAttributes(path); } - catch (Exception ex) { Logging.Log.WriteVerboseMessage(LOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", path, ex.Message); } + + try { fa = entry.Attributes; } + catch (Exception ex) { Logging.Log.WriteVerboseMessage(LOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", entry.Path, ex.Message); } + + if (fa.HasFlag(FileAttributes.Directory)) + entry = await provider.GetEntry(symlinkTarget, true, stopToken).ConfigureAwait(false); } // Proceed with non-folders - if (!((fa & FileAttributes.Directory) == FileAttributes.Directory)) + if (!entry.IsFolder) { m_result.FileCount++; var size = -1L; try { - size = snapshot.GetFileSize(path); + size = entry.Size; m_result.FileSize += size; } catch (Exception ex) { - Logging.Log.WriteVerboseMessage(LOGTAG, "SizeReadFailed", "Failed to read length of file {0}: {1}", path, ex.Message); + Logging.Log.WriteVerboseMessage(LOGTAG, "SizeReadFailed", "Failed to read length of file {0}: {1}", entry.Path, ex.Message); } if (m_options.SkipFilesLargerThan == long.MaxValue || m_options.SkipFilesLargerThan == 0 || size < m_options.SkipFilesLargerThan) - Logging.Log.WriteVerboseMessage(LOGTAG, "IncludeFile", "Including file: {0} ({1})", path, size < 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(size)); + Logging.Log.WriteVerboseMessage(LOGTAG, "IncludeFile", "Including file: {0} ({1})", entry.Path, size < 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(size)); else - Logging.Log.WriteVerboseMessage(LOGTAG, "ExcludeLargeFile", "Excluding file due to size: {0} ({1})", path, size < 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(size)); + Logging.Log.WriteVerboseMessage(LOGTAG, "ExcludeLargeFile", "Excluding file due to size: {0} ({1})", entry.Path, size < 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(size)); } } } diff --git a/Duplicati/Library/Modules/Builtin/ReportHelper.cs b/Duplicati/Library/Modules/Builtin/ReportHelper.cs index 78cbfae29..e37ee17ac 100644 --- a/Duplicati/Library/Modules/Builtin/ReportHelper.cs +++ b/Duplicati/Library/Modules/Builtin/ReportHelper.cs @@ -411,7 +411,7 @@ namespace Duplicati.Library.Modules.Builtin case DESTINATION_TYPE: // Only return the url scheme, as the rest could contain sensitive information var ix = m_remoteurl?.IndexOf("://", StringComparison.OrdinalIgnoreCase) ?? -1; - return ix >= 0 && ix < 15 ? m_remoteurl.Substring(0, ix) : "file"; + return Utility.Utility.GuessScheme(m_remoteurl) ?? "file"; default: return null; } diff --git a/Duplicati/Library/Modules/Builtin/SendJabberMessage.cs b/Duplicati/Library/Modules/Builtin/SendJabberMessage.cs index 2f8dac808..bbb565017 100644 --- a/Duplicati/Library/Modules/Builtin/SendJabberMessage.cs +++ b/Duplicati/Library/Modules/Builtin/SendJabberMessage.cs @@ -195,8 +195,7 @@ namespace Duplicati.Library.Modules.Builtin protected override void SendMessage(string subject, string body) { - - var uri = new Library.Utility.Uri(m_username.Contains("://") ? m_username : "http://" + m_username); + var uri = new Utility.Uri(m_username.Contains("://") ? m_username : "http://" + m_username); var resource = uri.Path ?? ""; if (resource.StartsWith("/", StringComparison.Ordinal)) resource = resource.Substring(1); diff --git a/Duplicati/Library/Snapshots/Duplicati.Library.Snapshots.csproj b/Duplicati/Library/Snapshots/Duplicati.Library.Snapshots.csproj index 2f5c08116..a166310db 100644 --- a/Duplicati/Library/Snapshots/Duplicati.Library.Snapshots.csproj +++ b/Duplicati/Library/Snapshots/Duplicati.Library.Snapshots.csproj @@ -3,10 +3,12 @@ net8.0 Copyright © 2025 Team Duplicati, MIT license + Duplicati.Library.Snapshots + @@ -14,7 +16,7 @@ - + diff --git a/Duplicati/Library/Snapshots/ISnapshotService.cs b/Duplicati/Library/Snapshots/ISnapshotService.cs index 9f61d6886..4deea290f 100644 --- a/Duplicati/Library/Snapshots/ISnapshotService.cs +++ b/Duplicati/Library/Snapshots/ISnapshotService.cs @@ -1,27 +1,32 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable using System; using System.Collections.Generic; using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Duplicati.Library.Interface; namespace Duplicati.Library.Snapshots { @@ -31,12 +36,28 @@ namespace Duplicati.Library.Snapshots public interface ISnapshotService : IDisposable { /// - /// Enumerates all files and folders in the snapshot, restricted to sources + /// Gets the source folders /// - /// Sources to enumerate - /// The callback to invoke with each found path - /// The callback used to report errors - IEnumerable EnumerateFilesAndFolders(IEnumerable sources, Utility.Utility.EnumerationFilterDelegate callback, Utility.Utility.ReportAccessError errorCallback); + IEnumerable SourceFolders { get; } + + /// + /// Enumerates the root source files and folders + /// + /// The source files and folders + IEnumerable EnumerateFilesystemEntries(); + /// + /// Enumerates the files and folders in a given folder + /// + /// The parent folder to enumerate + /// The files and folders in the parent folder + IEnumerable EnumerateFilesystemEntries(ISourceFileEntry parent); + /// + /// Gets a filesystem entry for a given path + /// + /// The path to get + /// True if the path is a folder + /// The filesystem entry + ISourceFileEntry GetFilesystemEntry(string path, bool isFolder); /// /// Gets the last write time of a given file in UTC @@ -59,6 +80,14 @@ namespace Duplicati.Library.Snapshots /// An open filestream that can be read and seeked Stream OpenRead(string localPath); + /// + /// Opens a file for reading + /// + /// The full path to the file in non-snapshot format + /// A token that can be used to cancel the operation + /// An open filestream that can be read and seeked + Task OpenReadAsync(string localPath, CancellationToken cancellationToken); + /// /// Returns the size of a file /// @@ -79,29 +108,28 @@ namespace Duplicati.Library.Snapshots /// The file attributes /// The file or folder to examine FileAttributes GetAttributes(string localPath); - + /// /// Gets the metadata for the given file or folder /// /// The metadata for the given file or folder /// The file or folder to examine /// A flag indicating if the target is a symlink - /// A flag indicating if a symlink should be followed - Dictionary GetMetadata(string localPath, bool isSymlink, bool followSymlink); - + Dictionary GetMetadata(string localPath, bool isSymlink); + /// /// Gets a value indicating if the path points to a block device /// /// true if this instance is a block device; otherwise, false. /// The file or folder to examine bool IsBlockDevice(string localPath); - + /// /// Gets a unique hardlink target ID /// /// The hardlink ID /// The file or folder to examine - string HardlinkTargetID(string localPath); + string? HardlinkTargetID(string localPath); /// /// Converts a snapshot path to a local path @@ -130,17 +158,5 @@ namespace Duplicati.Library.Snapshots /// The local path /// True if folder exists, false otherwise bool DirectoryExists(string localFolderPath); - - /// - /// Tests if a file or folder exists in the snapshot - /// - /// The local path - /// True if file or folder exists, false otherwise - bool PathExists(string localFileOrFolderPath); - - /// - /// Is true if the actual service is a real snapshot service - /// - bool IsSnapshot { get; } } } diff --git a/Duplicati/Library/Snapshots/LinuxSnapshot.cs b/Duplicati/Library/Snapshots/LinuxSnapshot.cs index f49044f04..9d5a51832 100644 --- a/Duplicati/Library/Snapshots/LinuxSnapshot.cs +++ b/Duplicati/Library/Snapshots/LinuxSnapshot.cs @@ -1,31 +1,33 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; using System.Diagnostics; +using System.Linq; using System.Runtime.Versioning; using System.Text; using Duplicati.Library.Common.IO; +using Duplicati.Library.Interface; namespace Duplicati.Library.Snapshots { @@ -40,7 +42,7 @@ namespace Duplicati.Library.Snapshots [SupportedOSPlatform("macOS")] public sealed class LinuxSnapshot : SnapshotBase { - /// + /// /// The tag used for logging messages /// public static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(WindowsSnapshot)); @@ -59,7 +61,9 @@ namespace Duplicati.Library.Snapshots /// Constructs a new snapshot module using LVM /// /// The list of folders to create snapshots for - public LinuxSnapshot(IEnumerable sources) + /// A flag indicating if symlinks should be followed + public LinuxSnapshot(IEnumerable sources, bool followSymlinks) + : base(followSymlinks) { try { @@ -79,7 +83,7 @@ namespace Duplicati.Library.Snapshots m_entries.Add(new KeyValuePair(path, snap)); } - m_snapShots = new List(snaps.Values); + m_snapShots = [.. snaps.Values]; // We have all the snapshots that we need, lets activate them foreach (var snap in m_snapShots) @@ -94,15 +98,44 @@ namespace Duplicati.Library.Snapshots { Dispose(); } - catch (Exception ex) + catch (Exception ex) { - Logging.Log.WriteVerboseMessage(LOGTAG, "SnapshotCleanupError", ex, "Failed to clean up after error"); + Logging.Log.WriteVerboseMessage(LOGTAG, "SnapshotCleanupError", ex, "Failed to clean up after error"); } throw; } } + /// + /// Gets the source folders + /// + public override IEnumerable SourceFolders => m_snapShots.Select(e => e.LocalPath); + + /// + /// Gets a filesystem entry for a given path + /// + /// The path to get the entry for + /// A flag indicating if the path is a folder + /// The filesystem entry + public override ISourceFileEntry GetFilesystemEntry(string path, bool isFolder) + => new SnapshotSourceFileEntry(this, path, isFolder, false); + + /// + /// Enumerates the root source files and folders + /// + /// The source files and folders + public override IEnumerable EnumerateFilesystemEntries() + { + foreach (var snap in m_snapShots) + { + if (DirectoryExists(snap.LocalPath) || snap.LocalPath.EndsWith(System.IO.Path.DirectorySeparatorChar)) + yield return new SnapshotSourceFileEntry(this, snap.LocalPath, true, true); + else + yield return new SnapshotSourceFileEntry(this, snap.LocalPath, false, true); + } + } + /// protected override void Dispose(bool disposing) { @@ -111,10 +144,10 @@ namespace Duplicati.Library.Snapshots if (disposing) { // Attempt to clean out as many as possible - foreach(var s in m_snapShots) + foreach (var s in m_snapShots) { try { s.Dispose(); } - catch (Exception ex) { Logging.Log.WriteVerboseMessage(LOGTAG, "SnapshotCloseError", ex, "Failed to close a snapshot"); } + catch (Exception ex) { Logging.Log.WriteVerboseMessage(LOGTAG, "SnapshotCloseError", ex, "Failed to close a snapshot"); } } } @@ -475,10 +508,9 @@ namespace Duplicati.Library.Snapshots /// The metadata for the given file or folder /// The file or folder to examine /// A flag indicating if the target is a symlink - /// A flag indicating if a symlink should be followed - public override Dictionary GetMetadata(string localPath, bool isSymlink, bool followSymlink) + public override Dictionary GetMetadata(string localPath, bool isSymlink) { - return SystemIO.IO_SYS.GetMetadata(ConvertToSnapshotPath(localPath), isSymlink, followSymlink); + return SystemIO.IO_SYS.GetMetadata(ConvertToSnapshotPath(localPath), isSymlink, FollowSymlinks); } /// @@ -500,8 +532,8 @@ namespace Duplicati.Library.Snapshots default: return true; } - } - catch + } + catch { if (!System.IO.File.Exists(SystemIOLinux.NormalizePath(localPath))) return false; @@ -509,7 +541,7 @@ namespace Duplicati.Library.Snapshots throw; } } - + /// /// Gets a unique hardlink target ID /// @@ -518,10 +550,10 @@ namespace Duplicati.Library.Snapshots public override string HardlinkTargetID(string localPath) { var snapshotPath = ConvertToSnapshotPath(localPath); - + if (PosixFile.GetHardlinkCount(snapshotPath) <= 1) return null; - + return PosixFile.GetInodeTargetID(snapshotPath); } @@ -537,9 +569,6 @@ namespace Duplicati.Library.Snapshots return FindSnapshotByLocalPath(localPath).ConvertToSnapshotPath(localPath); } - /// - public override bool IsSnapshot => true; - #endregion } } diff --git a/Duplicati/Library/Snapshots/NoSnapshotLinux.cs b/Duplicati/Library/Snapshots/NoSnapshotLinux.cs index f1601169b..3bad638eb 100644 --- a/Duplicati/Library/Snapshots/NoSnapshotLinux.cs +++ b/Duplicati/Library/Snapshots/NoSnapshotLinux.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -25,6 +25,7 @@ using System.IO; using System.Runtime.InteropServices; using System.Runtime.Versioning; using Duplicati.Library.Common.IO; +using Duplicati.Library.Interface; using Microsoft.Win32.SafeHandles; namespace Duplicati.Library.Snapshots @@ -65,15 +66,43 @@ namespace Duplicati.Library.Snapshots /// private readonly bool m_ignoreAdvisoryLocks; + /// + /// The list of folders to create snapshots of + /// + private readonly IEnumerable m_folders; + /// /// Initializes a new instance of the class. /// + /// The list of folders to create snapshots of /// A flag indicating if advisory locks should be ignored - public NoSnapshotLinux(bool ignoreAdvisoryLocks) + /// A flag indicating if symlinks should be followed + public NoSnapshotLinux(IEnumerable folders, bool ignoreAdvisoryLocks, bool followSymlinks) + : base(followSymlinks) { + m_folders = folders; m_ignoreAdvisoryLocks = ignoreAdvisoryLocks; } + /// + /// Gets the source folders + /// + public override IEnumerable SourceFolders => m_folders; + + /// + /// Enumerates the root source files and folders + /// + /// The source files and folders + public override IEnumerable EnumerateFilesystemEntries() + { + foreach (var folder in m_folders) + { + if (DirectoryExists(folder) || folder.EndsWith(Path.DirectorySeparatorChar)) + yield return new SnapshotSourceFileEntry(this, folder, true, true); + else + yield return new SnapshotSourceFileEntry(this, folder, false, true); + } + } /// /// Returns the symlink target if the entry is a symlink, and null otherwise @@ -91,10 +120,9 @@ namespace Duplicati.Library.Snapshots /// The metadata for the given file or folder /// The file or folder to examine /// A flag indicating if the target is a symlink - /// A flag indicating if a symlink should be followed - public override Dictionary GetMetadata(string localPath, bool isSymlink, bool followSymlink) + public override Dictionary GetMetadata(string localPath, bool isSymlink) { - return SystemIO.IO_SYS.GetMetadata(localPath, isSymlink, followSymlink); + return SystemIO.IO_SYS.GetMetadata(localPath, isSymlink, FollowSymlinks); } /// diff --git a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs index f3a3c7184..706554cfd 100644 --- a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs +++ b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -24,6 +24,7 @@ using System.Collections.Generic; using System.Linq; using System.Runtime.Versioning; using Duplicati.Library.Common.IO; +using Duplicati.Library.Interface; namespace Duplicati.Library.Snapshots { @@ -33,100 +34,114 @@ namespace Duplicati.Library.Snapshots [SupportedOSPlatform("windows")] public sealed class NoSnapshotWindows : SnapshotBase { + /// + /// The list of folders to create snapshots of + /// + private readonly IEnumerable m_folders; + + /// + /// Initializes a new instance of the class. + /// + /// The list of folders to create snapshots of + /// A flag indicating if symlinks should be followed + public NoSnapshotWindows(IEnumerable folders, bool followSymlinks) + : base(followSymlinks) + { + m_folders = folders; + } + /// /// Returns the symlink target if the entry is a symlink, and null otherwise /// /// The file or folder to examine /// The symlink target public override string GetSymlinkTarget(string localPath) - { - return SystemIO.IO_WIN.GetSymlinkTarget(localPath); - } + => SystemIO.IO_WIN.GetSymlinkTarget(localPath); /// /// Gets the attributes for the given file or folder /// /// The file attributes /// The file or folder to examine - public override System.IO.FileAttributes GetAttributes (string localPath) - { - return SystemIO.IO_WIN.GetFileAttributes(localPath); - } + public override System.IO.FileAttributes GetAttributes(string localPath) + => SystemIO.IO_WIN.GetFileAttributes(localPath); /// /// Returns the size of a file /// /// The full path to the file in non-snapshot format /// The length of the file - public override long GetFileSize (string localPath) - { - return SystemIO.IO_WIN.FileLength(localPath); - } + public override long GetFileSize(string localPath) + => SystemIO.IO_WIN.FileLength(localPath); /// - /// Enumerates all files and folders in the snapshot, restricted to sources + /// Gets the source folders /// - /// Sources to enumerate - /// The callback to invoke with each found path - /// The callback used to report errors - public override IEnumerable EnumerateFilesAndFolders(IEnumerable sources, Utility.Utility.EnumerationFilterDelegate callback, Utility.Utility.ReportAccessError errorCallback) + public override IEnumerable SourceFolders + => m_folders; + + /// + /// Enumerates the root source files and folders + /// + /// The source files and folders + public override IEnumerable EnumerateFilesystemEntries() { - // For Windows, ensure we don't store paths with extended device path prefixes (i.e., @"\\?\" or @"\\?\UNC\") - return base.EnumerateFilesAndFolders(sources.Select(SystemIOWindows.RemoveExtendedDevicePathPrefix), callback, errorCallback); + foreach (var folder in m_folders.Select(SystemIOWindows.AddExtendedDevicePathPrefix)) + { + if (DirectoryExists(folder) || folder.EndsWith(System.IO.Path.DirectorySeparatorChar)) + yield return new SnapshotSourceFileEntry(this, folder, true, true); + else + yield return new SnapshotSourceFileEntry(this, folder, false, true); + } } + /// /// Gets the last write time of a given file in UTC /// /// The full path to the file in non-snapshot format /// The last write time of the file - public override DateTime GetLastWriteTimeUtc (string localPath) - { - return SystemIO.IO_WIN.FileGetLastWriteTimeUtc(localPath); - } + public override DateTime GetLastWriteTimeUtc(string localPath) + => SystemIO.IO_WIN.FileGetLastWriteTimeUtc(localPath); /// /// Gets the last write time of a given file in UTC /// /// The full path to the file in non-snapshot format /// The last write time of the file - public override DateTime GetCreationTimeUtc (string localPath) - { - return SystemIO.IO_WIN.FileGetCreationTimeUtc(localPath); - } + public override DateTime GetCreationTimeUtc(string localPath) + => SystemIO.IO_WIN.FileGetCreationTimeUtc(localPath); /// /// Opens a file for reading /// /// The full path to the file in non-snapshot format /// An open filestream that can be read - public override System.IO.Stream OpenRead (string localPath) - { - return SystemIO.IO_WIN.FileOpenRead(localPath); - } + public override System.IO.Stream OpenRead(string localPath) + => SystemIO.IO_WIN.FileOpenRead(localPath); /// /// Lists all files in the given folder /// /// All folders found in the folder /// The folder to examinate - protected override string[] ListFiles (string localFolderPath) + protected override string[] ListFiles(string localFolderPath) { string[] tmp = SystemIO.IO_WIN.GetFiles(localFolderPath); string[] res = new string[tmp.Length]; - for(int i = 0; i < tmp.Length; i++) + for (int i = 0; i < tmp.Length; i++) res[i] = SystemIOWindows.RemoveExtendedDevicePathPrefix(tmp[i]); return res; } - + /// /// Lists all folders in the given folder /// /// All folders found in the folder /// The folder to examinate - protected override string[] ListFolders (string localFolderPath) + protected override string[] ListFolders(string localFolderPath) { string[] tmp = SystemIO.IO_WIN.GetDirectories(SystemIOWindows.AddExtendedDevicePathPrefix(localFolderPath)); string[] res = new string[tmp.Length]; @@ -142,11 +157,8 @@ namespace Duplicati.Library.Snapshots /// The metadata for the given file or folder /// The file or folder to examine /// A flag indicating if the target is a symlink - /// A flag indicating if a symlink should be followed - public override Dictionary GetMetadata(string localPath, bool isSymlink, bool followSymlink) - { - return SystemIO.IO_WIN.GetMetadata(localPath, isSymlink, followSymlink); - } + public override Dictionary GetMetadata(string localPath, bool isSymlink) + => SystemIO.IO_WIN.GetMetadata(localPath, isSymlink, FollowSymlinks); /// public override string ConvertToLocalPath(string snapshotPath) @@ -155,11 +167,9 @@ namespace Duplicati.Library.Snapshots } /// - public override string ConvertToSnapshotPath(string localPath) - { + public override string ConvertToSnapshotPath(string localPath) => // For Windows, ensure we don't store paths with extended device path prefixes (i.e., @"\\?\" or @"\\?\UNC\") - return SystemIOWindows.RemoveExtendedDevicePathPrefix(localPath); - } + SystemIOWindows.RemoveExtendedDevicePathPrefix(localPath); } } diff --git a/Duplicati/Library/Snapshots/Program.cs b/Duplicati/Library/Snapshots/Program.cs index 5f39417e9..6afd2647d 100644 --- a/Duplicati/Library/Snapshots/Program.cs +++ b/Duplicati/Library/Snapshots/Program.cs @@ -110,7 +110,7 @@ Where is the folder where files will be locked/created etc"); Console.WriteLine("Creating snapshot for folder: {0}", args[0]); Console.WriteLine("If this fails, try to run as " + ((OperatingSystem.IsMacOS() || OperatingSystem.IsLinux()) ? "root" : "Administrator")); - using (ISnapshotService snapshot = SnapshotUtility.CreateSnapshot(new[] { args[0] }, options)) + using (ISnapshotService snapshot = SnapshotUtility.CreateSnapshot(new[] { args[0] }, options, false)) { Console.WriteLine("Attempting to read locked file via snapshot"); try diff --git a/Duplicati/Library/Snapshots/RecursiveFilterHelper.cs b/Duplicati/Library/Snapshots/RecursiveFilterHelper.cs new file mode 100644 index 000000000..1e8fb9f02 --- /dev/null +++ b/Duplicati/Library/Snapshots/RecursiveFilterHelper.cs @@ -0,0 +1,229 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using Duplicati.Library.Interface; +using System.Threading.Tasks; +using System.Runtime.CompilerServices; + +namespace Duplicati.Library.Snapshots; + +public static class RecursiveFilterHelper +{ + /// + /// The log tag to use + /// + private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(RecursiveFilterHelper)); + + /// + /// Filters sources, returning sub-set having been modified since last + /// change, as specified by journalData. + /// + /// List of sources + /// Snapshot service + /// Filter callback to exclude filtered items + /// Cancellation token + /// Filtered sources + public static async IAsyncEnumerable GetModifiedSources(IAsyncEnumerable entries, ISnapshotService snapshot, Func> filter, [EnumeratorCancellation] CancellationToken token) + { + // prepare cache for includes (value = true) and excludes (value = false, will be populated + // on-demand) + var cache = new Dictionary(); + await foreach (var source in entries) + { + if (token.IsCancellationRequested) + break; + + cache[source.Path] = true; + } + + // Check the simplified folders, and their parent folders against the exclusion filter. + // This is needed because the filter may exclude "C:\A\", but this won't match the more + // specific "C:\A\B\" in our list, even though it's meant to be excluded. + // The reason why the filter doesn't exclude it is because during a regular (non-USN) full scan, + // FilterHandler.EnumerateFilesAndFolders() works top-down, and won't even enumerate child + // folders. + // The sources are needed to stop evaluating parent folders above the specified source folders + await foreach (var folder in FilterExcludedFolders(entries.Where(x => x.IsFolder), snapshot, filter, cache, token)) + { + if (token.IsCancellationRequested) + break; + + if (!snapshot.DirectoryExists(folder.Path)) + continue; + + yield return folder; + } + + // The simplified file list also needs to be checked against the exclusion filter, as it + // may contain entries excluded due to attributes, but also because they are below excluded + // folders, which themselves aren't in the folder list from step 1. + // Note that the simplified file list may contain entries that have been deleted! They need to + // be kept in the list (unless excluded by the filter) in order for the backup handler to record their + // deletion. + await foreach (var files in FilterExcludedFiles(entries.Where(x => !x.IsFolder), snapshot, filter, cache, token)) + { + if (token.IsCancellationRequested) + break; + + if (!snapshot.FileExists(files.Path)) + continue; + + yield return files; + } + } + + /// + /// Filter supplied files, removing any files which itself, or one + /// of its parent folders, is excluded by the filter. + /// + /// Files to filter + /// Snapshot service + /// Exclusion filter + /// Cache of included and excluded files / folders + /// Cancellation token + /// Filtered files + private static async IAsyncEnumerable FilterExcludedFiles( + IAsyncEnumerable files, + ISnapshotService snapshot, + Func> filter, + IDictionary cache, + [EnumeratorCancellation] CancellationToken token) + { + await foreach (var file in files.WithCancellation(token).ConfigureAwait(false)) + { + try + { + if (!await filter(file)) + continue; + + var parentPath = Utility.Utility.GetParent(file.Path, true); + if (!string.IsNullOrWhiteSpace(parentPath)) + { + var parent = snapshot.GetFilesystemEntry(parentPath, true); + if (await IsFolderOrAncestorsExcluded(parent, snapshot, filter, cache, token)) + continue; + } + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "FilterExcludedFilesError", ex, "Error while filtering file: {0}", ex.Message); + continue; + } + + yield return file; + } + } + + /// + /// Filter supplied folders, removing any folder which itself, or one + /// of its ancestors, is excluded by the filter. + /// + /// Folder to filter + /// Snapshot service + /// Exclusion filter + /// Cache of excluded folders + /// Cancellation token + /// Filtered folders + private static async IAsyncEnumerable FilterExcludedFolders( + IAsyncEnumerable folders, + ISnapshotService snapshot, + Func> filter, + IDictionary cache, + [EnumeratorCancellation] CancellationToken token) + { + await foreach (var folder in folders.WithCancellation(token).ConfigureAwait(false)) + { + try + { + if (await IsFolderOrAncestorsExcluded(folder, snapshot, filter, cache, token)) + continue; + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "FilterExcludedFoldersError", ex, "Error while filtering folder: {0}", ex.Message); + continue; + } + + yield return folder; + } + } + + /// + /// Tests if specified folder, or any of its ancestors, is excluded by the filter + /// + /// Folder to test + /// Filter + /// Cache of excluded folders (optional) + /// True if excluded, false otherwise + private static async ValueTask IsFolderOrAncestorsExcluded( + ISourceFileEntry folder, + ISnapshotService snapshot, + Func> filter, + IDictionary cache, + CancellationToken token) + { + List? parents = null; + while (folder != null) + { + if (token.IsCancellationRequested) + break; + + // first check cache + if (cache.TryGetValue(folder.Path, out var include)) + { + if (include) + return false; + + break; // hit! + } + + parents ??= []; // create on-demand + + // remember folder for cache + parents.Add(folder.Path); + + + if (!await filter(folder)) + break; // excluded + + var parentPath = Utility.Utility.GetParent(folder.Path, true); + if (string.IsNullOrWhiteSpace(parentPath)) + break; + + folder = snapshot.GetFilesystemEntry(parentPath, true); + } + + if (folder != null) + { + // update cache + parents?.ForEach(p => cache[p] = false); + } + + return folder != null; + } +} + diff --git a/Duplicati/Library/Snapshots/SnapshotBase.cs b/Duplicati/Library/Snapshots/SnapshotBase.cs index f5d6da147..1c6151137 100644 --- a/Duplicati/Library/Snapshots/SnapshotBase.cs +++ b/Duplicati/Library/Snapshots/SnapshotBase.cs @@ -20,10 +20,11 @@ // DEALINGS IN THE SOFTWARE. using System; -using System.Linq; using System.Collections.Generic; using System.IO; -using Duplicati.Library.Common.IO; +using Duplicati.Library.Interface; +using System.Threading.Tasks; +using System.Threading; namespace Duplicati.Library.Snapshots { @@ -40,19 +41,56 @@ namespace Duplicati.Library.Snapshots Dispose(false); } - #region ISnapshotService - /// - /// Enumerates all files and folders in the shadow copy + /// Initializes a new instance of the class. /// - /// Source paths to enumerate - /// The callback to invoke with each found path - /// The callback used to report errors - public virtual IEnumerable EnumerateFilesAndFolders(IEnumerable sources, Utility.Utility.EnumerationFilterDelegate callback, Utility.Utility.ReportAccessError errorCallback) + /// A flag indicating if symlinks should be followed + protected SnapshotBase(bool followSymlinks) { - return sources.SelectMany(s => Utility.Utility.EnumerateFileSystemEntries(s, callback, ListFolders, ListFiles, GetAttributes, errorCallback)); + FollowSymlinks = followSymlinks; } - + + #region ISnapshotService + + protected bool FollowSymlinks { get; } + + /// + /// Gets the source folders + /// + public abstract IEnumerable SourceFolders { get; } + + /// + /// Enuemrates the root source files and folders + /// + /// The source files and folders + public abstract IEnumerable EnumerateFilesystemEntries(); + + /// + /// Gets a filesystem entry for a given path + /// + /// The path to get the entry for + /// A flag indicating if the path is a folder + /// The filesystem entry + public virtual ISourceFileEntry GetFilesystemEntry(string path, bool isFolder) + => new SnapshotSourceFileEntry(this, path, isFolder, false); + + /// + /// Enumerates files and folders in the given folder + /// + /// Source to enumerate + /// The files and folders in the given folder + public virtual IEnumerable EnumerateFilesystemEntries(ISourceFileEntry source) + { + if (source.IsFolder) + { + foreach (var f in ListFiles(source.Path)) + yield return new SnapshotSourceFileEntry(this, f, false, false); + + foreach (var d in ListFolders(source.Path)) + yield return new SnapshotSourceFileEntry(this, d, true, false); + } + } + /// /// Gets the last write time of a given file in UTC /// @@ -72,7 +110,7 @@ namespace Duplicati.Library.Snapshots { return File.GetCreationTimeUtc(localPath); } - + /// /// Opens a file for reading /// @@ -83,6 +121,15 @@ namespace Duplicati.Library.Snapshots return File.Open(localPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); } + /// + /// Opens a file for reading + /// + /// The full path to the file in non-snapshot format + /// A token that can be used to cancel the operation + /// An open filestream that can be read + public virtual Task OpenReadAsync(string localPath, CancellationToken cancellationToken) + => Task.FromResult(OpenRead(localPath)); + /// /// Returns the size of a file /// @@ -116,9 +163,8 @@ namespace Duplicati.Library.Snapshots /// The metadata for the given file or folder /// The file or folder to examine /// A flag indicating if the target is a symlink - /// A flag indicating if a symlink should be followed - public abstract Dictionary GetMetadata(string localPath, bool isSymlink, bool followSymlink); - + public abstract Dictionary GetMetadata(string localPath, bool isSymlink); + /// /// Gets a value indicating if the path points to a block device /// @@ -128,7 +174,7 @@ namespace Duplicati.Library.Snapshots { return false; } - + /// /// Gets a unique hardlink target ID /// @@ -157,15 +203,6 @@ namespace Duplicati.Library.Snapshots return Directory.Exists(ConvertToSnapshotPath(localFolderPath)); } - /// - public virtual bool PathExists(string localFileOrFolderPath) - { - return FileExists(localFileOrFolderPath) || DirectoryExists(localFileOrFolderPath); - } - - /// - public virtual bool IsSnapshot => false; - #endregion /// @@ -198,7 +235,7 @@ namespace Duplicati.Library.Snapshots // Now suppress finalization for this object, since we've already handled our resource cleanup tasks GC.SuppressFinalize(this); - } + } #endregion diff --git a/Duplicati/Library/Snapshots/SnapshotSourceFileEntry.cs b/Duplicati/Library/Snapshots/SnapshotSourceFileEntry.cs new file mode 100644 index 000000000..932963611 --- /dev/null +++ b/Duplicati/Library/Snapshots/SnapshotSourceFileEntry.cs @@ -0,0 +1,143 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Duplicati.Library.Interface; + +namespace Duplicati.Library.Snapshots; + + +/// +/// A snapshot-based source file entry +/// +/// The snapshot service +/// The path of the file +/// True if the entry is a folder, false otherwise +/// True if the entry is the root entry, false otherwise +public class SnapshotSourceFileEntry(ISnapshotService service, string path, bool isFolder, bool isRoot) : ISourceFileEntry +{ + /// + /// The symlink target + /// + private string? symlinkTarget; + /// + /// The file attributes + /// + private FileAttributes? attributes; + /// + /// Gets the symlink target or null if the entry is not a symlink + /// + public string? SymlinkTarget + { + get + { + if (symlinkTarget != null) + return symlinkTarget; + + if (!service.IsSymlink(path, Attributes)) + return null; + + return symlinkTarget = service.GetSymlinkTarget(path); + } + } + + /// + /// True if the entry is a symlink, false otherwise + /// + public bool IsSymlink => SymlinkTarget != null; + + /// + /// Gets the entry attributes + /// + public FileAttributes Attributes + { + get + { + if (attributes != null) + return attributes.Value; + + attributes = service.GetAttributes(path); + return attributes.Value; + } + } + + /// + /// Gets the minor metadata for the file + /// + public Dictionary MinorMetadata => service.GetMetadata(path, IsSymlink); + + /// + /// Gets a value indicating if the entry is a block device + /// + public bool IsBlockDevice => service.IsBlockDevice(path); + + /// + /// Gets a value indicating if the entry is a character device + /// + public bool IsCharacterDevice => false; + + /// + /// Gets a value indicating if the entry is an alternate stream + /// + public bool IsAlternateStream => false; + + /// + /// Gets the hardlink target ID, or null if the entry is not a hardlink + /// + public string? HardlinkTargetId => service.HardlinkTargetID(path); + + /// + public bool IsFolder => isFolder; + + /// + public bool IsMetaEntry => false; + /// + public bool IsRootEntry => isRoot; + + /// + public DateTime CreatedUtc => service.GetCreationTimeUtc(path); + + /// + public DateTime LastModificationUtc => service.GetLastWriteTimeUtc(path); + + /// + public string Path => path; + + /// + public long Size => IsFolder ? -1 : service.GetFileSize(path); + + /// + public Task OpenRead(CancellationToken cancellationToken) => service.OpenReadAsync(path, cancellationToken); + + /// + public Task OpenMetadataRead(CancellationToken cancellationToken) => Task.FromResult(new MemoryStream(System.Text.Json.JsonSerializer.SerializeToUtf8Bytes(MinorMetadata))); + /// + public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) => service.EnumerateFilesystemEntries(this).ToAsyncEnumerable(); + /// + public Task FileExists(string filename, CancellationToken cancellationToken) => Task.FromResult(service.FileExists(System.IO.Path.Combine(path, filename))); +} \ No newline at end of file diff --git a/Duplicati/Library/Snapshots/SnapshotUtility.cs b/Duplicati/Library/Snapshots/SnapshotUtility.cs index 7ea314b14..10e9d6c6e 100644 --- a/Duplicati/Library/Snapshots/SnapshotUtility.cs +++ b/Duplicati/Library/Snapshots/SnapshotUtility.cs @@ -1,30 +1,29 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; using System.IO; using System.Runtime.Versioning; -using Duplicati.Library.Common; using Duplicati.Library.Common.IO; namespace Duplicati.Library.Snapshots @@ -37,24 +36,42 @@ namespace Duplicati.Library.Snapshots /// /// Loads a snapshot implementation for the current OS /// - /// The list of folders to create snapshots of + /// The list of paths to create snapshots of /// A set of commandline options + /// Whether to follow symlinks /// The ISnapshotService implementation - public static ISnapshotService CreateSnapshot(IEnumerable folders, Dictionary options) + public static ISnapshotService CreateSnapshot(IEnumerable paths, Dictionary options, bool followSymlinks) { - if (OperatingSystem.IsMacOS() || OperatingSystem.IsLinux()) + if (OperatingSystem.IsLinux()) { - return CreateLinuxSnapshot(folders); + return CreateLinuxSnapshot(paths, followSymlinks); } else if (OperatingSystem.IsWindows()) { - return CreateWindowsSnapshot(folders, options); + return CreateWindowsSnapshot(paths, options, followSymlinks); } else { throw new NotSupportedException("Unsupported Operating System"); } - + } + + /// + /// Creates a snapshot implementation that does not use snapshots (i.e., regular file access) + /// + /// The list of paths to create snapshots of + /// Flag to ignore advisory locking + /// Whether to follow symlinks + /// The ISnapshotService implementation + public static ISnapshotService CreateNoSnapshot(IEnumerable paths, bool ignoreAdvisoryLocking, bool followSymlinks) + { + if (OperatingSystem.IsMacOS() || OperatingSystem.IsLinux()) + return new NoSnapshotLinux(paths, ignoreAdvisoryLocking, followSymlinks); + else if (OperatingSystem.IsWindows()) + return new NoSnapshotWindows(paths, followSymlinks); + else + throw new NotSupportedException("Unsupported Operating System"); + } // The two loader methods below guard against the type system attempting to load types @@ -65,13 +82,14 @@ namespace Duplicati.Library.Snapshots /// Loads a snapshot implementation for Linux /// /// The list of folders to create snapshots of + /// Whether to follow symlinks /// The ISnapshotService implementation [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] [SupportedOSPlatform("linux")] [SupportedOSPlatform("macOS")] - private static ISnapshotService CreateLinuxSnapshot(IEnumerable folders) + private static ISnapshotService CreateLinuxSnapshot(IEnumerable folders, bool followSymlinks) { - return new LinuxSnapshot(folders); + return new LinuxSnapshot(folders, followSymlinks); } /// @@ -79,23 +97,13 @@ namespace Duplicati.Library.Snapshots /// /// The list of folders to create snapshots of /// A set of commandline options + /// Whether to follow symlinks /// The ISnapshotService implementation [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] [SupportedOSPlatform("windows")] - private static ISnapshotService CreateWindowsSnapshot(IEnumerable folders, Dictionary options) + private static ISnapshotService CreateWindowsSnapshot(IEnumerable folders, Dictionary options, bool followSymlinks) { - return new WindowsSnapshot(folders, options); - } - - /// - /// Extension method for ISnapshotService which determines whether the given path is a symlink. - /// - /// ISnapshotService implementation - /// File or folder path - /// Whether the path is a symlink - public static bool IsSymlink(this ISnapshotService snapshot, string path) - { - return snapshot.IsSymlink(path, snapshot.GetAttributes(path)); + return new WindowsSnapshot(folders, options, followSymlinks); } /// diff --git a/Duplicati/Library/Snapshots/UsnJournalService.cs b/Duplicati/Library/Snapshots/UsnJournalService.cs index 419dc802e..47cd08baa 100644 --- a/Duplicati/Library/Snapshots/UsnJournalService.cs +++ b/Duplicati/Library/Snapshots/UsnJournalService.cs @@ -40,25 +40,31 @@ namespace Duplicati.Library.Snapshots /// private static readonly string FILTER_LOGTAG = Logging.Log.LogTagFromType(typeof(UsnJournalService)); + /// + /// The snapshot service + /// private readonly ISnapshotService m_snapshot; - private readonly IEnumerable m_sources; + /// + /// The volume data dictionary + /// private readonly Dictionary m_volumeDataDict; + /// + /// The cancellation token + /// private readonly CancellationToken m_token; /// /// Constructor. /// - /// Sources to filter /// /// Emit filter /// /// /// Journal-data of previous fileset /// - public UsnJournalService(IEnumerable sources, ISnapshotService snapshot, IFilter emitFilter, FileAttributes fileAttributeFilter, + public UsnJournalService(ISnapshotService snapshot, IFilter emitFilter, FileAttributes fileAttributeFilter, long skipFilesLargerThan, IEnumerable prevJournalData, CancellationToken token) { - m_sources = sources; m_snapshot = snapshot; m_volumeDataDict = Initialize(emitFilter, fileAttributeFilter, skipFilesLargerThan, prevJournalData); m_token = token; @@ -85,7 +91,7 @@ namespace Duplicati.Library.Snapshots // get hash identifying current source filter / sources configuration var configHash = Utility.Utility.ByteArrayAsHexString(MD5HashHelper.GetHash(new string[] { emitFilter == null ? string.Empty : emitFilter.ToString(), - string.Join("; ", m_sources), + string.Join("; ", m_snapshot.SourceFolders), fileAttributeFilter.ToString(), skipFilesLargerThan.ToString() })); @@ -94,7 +100,7 @@ namespace Duplicati.Library.Snapshots var journalDataDict = prevJournalData.ToDictionary(data => data.Volume); // iterate over volumes - foreach (var sourcesPerVolume in SortByVolume(m_sources)) + foreach (var sourcesPerVolume in SortByVolume(m_snapshot.SourceFolders)) { Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "UsnInitialize", "Reading USN journal for volume: {0}", sourcesPerVolume.Key); @@ -180,8 +186,7 @@ namespace Duplicati.Library.Snapshots // Since that folder is going to be fully scanned, those files can be removed. // Note: it would be wrong to use the result from step 2. as the folder list! The entries removed // between 1. and 2. are *excluded* folders, and files below them are to be *excluded*, too. - volumeData.Files = - new HashSet(Utility.Utility.GetFilesNotInFolders(changedFiles, volumeData.Folders)); + volumeData.Files = [.. Utility.Utility.GetFilesNotInFolders(changedFiles, volumeData.Folders)]; // Record success for volume volumeData.IsFullScan = false; @@ -213,216 +218,11 @@ namespace Duplicati.Library.Snapshots return result; } - /// - /// Filters sources, returning sub-set having been modified since last - /// change, as specified by journalData. - /// - /// Filter callback to exclude filtered items - /// Filtered sources - public IEnumerable GetModifiedSources(Utility.Utility.EnumerationFilterDelegate filter) - { - // iterate over volumes - foreach (var volumeData in m_volumeDataDict) - { - // prepare cache for includes (value = true) and excludes (value = false, will be populated - // on-demand) - var cache = new Dictionary(); - foreach (var source in m_sources) - { - if (m_token.IsCancellationRequested) - { - break; - } - - cache[source] = true; - } - - // Check the simplified folders, and their parent folders against the exclusion filter. - // This is needed because the filter may exclude "C:\A\", but this won't match the more - // specific "C:\A\B\" in our list, even though it's meant to be excluded. - // The reason why the filter doesn't exclude it is because during a regular (non-USN) full scan, - // FilterHandler.EnumerateFilesAndFolders() works top-down, and won't even enumerate child - // folders. - // The sources are needed to stop evaluating parent folders above the specified source folders - if (volumeData.Value.Folders != null) - { - foreach (var folder in FilterExcludedFolders(volumeData.Value.Folders, filter, cache).Where(m_snapshot.DirectoryExists)) - { - if (m_token.IsCancellationRequested) - { - break; - } - - yield return folder; - } - } - - // The simplified file list also needs to be checked against the exclusion filter, as it - // may contain entries excluded due to attributes, but also because they are below excluded - // folders, which themselves aren't in the folder list from step 1. - // Note that the simplified file list may contain entries that have been deleted! They need to - // be kept in the list (unless excluded by the filter) in order for the backup handler to record their - // deletion. - if (volumeData.Value.Files != null) - { - foreach (var files in FilterExcludedFiles(volumeData.Value.Files, filter, cache).Where(m_snapshot.FileExists)) - { - if (m_token.IsCancellationRequested) - { - break; - } - - yield return files; - } - } - } - } - - /// - /// Filter supplied files, removing any files which itself, or one - /// of its parent folders, is excluded by the filter. - /// - /// Files to filter - /// Exclusion filter - /// Cache of included and excluded files / folders - /// - /// Filtered files - private IEnumerable FilterExcludedFiles(IEnumerable files, - Utility.Utility.EnumerationFilterDelegate filter, IDictionary cache, Utility.Utility.ReportAccessError errorCallback = null) - { - var result = new List(); - - foreach (var file in files) - { - if (m_token.IsCancellationRequested) - { - break; - } - - var attr = m_snapshot.FileExists(file) ? m_snapshot.GetAttributes(file) : FileAttributes.Normal; - try - { - if (!filter(file, file, attr)) - continue; - - if (!IsFolderOrAncestorsExcluded(Utility.Utility.GetParent(file, true), filter, cache)) - { - result.Add(file); - } - } - catch (System.Threading.ThreadAbortException) - { - throw; - } - catch (Exception ex) - { - errorCallback?.Invoke(file, file, ex); - filter(file, file, attr | Utility.Utility.ATTRIBUTE_ERROR); - } - } - - return result; - } - - /// - /// Filter supplied folders, removing any folder which itself, or one - /// of its ancestors, is excluded by the filter. - /// - /// Folder to filter - /// Exclusion filter - /// Cache of excluded folders (optional) - /// - /// Filtered folders - private IEnumerable FilterExcludedFolders(IEnumerable folders, - Utility.Utility.EnumerationFilterDelegate filter, IDictionary cache, Utility.Utility.ReportAccessError errorCallback = null) - { - var result = new List(); - - foreach (var folder in folders) - { - if (m_token.IsCancellationRequested) - { - break; - } - - try - { - if (!IsFolderOrAncestorsExcluded(folder, filter, cache)) - { - result.Add(folder); - } - } - catch (System.Threading.ThreadAbortException) - { - throw; - } - catch (Exception ex) - { - errorCallback?.Invoke(folder, folder, ex); - filter(folder, folder, FileAttributes.Directory | Utility.Utility.ATTRIBUTE_ERROR); - } - } - - return result; - } - - /// - /// Tests if specified folder, or any of its ancestors, is excluded by the filter - /// - /// Folder to test - /// Filter - /// Cache of excluded folders (optional) - /// True if excluded, false otherwise - private bool IsFolderOrAncestorsExcluded(string folder, Utility.Utility.EnumerationFilterDelegate filter, IDictionary cache) - { - List parents = null; - while (folder != null) - { - if (m_token.IsCancellationRequested) - { - break; - } - - // first check cache - if (cache.TryGetValue(folder, out var include)) - { - if (include) - return false; - - break; // hit! - } - - // remember folder for cache - if (parents == null) - { - parents = new List(); // create on-demand - } - parents.Add(folder); - - - var attr = m_snapshot.DirectoryExists(folder) ? m_snapshot.GetAttributes(folder) : FileAttributes.Directory; - - if (!filter(folder, folder, attr)) - break; // excluded - - folder = Utility.Utility.GetParent(folder, true); - } - - if (folder != null) - { - // update cache - parents?.ForEach(p => cache[p] = false); - } - - return folder != null; - } - /// /// Sort sources by root volume /// /// List of sources /// Dictionary of volumes, with list of sources as values - [SupportedOSPlatform("windows")] private static Dictionary> SortByVolume(IEnumerable sources) { var sourcesByVolume = new Dictionary>(); @@ -448,7 +248,6 @@ namespace Duplicati.Library.Snapshots /// /// /// - [SupportedOSPlatform("windows")] public bool IsPathEnumerated(string path) { // get NTFS volume root diff --git a/Duplicati/Library/Snapshots/WindowsSnapshot.cs b/Duplicati/Library/Snapshots/WindowsSnapshot.cs index 65df03a2c..ae36027ec 100644 --- a/Duplicati/Library/Snapshots/WindowsSnapshot.cs +++ b/Duplicati/Library/Snapshots/WindowsSnapshot.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. @@ -26,6 +26,7 @@ using System.IO; using System.Linq; using System.Runtime.Versioning; using Duplicati.Library.Common.IO; +using Duplicati.Library.Interface; namespace Duplicati.Library.Snapshots { @@ -54,7 +55,9 @@ namespace Duplicati.Library.Snapshots /// /// Sources to determine which volumes to include in snapshot /// A set of commandline options - public WindowsSnapshot(IEnumerable sources, IDictionary options) + /// A flag indicating if symlinks should be followed + public WindowsSnapshot(IEnumerable sources, IDictionary options, bool followSymlinks) + : base(followSymlinks) { // For Windows, ensure we don't store paths with extended device path prefixes (i.e., @"\\?\" or @"\\?\UNC\") sources = sources.Select(SystemIOWindows.RemoveExtendedDevicePathPrefix); @@ -103,6 +106,26 @@ namespace Duplicati.Library.Snapshots } } + /// + /// Gets the source folders + /// + public override IEnumerable SourceFolders => _vssBackupComponents.SnapshotLocalPaths; + + /// + /// Enumerates the root source files and folders + /// + /// The source files and folders + public override IEnumerable EnumerateFilesystemEntries() + { + foreach (var folder in _vssBackupComponents.SnapshotLocalPaths) + { + if (folder.EndsWith(Path.DirectorySeparatorChar) || DirectoryExists(folder)) + yield return new SnapshotSourceFileEntry(this, folder, true, true); + else + yield return new SnapshotSourceFileEntry(this, folder, false, true); + } + } + #region Private functions /// @@ -158,18 +181,6 @@ namespace Duplicati.Library.Snapshots #region ISnapshotService Members - /// - /// Enumerates all files and folders in the snapshot, restricted to sources - /// - /// Sources to enumerate - /// The callback to invoke with each found path - /// The callback used to report errors - public override IEnumerable EnumerateFilesAndFolders(IEnumerable sources, Utility.Utility.EnumerationFilterDelegate callback, Utility.Utility.ReportAccessError errorCallback) - { - // For Windows, ensure we don't store paths with extended device path prefixes (i.e., @"\\?\" or @"\\?\UNC\") - return base.EnumerateFilesAndFolders(sources.Select(SystemIOWindows.RemoveExtendedDevicePathPrefix), callback, errorCallback); - } - /// /// Gets the last write time of a given file in UTC /// @@ -241,23 +252,16 @@ namespace Duplicati.Library.Snapshots /// The metadata for the given file or folder /// The file or folder to examine /// A flag indicating if the target is a symlink - /// A flag indicating if a symlink should be followed - public override Dictionary GetMetadata(string localPath, bool isSymlink, bool followSymlink) + public override Dictionary GetMetadata(string localPath, bool isSymlink) { - return SystemIO.IO_WIN.GetMetadata(ConvertToSnapshotPath(localPath), isSymlink, followSymlink); + return SystemIO.IO_WIN.GetMetadata(ConvertToSnapshotPath(localPath), isSymlink, FollowSymlinks); } /// - public override bool IsBlockDevice(string localPath) - { - return false; - } + public override bool IsBlockDevice(string localPath) => false; /// - public override string HardlinkTargetID(string localPath) - { - return null; - } + public override string HardlinkTargetID(string localPath) => null; /// public override string ConvertToLocalPath(string snapshotPath) @@ -312,9 +316,6 @@ namespace Duplicati.Library.Snapshots return SystemIO.IO_WIN.DirectoryExists(ConvertToSnapshotPath(localFolderPath)); } - /// - public override bool IsSnapshot => true; - #endregion /// diff --git a/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs new file mode 100644 index 000000000..54ad99034 --- /dev/null +++ b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs @@ -0,0 +1,211 @@ +// Copyright (C) 2024, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using Duplicati.Library.Interface; +using Duplicati.Library.Utility; + +namespace Duplicati.Library.SourceProvider; + +/// +/// A source provider that wraps a backend +/// +/// The backend to wrap +public class BackendSourceProvider : ISourceProvider +{ + /// + /// The wrapped backend + /// + public IBackend WrappedBackend { get; init; } + + /// + /// The path key unique for the destination + /// + public readonly string PathKey; + + /// + /// Creates a new backend source provider + /// + /// The backend to wrap + /// The URL of the backend + public BackendSourceProvider(IBackend backend, string url) + { + WrappedBackend = backend; + var uri = new Utility.Uri(url); + // TODO: The username may be provided in other ways to the backend... + PathKey = $"{uri.Scheme}://{uri.Host}:{uri.Port}/~{uri.Username}/{uri.Path}/"; + } + + /// + /// Creates a new backend source entry + /// + /// The parent backend + /// The path of the entry + /// True if the entry is a folder + /// True if the entry is a meta entry + /// The creation time of the entry + /// The last modification time of the entry + /// The size of the entry + private class BackendSourceFileEntry(BackendSourceProvider parent, string path, bool isFolder, bool isMetaEntry, DateTime createdUtc, DateTime lastModificationUtc, long size) + : ISourceFileEntry + { + /// + public bool IsFolder => isFolder; + + /// + public bool IsMetaEntry => isMetaEntry; + + /// + public bool IsRootEntry => isMetaEntry; + + /// + public DateTime CreatedUtc => createdUtc; + + /// + public DateTime LastModificationUtc => lastModificationUtc; + + /// + public string Path => $"{parent.PathKey}{path}"; + + /// + public long Size => size; + /// + public bool IsSymlink => false; + + /// + public string? SymlinkTarget => null; + + /// + public FileAttributes Attributes => IsFolder + ? FileAttributes.Directory + : FileAttributes.Normal; + + /// + public Dictionary MinorMetadata => new Dictionary(); + + /// + public bool IsBlockDevice => false; + + /// + public bool IsCharacterDevice => false; + + /// + public bool IsAlternateStream => false; + + /// + public string? HardlinkTargetId => null; + + /// + public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) + { + if (!isFolder) + throw new InvalidOperationException("Enumerate can only be called on folders"); + + if (parent.WrappedBackend is IFolderEnabledBackend folderEnabledBackend) + { + return folderEnabledBackend.ListAsync(path, cancellationToken) + .Select(x => new BackendSourceFileEntry(parent, x.Path, x.IsFolder, false, new DateTime(0), x.LastModificationUtc, x.Size)); + } + else + { + + // If we do not support folders, we can only list the root + if (this.IsMetaEntry) + return parent.WrappedBackend.List() + .Select(x => new BackendSourceFileEntry(parent, x.Name, x.IsFolder, false, new DateTime(0), x.LastModification, x.Size)) + .ToAsyncEnumerable(); + + return Array.Empty().ToAsyncEnumerable(); + } + } + + /// + public async Task FileExists(string path, CancellationToken cancellationToken) + { + if (!isFolder) + throw new InvalidOperationException("FileExists can only be called on folders"); + + if (parent.WrappedBackend is IFolderEnabledBackend folderEnabledBackend) + { + try + { + var entry = await folderEnabledBackend.GetEntryAsync(path, cancellationToken); + return entry != null && !entry.IsFolder; + } + catch (FileNotFoundException) + { + return false; + } + } + + return false; + } + + /// + public Task OpenMetadataRead(CancellationToken cancellationToken) + => Task.FromResult(null); + + /// + public async Task OpenRead(CancellationToken cancellationToken) + { + if (isFolder) + throw new InvalidOperationException("OpenRead can only be called on files"); + + if (parent.WrappedBackend is IStreamingBackend streamingBackend) + { + var mapStream = new MapStream(); + mapStream.CopyTask = streamingBackend.GetAsync(this.Path, mapStream, cancellationToken); + return mapStream; + } + else + { + TempFileStream? file = null; + try + { + file = TempFileStream.Create(); + await parent.WrappedBackend.GetAsync(this.Path, file.Path, cancellationToken); + file.Position = 0; + return file; + } + catch + { + file?.Dispose(); + throw; + } + } + } + } + + /// + public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) + => new[] { new BackendSourceFileEntry(this, "", true, true, new DateTime(0), new DateTime(0), 0) }.ToAsyncEnumerable(); + + /// + public Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) + // TODO: If we cache the enumerate result, we could check if the path exists in the cache and return it + => Task.FromResult(new BackendSourceFileEntry(this, path, isFolder, false, new DateTime(0), new DateTime(0), -1)); + + /// + public void Dispose() + { + WrappedBackend.Dispose(); + GC.SuppressFinalize(this); + } +} diff --git a/Duplicati/Library/SourceProvider/Builtin/Combiner.cs b/Duplicati/Library/SourceProvider/Builtin/Combiner.cs new file mode 100644 index 000000000..5ef6496ba --- /dev/null +++ b/Duplicati/Library/SourceProvider/Builtin/Combiner.cs @@ -0,0 +1,82 @@ +// Copyright (C) 2024, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using System.Runtime.CompilerServices; +using Duplicati.Library.Interface; + +namespace Duplicati.Library.SourceProvider; + +/// +/// Helper class that wraps multiple source providers into a single source provider +/// +public class Combiner(IEnumerable providers) : ISourceProvider +{ + /// + /// The providers to combine + /// + private readonly List providers = providers.SelectMany(x => x is Combiner c ? c.providers.AsEnumerable() : [x]).ToList(); + + /// + public async IAsyncEnumerable Enumerate([EnumeratorCancellation] CancellationToken cancellationToken) + { + foreach (var provider in providers) + { + await foreach (var entry in provider.Enumerate(cancellationToken).ConfigureAwait(false)) + yield return entry; + } + } + + /// + /// Provides access to the combined providers + /// + public IEnumerable Providers => providers; + + /// + /// Gets a filesystem entry for a given path + /// + /// The path to get + /// True if the path is a folder + /// The cancellation token + /// The filesystem entry + public Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) + { + // TODO: Map roots to the paths so we get the right provider + throw new NotImplementedException(); + } + + /// + /// Creates a combined source provider from a list of providers + /// + /// The providers to combine + /// The combined provider + public static ISourceProvider Combine(IEnumerable providers) + { + var providerTotal = providers.SelectMany(x => x is Combiner c ? c.providers.AsEnumerable() : [x]).ToList(); + return providerTotal.Count == 1 ? providerTotal[0] : new Combiner(providers); + } + + /// + public void Dispose() + { + foreach (var provider in providers) + provider.Dispose(); + } +} diff --git a/Duplicati/Library/SourceProvider/Builtin/Duplicati.Library.SourceProvider.Builtin.csproj b/Duplicati/Library/SourceProvider/Builtin/Duplicati.Library.SourceProvider.Builtin.csproj new file mode 100644 index 000000000..4ba5377b7 --- /dev/null +++ b/Duplicati/Library/SourceProvider/Builtin/Duplicati.Library.SourceProvider.Builtin.csproj @@ -0,0 +1,27 @@ + + + + net8.0 + Library + Duplicati.Library.SourceProvider + Copyright © 2024 Team Duplicati, MIT license + enable + enable + Duplicati.Library.SourceProvider + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + diff --git a/Duplicati/Library/SourceProvider/Builtin/File.cs b/Duplicati/Library/SourceProvider/Builtin/File.cs new file mode 100644 index 000000000..82e43997a --- /dev/null +++ b/Duplicati/Library/SourceProvider/Builtin/File.cs @@ -0,0 +1,52 @@ +// Copyright (C) 2024, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using Duplicati.Library.Interface; +using Duplicati.Library.Snapshots; + +namespace Duplicati.Library.SourceProvider; + +/// +/// A source provider that wraps a file system through a snapshot service +/// +/// The snapshot service to use +public class File(ISnapshotService snapshotService) : ISourceProvider +{ + /// + public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) + => snapshotService.EnumerateFilesystemEntries().ToAsyncEnumerable(); + + /// + /// The snapshot service being used + /// + public ISnapshotService SnapshotService => snapshotService; + + /// + public Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) + => Task.FromResult(snapshotService.GetFilesystemEntry(path, isFolder)); + + /// + public void Dispose() + { + snapshotService.Dispose(); + GC.SuppressFinalize(this); + } +} diff --git a/Duplicati/Library/SourceProvider/Builtin/Strings.cs b/Duplicati/Library/SourceProvider/Builtin/Strings.cs new file mode 100644 index 000000000..5707fccd5 --- /dev/null +++ b/Duplicati/Library/SourceProvider/Builtin/Strings.cs @@ -0,0 +1,29 @@ +// Copyright (C) 2024, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +using Duplicati.Library.Localization.Short; +namespace Duplicati.Library.SourceProvider.Strings +{ + internal static class File + { + public static string Description { get { return LC.L(@"This backend can read and write data to an file based backend. Allowed formats are ""file://hostname/folder"" and ""file://username:password@hostname/folder"". You may supply UNC paths (e.g.: ""file://\\server\folder"") or local paths (e.g.: (win) ""file://c:\folder"", (linux) ""file:///usr/pub/files""). Use the prefix ""vss://"" or ""lvm://"" to create snapshot based folders."); } } + public static string DisplayName { get { return LC.L(@"Local folder or drive"); } } + } +} diff --git a/Duplicati/Library/SourceProviders/Duplicati.Library.SourceProviders.csproj b/Duplicati/Library/SourceProviders/Duplicati.Library.SourceProviders.csproj new file mode 100644 index 000000000..64dae938d --- /dev/null +++ b/Duplicati/Library/SourceProviders/Duplicati.Library.SourceProviders.csproj @@ -0,0 +1,14 @@ + + + net8.0 + Central reference to all source providers + Duplicati.Library.Backend + Copyright © 2024 Team Duplicati, MIT license + enable + enable + + + + + + diff --git a/Duplicati/Library/SourceProviders/SourceProviderModules.cs b/Duplicati/Library/SourceProviders/SourceProviderModules.cs new file mode 100644 index 000000000..1bf09d104 --- /dev/null +++ b/Duplicati/Library/SourceProviders/SourceProviderModules.cs @@ -0,0 +1,45 @@ +// Copyright (C) 2024, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using Duplicati.Library.Interface; + +namespace Duplicati.Library.SourceProviders; + +/// +/// Shared list of statically linked source-provider modules +/// +public static class SourceProviderModules +{ + /// + /// The list of all built-in source-provider modules + /// + public static IReadOnlyList BuiltInSourceProviderModules => SupportedSourceProviders; + + /// + /// Calculate list once and cache it + /// + private static readonly IReadOnlyList SupportedSourceProviders = new ISourceProviderModule[] { + // ... No providers currently implemented + } + .Where(x => x != null) + .ToList(); +} + diff --git a/Duplicati/Library/Utility/MapStream.cs b/Duplicati/Library/Utility/MapStream.cs new file mode 100644 index 000000000..b2c717739 --- /dev/null +++ b/Duplicati/Library/Utility/MapStream.cs @@ -0,0 +1,176 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +using System; +using System.IO; +using System.Threading; +using System.Threading.Channels; +using System.Threading.Tasks; + +namespace Duplicati.Library.Utility; + +/// +/// A stream that maps data with an internal buffer. +/// This allows a reader and a writer to share a stream without blocking. +/// +public class MapStream : Stream +{ + /// + /// The channel used to communicate between the reader and writer. + /// + private readonly Channel channel = Channel.CreateBounded(0); + + /// + /// The buffer used to store data that has been written but not consumed by the reader. + /// + private Memory readBuffer = Array.Empty(); + + /// + /// The task that is using the map stream. + /// + public Task? CopyTask { get; set; } + + /// + public override bool CanRead => true; + + /// + public override bool CanSeek => false; + + /// + public override bool CanWrite => true; + + /// + public override long Length => throw new NotImplementedException(); + + /// + public override long Position { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } + + /// + public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + int read = 0; + // If we have left-over data from the previous read, return it + if (readBuffer.Length > 0) + { + // Return as much as possible + var requested = Math.Min(readBuffer.Length, count); + readBuffer.Span.Slice(0, requested).CopyTo(buffer.AsSpan(offset, count)); + // Adjust the remaining buffer + if (readBuffer.Length > requested) + readBuffer = readBuffer.Slice(requested); + else + readBuffer = Array.Empty(); + + // Adjust the counters + offset += requested; + count -= requested; + read += requested; + + // Return if we have read all that was requested + if (count == 0) + return read; + } + + // Handle external task + var ct = CopyTask; + + // If the task is completed, clear it + if (ct != null && ct.IsCompletedSuccessfully) + CopyTask = ct = null; + // If the task failed, stop reading + else if (ct != null && (ct.IsCanceled || ct.IsFaulted)) + { + // If we have any data in the buffer, return it + if (read > 0) + return read; + + // Otherwise, throw the exception + await ct; + } + + if (!await channel.Reader.WaitToReadAsync(cancellationToken)) + { + // If we have any data in the buffer, return it + if (read > 0) + return read; + + // If the copy crashed, throw the exception + ct = CopyTask; + if (ct != null && (ct.IsCanceled || ct.IsFaulted)) + await ct; + + // Otherwise, return 0 + return read; + } + + // Read the data from the channel + var res = await channel.Reader.ReadAsync(cancellationToken); + + // Consume as much as was requested + var consumed = Math.Min(res.Length, count); + + // Keep the rest in the buffer + if (res.Length > count) + readBuffer = res.AsMemory(count); + + res.AsSpan(0, consumed).CopyTo(buffer.AsSpan(offset, count)); + read += consumed; + return read; + } + + /// + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + var ct = CopyTask; + // Throw if the task failed + if (ct != null && (ct.IsCanceled || ct.IsFaulted)) + await ct; + await channel.Writer.WriteAsync(buffer.AsSpan(offset, count).ToArray(), cancellationToken); + } + + public override void Close() + { + base.Close(); + channel.Writer.Complete(); + } + + protected override void Dispose(bool disposing) + { + channel.Writer.Complete(); + base.Dispose(disposing); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) + => ReadAsync(buffer, offset, count, CancellationToken.None).Await(); + + public override long Seek(long offset, SeekOrigin origin) + => throw new NotImplementedException(); + + public override void SetLength(long value) + => throw new NotImplementedException(); + + public override void Write(byte[] buffer, int offset, int count) + => WriteAsync(buffer, offset, count, CancellationToken.None).Await(); +} diff --git a/Duplicati/Library/Utility/TempFileStream.cs b/Duplicati/Library/Utility/TempFileStream.cs new file mode 100644 index 000000000..104eab261 --- /dev/null +++ b/Duplicati/Library/Utility/TempFileStream.cs @@ -0,0 +1,87 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace Duplicati.Library.Utility; + +/// +/// A stream that wraps a temporary file. +/// The temporary file is automatically deleted when the stream is disposed. +/// +public class TempFileStream : StreamUtil.WrappingAsyncStream +{ + /// + /// The temporary file instance. + /// + private readonly TempFile tempFile; + /// + /// The stream for the temporary file. + /// + private readonly Stream stream; + + /// + /// The path to the temporary file. + /// + public string Path => tempFile.Name; + + /// + /// Creates a new TempFileStream instance. + /// + /// The temporary file instance. + /// The stream for the temporary file. + private TempFileStream(TempFile tempFile, Stream stream) + : base(stream) + { + this.tempFile = tempFile; + this.stream = stream; + } + + /// + /// Creates a new TempFileStream instance. + /// + /// The new TempFileStream instance. + public static TempFileStream Create() + { + var tempFile = new TempFile(); + return new TempFileStream(tempFile, File.Open(tempFile.Name, FileMode.Open, FileAccess.ReadWrite, FileShare.None)); + } + + /// + protected override Task ReadImplAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + => stream.ReadAsync(buffer, offset, count, cancellationToken); + + /// + protected override Task WriteImplAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + => stream.WriteAsync(buffer, offset, count, cancellationToken); + + /// + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + stream.Dispose(); + tempFile.Dispose(); + } +} diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index 4d2d8e423..c764f3165 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -246,7 +246,7 @@ namespace Duplicati.Library.Utility /// A list of the full filenames and foldernames. Foldernames ends with the directoryseparator char public static IEnumerable EnumerateFileSystemEntries(string basepath) { - return EnumerateFileSystemEntries(basepath, (rootpath, path, attributes) => true, SystemIO.IO_OS.GetDirectories, Directory.GetFiles, null); + return EnumerateFileSystemEntries(basepath, SystemIO.IO_OS.GetDirectories, Directory.GetFiles, null); } /// @@ -276,13 +276,12 @@ namespace Duplicati.Library.Utility /// The search is recursive. /// /// The folder to look in - /// The function to call with the filenames /// A function to call that lists all folders in the supplied folder /// A function to call that lists all files in the supplied folder /// A function to call that obtains the attributes for an element, set to null to avoid reading attributes /// An optional function to call with error messages. /// A list of the full filenames - public static IEnumerable EnumerateFileSystemEntries(string rootpath, EnumerationFilterDelegate callback, FileSystemInteraction folderList, FileSystemInteraction fileList, ExtractFileAttributes? attributeReader, ReportAccessError? errorCallback = null) + public static IEnumerable EnumerateFileSystemEntries(string rootpath, FileSystemInteraction folderList, FileSystemInteraction fileList, ExtractFileAttributes? attributeReader, ReportAccessError? errorCallback = null) { var lst = new Stack(); @@ -292,8 +291,7 @@ namespace Duplicati.Library.Utility try { var attr = attributeReader?.Invoke(rootpath) ?? FileAttributes.Directory; - if (callback(rootpath, rootpath, attr)) - lst.Push(rootpath); + lst.Push(rootpath); } catch (System.Threading.ThreadAbortException) { @@ -302,7 +300,6 @@ namespace Duplicati.Library.Utility catch (Exception ex) { errorCallback?.Invoke(rootpath, rootpath, ex); - callback(rootpath, rootpath, FileAttributes.Directory | ATTRIBUTE_ERROR); } while (lst.Count > 0) @@ -319,8 +316,7 @@ namespace Duplicati.Library.Utility try { var attr = attributeReader?.Invoke(sf) ?? FileAttributes.Directory; - if (callback(rootpath, sf, attr)) - lst.Push(sf); + lst.Push(sf); } catch (System.Threading.ThreadAbortException) { @@ -329,7 +325,6 @@ namespace Duplicati.Library.Utility catch (Exception ex) { errorCallback?.Invoke(rootpath, sf, ex); - callback(rootpath, sf, FileAttributes.Directory | ATTRIBUTE_ERROR); } } } @@ -340,7 +335,6 @@ namespace Duplicati.Library.Utility catch (Exception ex) { errorCallback?.Invoke(rootpath, f, ex); - callback(rootpath, f, FileAttributes.Directory | ATTRIBUTE_ERROR); } string[]? files = null; @@ -357,54 +351,18 @@ namespace Duplicati.Library.Utility catch (Exception ex) { errorCallback?.Invoke(rootpath, f, ex); - callback(rootpath, f, FileAttributes.Directory | ATTRIBUTE_ERROR); } } if (files != null) { foreach (var s in files) - { - try - { - var attr = attributeReader?.Invoke(s) ?? FileAttributes.Normal; - if (!callback(rootpath, s, attr)) - continue; - } - catch (System.Threading.ThreadAbortException) - { - throw; - } - catch (Exception ex) - { - errorCallback?.Invoke(rootpath, s, ex); - callback(rootpath, s, ATTRIBUTE_ERROR); - continue; - } yield return s; - } } } } else { - try - { - var attr = attributeReader?.Invoke(rootpath) ?? FileAttributes.Normal; - if (!callback(rootpath, rootpath, attr)) - yield break; - } - catch (System.Threading.ThreadAbortException) - { - throw; - } - catch (Exception ex) - { - errorCallback?.Invoke(rootpath, rootpath, ex); - callback(rootpath, rootpath, ATTRIBUTE_ERROR); - yield break; - } - yield return rootpath; } } @@ -1591,5 +1549,20 @@ namespace Duplicati.Library.Utility foreach (var iex in FlattenException(ex.InnerException)) yield return iex; } + + /// + /// Guesses the URL scheme and returns it + /// + /// The URL to guess the scheme for + /// The guessed scheme, or null if no scheme was found + public static string? GuessScheme(string url) + { + var idx = url.IndexOf("://"); + if (idx < 0 && idx < 15 && idx + "://".Length < url.Length) + return null; + + return url.Substring(0, idx); + + } } } diff --git a/Duplicati/WebserverCore/Endpoints/V1/Filesystem.cs b/Duplicati/WebserverCore/Endpoints/V1/Filesystem.cs index 9a7625a0d..bb1e9509d 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/Filesystem.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/Filesystem.cs @@ -254,11 +254,11 @@ public class Filesystem : IEndpointV1 { var attr = SystemIO.IO_OS.GetFileAttributes(s); var isSymlink = SystemIO.IO_OS.IsSymlink(s, attr); - var isFolder = (attr & FileAttributes.Directory) != 0; + var isFolder = attr.HasFlag(FileAttributes.Directory); var isFile = !isFolder; - var isHidden = (attr & FileAttributes.Hidden) != 0; - bool isSystem = (attr & FileAttributes.System) != 0; - bool isTemporary = (attr & FileAttributes.Temporary) != 0; + var isHidden = attr.HasFlag(FileAttributes.Hidden); + bool isSystem = attr.HasFlag(FileAttributes.System); + bool isTemporary = attr.HasFlag(FileAttributes.Temporary); long fileSize = -1; var accessible = isFile || canAccess(s); From 356d117b86e74939b4d990973e346be720e67748 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sun, 16 Feb 2025 16:08:28 +0100 Subject: [PATCH 02/28] Ensure directory paths end with a separator --- Duplicati/Library/Snapshots/LinuxSnapshot.cs | 4 ++-- Duplicati/Library/Snapshots/NoSnapshotLinux.cs | 2 +- Duplicati/Library/Snapshots/NoSnapshotWindows.cs | 2 +- Duplicati/Library/Snapshots/SnapshotBase.cs | 5 +++-- Duplicati/Library/Snapshots/WindowsSnapshot.cs | 2 +- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/Duplicati/Library/Snapshots/LinuxSnapshot.cs b/Duplicati/Library/Snapshots/LinuxSnapshot.cs index 9d5a51832..0797f5108 100644 --- a/Duplicati/Library/Snapshots/LinuxSnapshot.cs +++ b/Duplicati/Library/Snapshots/LinuxSnapshot.cs @@ -119,7 +119,7 @@ namespace Duplicati.Library.Snapshots /// A flag indicating if the path is a folder /// The filesystem entry public override ISourceFileEntry GetFilesystemEntry(string path, bool isFolder) - => new SnapshotSourceFileEntry(this, path, isFolder, false); + => new SnapshotSourceFileEntry(this, isFolder ? Util.AppendDirSeparator(path) : path, isFolder, false); /// /// Enumerates the root source files and folders @@ -130,7 +130,7 @@ namespace Duplicati.Library.Snapshots foreach (var snap in m_snapShots) { if (DirectoryExists(snap.LocalPath) || snap.LocalPath.EndsWith(System.IO.Path.DirectorySeparatorChar)) - yield return new SnapshotSourceFileEntry(this, snap.LocalPath, true, true); + yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(snap.LocalPath), true, true); else yield return new SnapshotSourceFileEntry(this, snap.LocalPath, false, true); } diff --git a/Duplicati/Library/Snapshots/NoSnapshotLinux.cs b/Duplicati/Library/Snapshots/NoSnapshotLinux.cs index 3bad638eb..347fc539d 100644 --- a/Duplicati/Library/Snapshots/NoSnapshotLinux.cs +++ b/Duplicati/Library/Snapshots/NoSnapshotLinux.cs @@ -98,7 +98,7 @@ namespace Duplicati.Library.Snapshots foreach (var folder in m_folders) { if (DirectoryExists(folder) || folder.EndsWith(Path.DirectorySeparatorChar)) - yield return new SnapshotSourceFileEntry(this, folder, true, true); + yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(folder), true, true); else yield return new SnapshotSourceFileEntry(this, folder, false, true); } diff --git a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs index 706554cfd..67c21a282 100644 --- a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs +++ b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs @@ -89,7 +89,7 @@ namespace Duplicati.Library.Snapshots foreach (var folder in m_folders.Select(SystemIOWindows.AddExtendedDevicePathPrefix)) { if (DirectoryExists(folder) || folder.EndsWith(System.IO.Path.DirectorySeparatorChar)) - yield return new SnapshotSourceFileEntry(this, folder, true, true); + yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(folder), true, true); else yield return new SnapshotSourceFileEntry(this, folder, false, true); } diff --git a/Duplicati/Library/Snapshots/SnapshotBase.cs b/Duplicati/Library/Snapshots/SnapshotBase.cs index 1c6151137..0c1516790 100644 --- a/Duplicati/Library/Snapshots/SnapshotBase.cs +++ b/Duplicati/Library/Snapshots/SnapshotBase.cs @@ -25,6 +25,7 @@ using System.IO; using Duplicati.Library.Interface; using System.Threading.Tasks; using System.Threading; +using Duplicati.Library.Common.IO; namespace Duplicati.Library.Snapshots { @@ -72,7 +73,7 @@ namespace Duplicati.Library.Snapshots /// A flag indicating if the path is a folder /// The filesystem entry public virtual ISourceFileEntry GetFilesystemEntry(string path, bool isFolder) - => new SnapshotSourceFileEntry(this, path, isFolder, false); + => new SnapshotSourceFileEntry(this, isFolder ? Util.AppendDirSeparator(path) : path, isFolder, false); /// /// Enumerates files and folders in the given folder @@ -87,7 +88,7 @@ namespace Duplicati.Library.Snapshots yield return new SnapshotSourceFileEntry(this, f, false, false); foreach (var d in ListFolders(source.Path)) - yield return new SnapshotSourceFileEntry(this, d, true, false); + yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(d), true, false); } } diff --git a/Duplicati/Library/Snapshots/WindowsSnapshot.cs b/Duplicati/Library/Snapshots/WindowsSnapshot.cs index ae36027ec..cc05ca35e 100644 --- a/Duplicati/Library/Snapshots/WindowsSnapshot.cs +++ b/Duplicati/Library/Snapshots/WindowsSnapshot.cs @@ -120,7 +120,7 @@ namespace Duplicati.Library.Snapshots foreach (var folder in _vssBackupComponents.SnapshotLocalPaths) { if (folder.EndsWith(Path.DirectorySeparatorChar) || DirectoryExists(folder)) - yield return new SnapshotSourceFileEntry(this, folder, true, true); + yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(folder), true, true); else yield return new SnapshotSourceFileEntry(this, folder, false, true); } From d8f6b3370aafc9e2e1c6b07769e2573e65a691e3 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sun, 16 Feb 2025 16:09:20 +0100 Subject: [PATCH 03/28] Removed unused code --- Duplicati/UnitTest/TestUtils.cs | 62 ++++++++++----------------------- 1 file changed, 19 insertions(+), 43 deletions(-) diff --git a/Duplicati/UnitTest/TestUtils.cs b/Duplicati/UnitTest/TestUtils.cs index d8b4a37e8..596a193f1 100644 --- a/Duplicati/UnitTest/TestUtils.cs +++ b/Duplicati/UnitTest/TestUtils.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -24,25 +24,17 @@ using System.IO; using System.Collections.Generic; using Duplicati.Library.Utility; using System.Linq; -using Duplicati.Library.Logging; using System.Reflection; using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Common.IO; using NUnit.Framework; -using System.Runtime.InteropServices; -using Duplicati.Library.Common; using Duplicati.Library.Interface; namespace Duplicati.UnitTest { public static class TestUtils { - /// - /// The log tag - /// - private static readonly string LOGTAG = Library.Logging.Log.LogTagFromType(typeof(TestUtils)); - public static Dictionary DefaultOptions { get @@ -161,22 +153,6 @@ namespace Duplicati.UnitTest Console.WriteLine("Encountered additional {0} timestamp errors!", timestampfailures); } - /// - /// Returns the index of a given string, using the file system case sensitivity - /// - /// The index of the entry or -1 if no entry was found - /// The list to search - /// The string to find - private static int IndexOf(List lst, string m) - { - StringComparison sc = Duplicati.Library.Utility.Utility.ClientFilenameStringComparison; - for (int i = 0; i < lst.Count; i++) - if (lst[i].Equals(m, sc)) - return i; - - return -1; - } - /// /// Asserts that the two directory trees are equivalent; i.e., /// that they they contain the same directories and files, recursively. From 8cead82274d4002b950ba91e9d7c8fbd879270b0 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sun, 16 Feb 2025 16:09:48 +0100 Subject: [PATCH 04/28] Moved a sanity check up --- Duplicati/UnitTest/SVNCheckoutsTest.cs | 52 +++++++++++++------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/Duplicati/UnitTest/SVNCheckoutsTest.cs b/Duplicati/UnitTest/SVNCheckoutsTest.cs index 22a74dccf..a7808cf7e 100644 --- a/Duplicati/UnitTest/SVNCheckoutsTest.cs +++ b/Duplicati/UnitTest/SVNCheckoutsTest.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. @@ -406,6 +406,13 @@ namespace Duplicati.UnitTest restoredname = System.IO.Path.Combine(System.IO.Path.Combine(tempfolder, System.IO.Path.GetFileName(rootfolder.Split(System.IO.Path.PathSeparator)[int.Parse(s.Substring(0, six))])), s.Substring(six + 1)); } + if (!System.IO.File.Exists(sourcename)) + { + Log.WriteErrorMessage(LOGTAG, "PartialRestoreMissingFile", null, "Partial restore, missing SOURCE file: {0}", sourcename); + BasicSetupHelper.ProgressWriteLine("Partial restore missing file: " + sourcename); + throw new Exception("Unittest is broken"); + } + if (!System.IO.File.Exists(restoredname)) { Log.WriteErrorMessage(LOGTAG, "PartialRestoreMissingFile", null, "Partial restore missing file: {0}", restoredname); @@ -413,13 +420,6 @@ namespace Duplicati.UnitTest } else { - if (!System.IO.File.Exists(sourcename)) - { - Log.WriteErrorMessage(LOGTAG, "PartialRestoreMissingFile", null, "Partial restore missing file: {0}", sourcename); - BasicSetupHelper.ProgressWriteLine("Partial restore missing file: " + sourcename); - throw new Exception("Unittest is broken"); - } - TestUtils.AssertFilesAreEqual(sourcename, restoredname, verifymetadata, $"Partial restore file differs: {s}"); } } From 4a9ad45250fa4bf5376cf2ec4e55a348ed07c269 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sun, 16 Feb 2025 16:33:15 +0100 Subject: [PATCH 05/28] Repaired import/export test --- Duplicati/UnitTest/ImportExportTests.cs | 90 ++++++++++++++----------- 1 file changed, 51 insertions(+), 39 deletions(-) diff --git a/Duplicati/UnitTest/ImportExportTests.cs b/Duplicati/UnitTest/ImportExportTests.cs index 9663d5ec1..26062445c 100644 --- a/Duplicati/UnitTest/ImportExportTests.cs +++ b/Duplicati/UnitTest/ImportExportTests.cs @@ -1,28 +1,29 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; using System.IO; using Duplicati.Library.RestAPI; +using Duplicati.Library.SQLiteHelper; using Duplicati.Server; using Duplicati.Server.Database; using Duplicati.Server.Serializable; @@ -126,46 +127,57 @@ namespace Duplicati.UnitTest [Category("ImportExport")] public void RoundTrip() { - Dictionary metadata = new Dictionary { { "SourceFilesCount", "1" } }; - Dictionary advancedOptions = new Dictionary { { "server-datafolder", this.serverDatafolder } }; + var metadata = new Dictionary { { "SourceFilesCount", "1" } }; + var advancedOptions = new Dictionary { { "server-datafolder", this.serverDatafolder } }; + + // Mock the setup enough to get the import/export working var serviceCollection = new ServiceCollection(); serviceCollection.AddSingleton(); serviceCollection.AddSingleton(new EventPollNotify()); FIXMEGlobal.Provider = new DefaultServiceProviderFactory().CreateServiceProvider(serviceCollection); + FIXMEGlobal.DataFolder = this.serverDatafolder; - using (Program.DataConnection = Program.GetDatabaseConnection(advancedOptions, true)) + var dbpath = Path.Combine(this.serverDatafolder, Library.AutoUpdater.DataFolderManager.SERVER_DATABASE_FILENAME); + if (File.Exists(dbpath)) + File.Delete(dbpath); + + var con = SQLiteLoader.LoadConnection(); + SQLiteLoader.OpenDatabase(con, dbpath, null); + DatabaseUpgrader.UpgradeDatabase(con, dbpath, typeof(Duplicati.Library.RestAPI.Database.DatabaseConnectionSchemaMarker)); + + using (var connection = new Connection(con, true, null)) { // Unencrypted file, don't import metadata. string unencryptedWithoutMetadata = Path.Combine(this.serverDatafolder, Path.GetRandomFileName()); - File.WriteAllBytes(unencryptedWithoutMetadata, BackupImportExportHandler.ExportToJSON(Program.DataConnection, this.CreateBackup("unencrypted without metadata", "user", "password", metadata), null)); - BackupImportExportHandler.ImportBackup(Program.DataConnection, unencryptedWithoutMetadata, false, () => null); - Assert.AreEqual(1, Program.DataConnection.Backups.Length); - Assert.AreEqual(0, Program.DataConnection.Backups[0].Metadata.Count); + File.WriteAllBytes(unencryptedWithoutMetadata, BackupImportExportHandler.ExportToJSON(connection, this.CreateBackup("unencrypted without metadata", "user", "password", metadata), null)); + BackupImportExportHandler.ImportBackup(connection, unencryptedWithoutMetadata, false, () => null); + Assert.AreEqual(1, connection.Backups.Length); + Assert.AreEqual(0, connection.Backups[0].Metadata.Count); // Unencrypted file, import metadata. string unencryptedWithMetadata = Path.Combine(this.serverDatafolder, Path.GetRandomFileName()); - File.WriteAllBytes(unencryptedWithMetadata, BackupImportExportHandler.ExportToJSON(Program.DataConnection, this.CreateBackup("unencrypted with metadata", "user", "password", metadata), null)); - BackupImportExportHandler.ImportBackup(Program.DataConnection, unencryptedWithMetadata, true, () => null); - Assert.AreEqual(2, Program.DataConnection.Backups.Length); - Assert.AreEqual(metadata.Count, Program.DataConnection.Backups[1].Metadata.Count); + File.WriteAllBytes(unencryptedWithMetadata, BackupImportExportHandler.ExportToJSON(connection, this.CreateBackup("unencrypted with metadata", "user", "password", metadata), null)); + BackupImportExportHandler.ImportBackup(connection, unencryptedWithMetadata, true, () => null); + Assert.AreEqual(2, connection.Backups.Length); + Assert.AreEqual(metadata.Count, connection.Backups[1].Metadata.Count); // Encrypted file, don't import metadata. string encryptedWithoutMetadata = Path.Combine(this.serverDatafolder, Path.GetRandomFileName()); string passphrase = "abcde"; - File.WriteAllBytes(encryptedWithoutMetadata, BackupImportExportHandler.ExportToJSON(Program.DataConnection, this.CreateBackup("encrypted without metadata", "user", "password", metadata), passphrase)); - BackupImportExportHandler.ImportBackup(Program.DataConnection, encryptedWithoutMetadata, false, () => passphrase); - Assert.AreEqual(3, Program.DataConnection.Backups.Length); - Assert.AreEqual(0, Program.DataConnection.Backups[2].Metadata.Count); + File.WriteAllBytes(encryptedWithoutMetadata, BackupImportExportHandler.ExportToJSON(connection, this.CreateBackup("encrypted without metadata", "user", "password", metadata), passphrase)); + BackupImportExportHandler.ImportBackup(connection, encryptedWithoutMetadata, false, () => passphrase); + Assert.AreEqual(3, connection.Backups.Length); + Assert.AreEqual(0, connection.Backups[2].Metadata.Count); // Encrypted file, import metadata. string encryptedWithMetadata = Path.Combine(this.serverDatafolder, Path.GetRandomFileName()); - File.WriteAllBytes(encryptedWithMetadata, BackupImportExportHandler.ExportToJSON(Program.DataConnection, this.CreateBackup("encrypted with metadata", "user", "password", metadata), passphrase)); - BackupImportExportHandler.ImportBackup(Program.DataConnection, encryptedWithMetadata, true, () => passphrase); - Assert.AreEqual(4, Program.DataConnection.Backups.Length); - Assert.AreEqual(metadata.Count, Program.DataConnection.Backups[3].Metadata.Count); + File.WriteAllBytes(encryptedWithMetadata, BackupImportExportHandler.ExportToJSON(connection, this.CreateBackup("encrypted with metadata", "user", "password", metadata), passphrase)); + BackupImportExportHandler.ImportBackup(connection, encryptedWithMetadata, true, () => passphrase); + Assert.AreEqual(4, connection.Backups.Length); + Assert.AreEqual(metadata.Count, connection.Backups[3].Metadata.Count); // Encrypted file, incorrect passphrase. - Assert.Throws(Is.InstanceOf(), () => BackupImportExportHandler.ImportBackup(Program.DataConnection, encryptedWithMetadata, true, () => passphrase + " ")); + Assert.Throws(Is.InstanceOf(), () => BackupImportExportHandler.ImportBackup(connection, encryptedWithMetadata, true, () => passphrase + " ")); } } } From a621bb3d1419e63609b5d4afe2822582db838f25 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 17 Feb 2025 08:19:19 +0100 Subject: [PATCH 06/28] Fixed issue after merge --- Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs b/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs index 6ae8a85c3..02a983aac 100644 --- a/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs +++ b/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs @@ -120,7 +120,7 @@ namespace Duplicati.Library.Main.Operation.Backup else if (e.TimestampChanged) { await stats.AddTimestampChangedFile(); - Logging.Log.WriteVerboseMessage(FILELOGTAG, "FileTimestampChanged", "File has only timestamp changes {0}", e.Path); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "FileTimestampChanged", "File has only timestamp changes {0}", e.Entry.Path); await database.AddUnmodifiedAsync(e.OldId, e.LastWrite); } else /*if (e.OldId >= 0)*/ From 989ba602117857abfbbeea254a8241228c30ff61 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 17 Feb 2025 08:52:01 +0100 Subject: [PATCH 07/28] Updated license copyright years --- Duplicati/Library/DynamicLoader/SourceProviderLoader.cs | 2 +- .../Library/SourceProvider/Builtin/BackendSourceProvider.cs | 2 +- Duplicati/Library/SourceProvider/Builtin/Combiner.cs | 2 +- Duplicati/Library/SourceProvider/Builtin/File.cs | 2 +- Duplicati/Library/SourceProvider/Builtin/Strings.cs | 2 +- Duplicati/Library/SourceProviders/SourceProviderModules.cs | 2 +- ReleaseBuilder/Resources/Windows/Agent/License.rtf | 2 +- ReleaseBuilder/Resources/Windows/TrayIcon/License.rtf | 2 +- thirdparty/SharpAESCrypt/License.txt | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs b/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs index 5b38ac617..23920c594 100644 --- a/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs +++ b/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2024, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a diff --git a/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs index 54ad99034..9b144a7ab 100644 --- a/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs +++ b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2024, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a diff --git a/Duplicati/Library/SourceProvider/Builtin/Combiner.cs b/Duplicati/Library/SourceProvider/Builtin/Combiner.cs index 5ef6496ba..1dcb01fa8 100644 --- a/Duplicati/Library/SourceProvider/Builtin/Combiner.cs +++ b/Duplicati/Library/SourceProvider/Builtin/Combiner.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2024, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a diff --git a/Duplicati/Library/SourceProvider/Builtin/File.cs b/Duplicati/Library/SourceProvider/Builtin/File.cs index 82e43997a..985fc072c 100644 --- a/Duplicati/Library/SourceProvider/Builtin/File.cs +++ b/Duplicati/Library/SourceProvider/Builtin/File.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2024, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a diff --git a/Duplicati/Library/SourceProvider/Builtin/Strings.cs b/Duplicati/Library/SourceProvider/Builtin/Strings.cs index 5707fccd5..64cc9e118 100644 --- a/Duplicati/Library/SourceProvider/Builtin/Strings.cs +++ b/Duplicati/Library/SourceProvider/Builtin/Strings.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2024, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a diff --git a/Duplicati/Library/SourceProviders/SourceProviderModules.cs b/Duplicati/Library/SourceProviders/SourceProviderModules.cs index 1bf09d104..7bc0b612c 100644 --- a/Duplicati/Library/SourceProviders/SourceProviderModules.cs +++ b/Duplicati/Library/SourceProviders/SourceProviderModules.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2024, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a diff --git a/ReleaseBuilder/Resources/Windows/Agent/License.rtf b/ReleaseBuilder/Resources/Windows/Agent/License.rtf index 9b2b7cff7..7198f91ff 100644 --- a/ReleaseBuilder/Resources/Windows/Agent/License.rtf +++ b/ReleaseBuilder/Resources/Windows/Agent/License.rtf @@ -6,7 +6,7 @@ \deftab720 \pard\pardeftab720\ri0\sl380\sa240\partightenfactor0 -\f0\fs25 \cf0 Copyright (c) 2024 Team Duplicati, https://duplicati.com, hello@duplicati.com +\f0\fs25 \cf0 Copyright (c) 2025 Team Duplicati, https://duplicati.com, hello@duplicati.com \f1\fs24 \ \pard\pardeftab720\ri0\sl380\sb240\sa240\partightenfactor0 diff --git a/ReleaseBuilder/Resources/Windows/TrayIcon/License.rtf b/ReleaseBuilder/Resources/Windows/TrayIcon/License.rtf index 9b2b7cff7..7198f91ff 100644 --- a/ReleaseBuilder/Resources/Windows/TrayIcon/License.rtf +++ b/ReleaseBuilder/Resources/Windows/TrayIcon/License.rtf @@ -6,7 +6,7 @@ \deftab720 \pard\pardeftab720\ri0\sl380\sa240\partightenfactor0 -\f0\fs25 \cf0 Copyright (c) 2024 Team Duplicati, https://duplicati.com, hello@duplicati.com +\f0\fs25 \cf0 Copyright (c) 2025 Team Duplicati, https://duplicati.com, hello@duplicati.com \f1\fs24 \ \pard\pardeftab720\ri0\sl380\sb240\sa240\partightenfactor0 diff --git a/thirdparty/SharpAESCrypt/License.txt b/thirdparty/SharpAESCrypt/License.txt index 09905eb1a..4ac9a5e9c 100755 --- a/thirdparty/SharpAESCrypt/License.txt +++ b/thirdparty/SharpAESCrypt/License.txt @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2024 Duplicati +Copyright (c) 2025 Duplicati Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal From bb5fd80a29c7c8ee926523e806f54143f5843844 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 17 Feb 2025 14:01:08 +0100 Subject: [PATCH 08/28] Implemented support for some backends as sources: SSH, CIFS, and S3 --- Duplicati/Library/Backend/CIFS/CIFS.cs | 73 ++++-- .../Duplicati.Library.Backend.CIFS.csproj | 1 + .../Backend/CIFS/SMBShareConnection.cs | 8 +- Duplicati/Library/Backend/File/FileBackend.cs | 72 +----- .../Duplicati.Library.Backend.Idrivee2.csproj | 1 + .../Backend/Idrivee2/Idrivee2Backend.cs | 55 ++++- .../S3/Duplicati.Library.Backend.S3.csproj | 2 +- Duplicati/Library/Backend/S3/IS3Client.cs | 2 +- Duplicati/Library/Backend/S3/S3AwsClient.cs | 44 ++-- Duplicati/Library/Backend/S3/S3Backend.cs | 94 ++++++-- Duplicati/Library/Backend/S3/S3MinioClient.cs | 30 ++- .../Duplicati.Library.Backend.SSHv2.csproj | 1 + .../Library/Backend/SSHv2/SSHv2Backend.cs | 45 +++- Duplicati/Library/Backend/Storj/StorjFile.cs | 1 + .../Backend/pCloud/Model/pCloudFileEntry.cs | 13 +- Duplicati/Library/Common/IO/FileEntry.cs | 50 ++-- Duplicati/Library/Interface/IFileEntry.cs | 44 ++-- .../Interface/IFolderEnabledBackend.cs | 12 +- .../Library/Interface/ISourceProvider.cs | 7 +- .../Backup/FileEnumerationProcess.cs | 31 +-- .../Library/Main/Operation/BackupHandler.cs | 15 +- .../Main/Operation/FilelistProcessor.cs | 1 - .../Main/Operation/TestFilterHandler.cs | 19 +- .../Library/Snapshots/ISnapshotService.cs | 2 +- Duplicati/Library/Snapshots/LinuxSnapshot.cs | 9 - Duplicati/Library/Snapshots/SnapshotBase.cs | 9 +- .../Builtin/BackendSourceFileEntry.cs | 216 ++++++++++++++++++ .../Builtin/BackendSourceProvider.cs | 172 +------------- .../SourceProvider/Builtin/Combiner.cs | 22 +- .../Builtin/{File.cs => LocalFileSource.cs} | 7 +- 30 files changed, 667 insertions(+), 391 deletions(-) create mode 100644 Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs rename Duplicati/Library/SourceProvider/Builtin/{File.cs => LocalFileSource.cs} (86%) diff --git a/Duplicati/Library/Backend/CIFS/CIFS.cs b/Duplicati/Library/Backend/CIFS/CIFS.cs index 815773fad..eafcdfc45 100644 --- a/Duplicati/Library/Backend/CIFS/CIFS.cs +++ b/Duplicati/Library/Backend/CIFS/CIFS.cs @@ -30,13 +30,15 @@ using Duplicati.Library.Backend.CIFS; using Duplicati.Library.Backend.CIFS.Model; using Duplicati.Library.Utility; using SMBLibrary; +using Duplicati.Library.SourceProvider; +using System.Runtime.CompilerServices; namespace Duplicati.Library.Backend; /// /// Native CIFS/SMB Backend implementation /// -public class CIFSBackend : IStreamingBackend +public class CIFSBackend : IStreamingBackend, IFolderEnabledBackend { /// /// Implementation of interface property for the backend key @@ -58,6 +60,16 @@ public class CIFSBackend : IStreamingBackend /// private string _DnsName; + /// + /// Source provider to be used if the backend is a source + /// + private readonly BackendSourceProvider _sourceProvider; + + /// + /// Key to be used in the path for the source provider + /// + private readonly string _sourcePathKey; + /// /// Path separators (both Windows \ and unix /) to be used in path manipulation /// @@ -72,12 +84,12 @@ public class CIFSBackend : IStreamingBackend /// Shared connection between all methods to avoid re-authentication /// private SMBShareConnection _sharedConnection; - + /// /// Read buffer size for SMB operations (will be capped automatically by SMB negotiated values) /// private const string READ_BUFFER_SIZE_OPTION = "read-buffer-size"; - + /// /// Write buffer size for SMB operations (will be capped automatically by SMB negotiated values) /// @@ -139,33 +151,33 @@ public class CIFSBackend : IStreamingBackend var uri = new Utility.Uri(url); uri.RequireHost(); _DnsName = uri.Host; - + var input = uri.Path.TrimEnd('/'); var slashIndex = input.IndexOf('/'); // Find first slash to separate server and share if present. - + options.TryGetValue(AUTH_USERNAME_OPTION, out string authUsername); options.TryGetValue(AUTH_PASSWORD_OPTION, out string authPassword); options.TryGetValue(AUTH_DOMAIN_OPTION, out string authDomain); options.TryGetValue(TRANSPORT_OPTION, out string transport); - + int? readBufferSize = null, writeBufferSize = null; - + options.TryGetValue(READ_BUFFER_SIZE_OPTION, out string readBufferSizeConfig); if (!string.IsNullOrWhiteSpace(readBufferSizeConfig)) readBufferSize = Int32.TryParse(readBufferSizeConfig, out int value) ? value : null; - + options.TryGetValue(WRITE_BUFFER_SIZE_OPTION, out string writeBufferSizeConfig); if (!string.IsNullOrWhiteSpace(writeBufferSizeConfig)) writeBufferSize = Int32.TryParse(readBufferSizeConfig, out int value) ? value : null; - + // Normalize to 10KB minimum buffers size readBufferSize = readBufferSize < 1024 * 10 ? null : readBufferSize; writeBufferSize = writeBufferSize < 1024 * 10 ? null : writeBufferSize; SMBTransportType transportType = _transportMap.TryGetValue( - string.IsNullOrEmpty(transport) ? DEFAULT_TRANSPORT : transport.ToLower(), + string.IsNullOrEmpty(transport) ? DEFAULT_TRANSPORT : transport.ToLower(), out SMBTransportType type) ? type - : throw new UserInformationException($"Transport must be one of: {string.Join(", ", _transportMap.Keys)}","CIFSConfig"); - + : throw new UserInformationException($"Transport must be one of: {string.Join(", ", _transportMap.Keys)}", "CIFSConfig"); + _connectionParameters = new SMBConnectionParameters( uri.Host, transportType, @@ -177,6 +189,9 @@ public class CIFSBackend : IStreamingBackend readBufferSize, writeBufferSize ); + + _sourcePathKey = $"{ProtocolKey}://{_connectionParameters.ServerName}/~{_connectionParameters.AuthUser}/{_connectionParameters.Path}/"; + _sourceProvider = new BackendSourceProvider(this); } /// @@ -198,7 +213,7 @@ public class CIFSBackend : IStreamingBackend /// List of IFileEntry with directory listing result private async Task> ListAsync(CancellationToken cancellationToken) { - return await GetConnection().ListAsync(cancellationToken).ConfigureAwait(false); + return await GetConnection().ListAsync(_connectionParameters.Path, cancellationToken).ConfigureAwait(false); } /// @@ -282,7 +297,7 @@ public class CIFSBackend : IStreamingBackend /// /// CancellationToken, in this call not used. /// - public Task GetDNSNamesAsync(CancellationToken cancellationToken) => + public Task GetDNSNamesAsync(CancellationToken cancellationToken) => Task.FromResult(new[] { _DnsName ?? string.Empty }); /// @@ -293,7 +308,7 @@ public class CIFSBackend : IStreamingBackend public async Task TestAsync(CancellationToken cancellationToken) { // This will throw an exception if the folder is missing - await GetConnection().ListAsync(cancellationToken).ConfigureAwait(false); + await GetConnection().ListAsync(_connectionParameters.Path, cancellationToken).ConfigureAwait(false); } /// @@ -306,10 +321,10 @@ public class CIFSBackend : IStreamingBackend { var pathParts = _connectionParameters.Path? .Split(PATH_SEPARATORS, StringSplitOptions.RemoveEmptyEntries); - + if (pathParts == null || pathParts.Length == 0) return; - + await GetConnection().CreateFolderAsync(_connectionParameters.Path, cancellationToken) .ConfigureAwait(false); } @@ -320,6 +335,8 @@ public class CIFSBackend : IStreamingBackend /// An SMB connection that can be used for file operations private SMBShareConnection GetConnection() => _sharedConnection ??= new SMBShareConnection(_connectionParameters); + + /// /// Implementation of Dispose pattern enforced by interface /// @@ -335,4 +352,26 @@ public class CIFSBackend : IStreamingBackend System.Diagnostics.Debug.WriteLine($"Error disposing CIFS connection: {ex.Message}"); } } + + /// + public string PathKey => _sourcePathKey; + + /// + public async IAsyncEnumerable ListAsync(string path, [EnumeratorCancellation] CancellationToken cancellationToken) + { + var sourcePath = _connectionParameters.Path; + if (!sourcePath.EndsWith('\\')) + sourcePath += '\\'; + + var list = (await GetConnection().ListAsync(sourcePath + path, cancellationToken).ConfigureAwait(false)) + .Select(f => BackendSourceFileEntry.FromFileEntry(_sourceProvider, path, f)) + .Cast(); + + foreach (var v in list) + yield return v; + } + + /// + public Task GetEntryAsync(string path, CancellationToken cancellationToken) + => Task.FromResult(null); } \ No newline at end of file diff --git a/Duplicati/Library/Backend/CIFS/Duplicati.Library.Backend.CIFS.csproj b/Duplicati/Library/Backend/CIFS/Duplicati.Library.Backend.CIFS.csproj index b638e8235..11375a4ab 100644 --- a/Duplicati/Library/Backend/CIFS/Duplicati.Library.Backend.CIFS.csproj +++ b/Duplicati/Library/Backend/CIFS/Duplicati.Library.Backend.CIFS.csproj @@ -10,6 +10,7 @@ + diff --git a/Duplicati/Library/Backend/CIFS/SMBShareConnection.cs b/Duplicati/Library/Backend/CIFS/SMBShareConnection.cs index 44b936464..710e36ea2 100644 --- a/Duplicati/Library/Backend/CIFS/SMBShareConnection.cs +++ b/Duplicati/Library/Backend/CIFS/SMBShareConnection.cs @@ -194,9 +194,10 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable /// /// Lists the folder contents of the share and path specified in the connection parameters. /// + /// Path to list /// Cancellation Token /// Exception to be displayed to user - public async Task> ListAsync(CancellationToken cancellationToken) + public async Task> ListAsync(string path, CancellationToken cancellationToken) { await _semaphore.WaitAsync(cancellationToken); @@ -210,7 +211,7 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable var status = _smbFileStore.CreateFile( out directoryHandle, out fileStatus, - NormalizeSlashes(_connectionParameters.Path), + NormalizeSlashes(path), AccessMask.GENERIC_READ, FileAttributes.Directory, ShareAccess.Read | ShareAccess.Write, @@ -244,7 +245,8 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable info.LastAccessTime, info.LastWriteTime) { - IsFolder = info.FileAttributes == FileAttributes.Directory + IsFolder = info.FileAttributes == FileAttributes.Directory, + Created = info.CreationTime }) .ToList() ]; diff --git a/Duplicati/Library/Backend/File/FileBackend.cs b/Duplicati/Library/Backend/File/FileBackend.cs index de049627b..3a8b76ca8 100644 --- a/Duplicati/Library/Backend/File/FileBackend.cs +++ b/Duplicati/Library/Backend/File/FileBackend.cs @@ -468,6 +468,9 @@ namespace Duplicati.Library.Backend } } + /// + public string PathKey => m_path; + /// public async IAsyncEnumerable ListAsync(string path, [EnumeratorCancellation] CancellationToken cancellationToken) { @@ -481,10 +484,10 @@ namespace Duplicati.Library.Backend var service = SnapshotUtility.CreateNoSnapshot([remotePath], false, false); await foreach (var folderEntry in systemIO.EnumerateDirectories(remotePath).ToAsyncEnumerable().WithCancellation(cancellationToken).ConfigureAwait(false)) - yield return new SourceFileEntry(service, folderEntry, true); + yield return new SnapshotSourceFileEntry(service, folderEntry, true, false); await foreach (var fileEntry in systemIO.EnumerateFiles(remotePath).ToAsyncEnumerable().WithCancellation(cancellationToken).ConfigureAwait(false)) - yield return new SourceFileEntry(service, fileEntry, false); + yield return new SnapshotSourceFileEntry(service, fileEntry, false, false); } /// @@ -492,70 +495,7 @@ namespace Duplicati.Library.Backend { var service = SnapshotUtility.CreateNoSnapshot([path], false, false); var isFolder = service.DirectoryExists(path); - return Task.FromResult(new SourceFileEntry(service, path, isFolder)); - } - - private class SourceFileEntry(ISnapshotService service, string path, bool isFolder) : ISourceFileEntry - { - /// - public bool IsFolder => isFolder; - - /// - public bool IsMetaEntry => false; - /// - public bool IsRootEntry => false; - - /// - public DateTime CreatedUtc => service.GetCreationTimeUtc(path); - - /// - public DateTime LastModificationUtc => service.GetLastWriteTimeUtc(path); - - /// - public string Path => path; - - /// - public long Size => isFolder ? -1 : service.GetFileSize(path); - - /// - public bool IsSymlink => service.IsSymlink(path, Attributes); - - /// - public string SymlinkTarget => service.GetSymlinkTarget(path); - - /// - public FileAttributes Attributes => service.GetAttributes(path); - - /// - public Dictionary MinorMetadata => service.GetMetadata(path, IsSymlink); - - /// - public bool IsBlockDevice => service.IsBlockDevice(path); - - /// - public bool IsCharacterDevice => false; - - /// - public bool IsAlternateStream => false; - - /// - public string HardlinkTargetId => service.HardlinkTargetID(path); - - /// - public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) - => service.EnumerateFilesystemEntries(this).ToAsyncEnumerable(); - - /// - public Task FileExists(string filename, CancellationToken cancellationToken) - => Task.FromResult(service.FileExists(systemIO.PathCombine(Path, filename))); - - /// - public Task OpenMetadataRead(CancellationToken cancellationToken) - => Task.FromResult(new MemoryStream(System.Text.Json.JsonSerializer.SerializeToUtf8Bytes(MinorMetadata))); - - /// - public Task OpenRead(CancellationToken cancellationToken) - => service.OpenReadAsync(path, cancellationToken); + return Task.FromResult(new SnapshotSourceFileEntry(service, path, isFolder, false)); } } } diff --git a/Duplicati/Library/Backend/Idrivee2/Duplicati.Library.Backend.Idrivee2.csproj b/Duplicati/Library/Backend/Idrivee2/Duplicati.Library.Backend.Idrivee2.csproj index c8664ccd2..756380cd2 100644 --- a/Duplicati/Library/Backend/Idrivee2/Duplicati.Library.Backend.Idrivee2.csproj +++ b/Duplicati/Library/Backend/Idrivee2/Duplicati.Library.Backend.Idrivee2.csproj @@ -8,6 +8,7 @@ + diff --git a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs index 59dca09bd..f495ddf52 100644 --- a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs +++ b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs @@ -21,17 +21,18 @@ using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; -using Duplicati.Library.Utility; +using Duplicati.Library.SourceProvider; using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; namespace Duplicati.Library.Backend { - public class Idrivee2Backend : IBackend, IStreamingBackend + public class Idrivee2Backend : IBackend, IStreamingBackend, IFolderEnabledBackend { private static readonly string LOGTAG = Logging.Log.LogTagFromType(); @@ -43,6 +44,15 @@ namespace Duplicati.Library.Backend private readonly string m_prefix; private readonly string m_bucket; + /// + /// The source provider for this backend + /// + private readonly BackendSourceProvider m_sourceProvider; + /// + /// The key the backend is identified by if it is a source + /// + private readonly string m_sourcePathKey; + private IS3Client m_s3Client; public Idrivee2Backend() @@ -80,7 +90,8 @@ namespace Duplicati.Library.Backend m_s3Client = new S3AwsClient(accessKeyId, accessKeySecret, null, host, null, true, false, options); - + m_sourcePathKey = $"{ProtocolKey}://{host}/{m_bucket}/{m_prefix}/"; + m_sourceProvider = new BackendSourceProvider(this); } public string GetRegionEndpoint(string url) @@ -127,8 +138,11 @@ namespace Duplicati.Library.Backend public IEnumerable List() + => ListAsync(CancellationToken.None).ToBlockingEnumerable(); + + public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { - foreach (IFileEntry file in Connection.ListBucket(m_bucket, m_prefix)) + await foreach (IFileEntry file in Connection.ListBucketAsync(m_bucket, m_prefix, cancelToken).ConfigureAwait(false)) { ((FileEntry)file).Name = file.Name.Substring(m_prefix.Length); if (file.Name.StartsWith("/", StringComparison.Ordinal) && !m_prefix.StartsWith("/", StringComparison.Ordinal)) @@ -230,5 +244,38 @@ namespace Duplicati.Library.Backend //AWS SDK encodes the filenames correctly return m_prefix + name; } + + /// + public string PathKey => m_sourcePathKey; + + /// + public async IAsyncEnumerable ListAsync(string path, [EnumeratorCancellation] CancellationToken cancellationToken) + { + var prefixPath = m_prefix; + if (!prefixPath.EndsWith("/", StringComparison.Ordinal)) + prefixPath += "/"; + + var filterPath = prefixPath + path; + if (!filterPath.EndsWith("/", StringComparison.Ordinal)) + filterPath += "/"; + + await foreach (var f in m_s3Client.ListBucketAsync(m_bucket, filterPath, cancellationToken).ConfigureAwait(false)) + { + if (!f.Name.StartsWith(prefixPath, StringComparison.Ordinal)) + continue; + + ((FileEntry)f).Name = f.Name.Substring(prefixPath.Length); + + // Skip sub-folder items + if (f.Name.Contains("/")) + continue; + + yield return BackendSourceFileEntry.FromFileEntry(m_sourceProvider, path, f); + } + } + + /// + public Task GetEntryAsync(string path, CancellationToken cancellationToken) + => Task.FromResult(null); } } diff --git a/Duplicati/Library/Backend/S3/Duplicati.Library.Backend.S3.csproj b/Duplicati/Library/Backend/S3/Duplicati.Library.Backend.S3.csproj index 90016385f..148804d3b 100644 --- a/Duplicati/Library/Backend/S3/Duplicati.Library.Backend.S3.csproj +++ b/Duplicati/Library/Backend/S3/Duplicati.Library.Backend.S3.csproj @@ -12,10 +12,10 @@ - + diff --git a/Duplicati/Library/Backend/S3/IS3Client.cs b/Duplicati/Library/Backend/S3/IS3Client.cs index fee68361b..6d32833ee 100644 --- a/Duplicati/Library/Backend/S3/IS3Client.cs +++ b/Duplicati/Library/Backend/S3/IS3Client.cs @@ -28,7 +28,7 @@ namespace Duplicati.Library.Backend { public interface IS3Client : IDisposable { - IEnumerable ListBucket(string bucketName, string prefix); + IAsyncEnumerable ListBucketAsync(string bucketName, string prefix, CancellationToken cancellationToken); Task AddBucketAsync(string bucketName, CancellationToken cancelToken); diff --git a/Duplicati/Library/Backend/S3/S3AwsClient.cs b/Duplicati/Library/Backend/S3/S3AwsClient.cs index e3a480159..94e578182 100644 --- a/Duplicati/Library/Backend/S3/S3AwsClient.cs +++ b/Duplicati/Library/Backend/S3/S3AwsClient.cs @@ -1,33 +1,33 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Amazon.S3; using Amazon.S3.Model; -using Duplicati.Library.Backend.Strings; using Duplicati.Library.Interface; using Duplicati.Library.Utility; using System; using System.Collections.Generic; using System.Linq; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; @@ -192,7 +192,7 @@ namespace Duplicati.Library.Backend return m_client.DeleteObjectAsync(objectDeleteRequest, cancellationToken); } - public virtual IEnumerable ListBucket(string bucketName, string prefix) + public async IAsyncEnumerable ListBucketAsync(string bucketName, string prefix, [EnumeratorCancellation] CancellationToken cancellationToken) { bool isTruncated = true; string filename = null; @@ -217,7 +217,7 @@ namespace Duplicati.Library.Backend ListObjectsResponse listResponse; try { - listResponse = m_client.ListObjectsAsync(listRequest).Await(); + listResponse = await m_client.ListObjectsAsync(listRequest, cancellationToken).ConfigureAwait(false); } catch (AmazonS3Exception e) { diff --git a/Duplicati/Library/Backend/S3/S3Backend.cs b/Duplicati/Library/Backend/S3/S3Backend.cs index 8df4c8355..e6d4d99bb 100644 --- a/Duplicati/Library/Backend/S3/S3Backend.cs +++ b/Duplicati/Library/Backend/S3/S3Backend.cs @@ -1,31 +1,32 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; -using Duplicati.Library.Utility; +using Duplicati.Library.SourceProvider; using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Runtime.CompilerServices; using System.Text; using System.Text.RegularExpressions; using System.Threading; @@ -33,7 +34,7 @@ using System.Threading.Tasks; namespace Duplicati.Library.Backend { - public class S3 : IBackend, IStreamingBackend, IRenameEnabledBackend + public class S3 : IBackend, IStreamingBackend, IRenameEnabledBackend, IFolderEnabledBackend { private static readonly string LOGTAG = Logging.Log.LogTagFromType(); @@ -174,6 +175,15 @@ namespace Duplicati.Library.Backend private readonly string m_bucket; private readonly string m_prefix; + /// + /// Source provider to be used if the backend is a source + /// + private readonly BackendSourceProvider m_sourceProvider; + /// + /// The key path for the backend as a source + /// + private readonly string m_sourcePathKey; + private const string DEFAULT_S3_HOST = "s3.amazonaws.com"; private IS3Client s3Client; @@ -245,6 +255,9 @@ namespace Duplicati.Library.Backend { throw new UserInformationException(Strings.S3Backend.UnknownS3ClientError(s3ClientOptionValue), "UnknownS3Client"); } + + m_sourcePathKey = $"{ProtocolKey}://{hostname}/{m_bucket}/{m_prefix}/"; + m_sourceProvider = new BackendSourceProvider(this); } public static bool IsValidHostname(string bucketname) @@ -269,10 +282,12 @@ namespace Duplicati.Library.Backend get { return true; } } - public IEnumerable List() + => ListAsync(CancellationToken.None).ToBlockingEnumerable(); + + public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { - foreach (IFileEntry file in Connection.ListBucket(m_bucket, m_prefix)) + await foreach (IFileEntry file in Connection.ListBucketAsync(m_bucket, m_prefix, cancelToken)) { ((FileEntry)file).Name = file.Name.Substring(m_prefix.Length); @@ -396,5 +411,38 @@ namespace Duplicati.Library.Backend //AWS SDK encodes the filenames correctly return m_prefix + name; } + + /// + public string PathKey => m_sourcePathKey; + + /// + public async IAsyncEnumerable ListAsync(string path, [EnumeratorCancellation] CancellationToken cancellationToken) + { + var prefixPath = m_prefix; + if (!prefixPath.EndsWith("/", StringComparison.Ordinal)) + prefixPath += "/"; + + var filterPath = prefixPath + path; + if (!filterPath.EndsWith("/", StringComparison.Ordinal)) + filterPath += "/"; + + await foreach (var f in s3Client.ListBucketAsync(m_bucket, filterPath, cancellationToken).ConfigureAwait(false)) + { + if (!f.Name.StartsWith(prefixPath, StringComparison.Ordinal)) + continue; + + ((FileEntry)f).Name = f.Name.Substring(prefixPath.Length); + + // Skip sub-folder items + if (f.Name.Contains("/")) + continue; + + yield return BackendSourceFileEntry.FromFileEntry(m_sourceProvider, path, f); + } + } + + /// + public Task GetEntryAsync(string path, CancellationToken cancellationToken) + => Task.FromResult(null); } } diff --git a/Duplicati/Library/Backend/S3/S3MinioClient.cs b/Duplicati/Library/Backend/S3/S3MinioClient.cs index 09d2e5d52..cee966473 100644 --- a/Duplicati/Library/Backend/S3/S3MinioClient.cs +++ b/Duplicati/Library/Backend/S3/S3MinioClient.cs @@ -22,7 +22,9 @@ using System; using System.Collections.Generic; using System.IO; using System.Reactive.Linq; +using System.Runtime.CompilerServices; using System.Threading; +using System.Threading.Channels; using System.Threading.Tasks; using Duplicati.Library.Interface; using Duplicati.Library.Utility; @@ -58,13 +60,33 @@ namespace Duplicati.Library.Backend m_dnsHost = servername; } - public IEnumerable ListBucket(string bucketName, string prefix) + private static async IAsyncEnumerable ToAsyncEnumerable( + IObservable observable, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var channel = Channel.CreateUnbounded(); // Buffered channel for async iteration + + using var subscription = observable.Subscribe( + item => channel.Writer.TryWrite(item), + ex => channel.Writer.TryComplete(ex), + () => channel.Writer.TryComplete() + ); + + while (await channel.Reader.WaitToReadAsync(cancellationToken)) + { + while (channel.Reader.TryRead(out var item)) + { + yield return item; + } + } + } + + public async IAsyncEnumerable ListBucketAsync(string bucketName, string prefix, [EnumeratorCancellation] CancellationToken cancellationToken) { ThrowExceptionIfBucketDoesNotExist(bucketName); - var observable = m_client.ListObjectsAsync(bucketName, prefix, true); - - foreach (var obj in observable.ToEnumerable()) + var observable = m_client.ListObjectsAsync(bucketName, prefix, true, cancellationToken); + await foreach (var obj in ToAsyncEnumerable(observable, cancellationToken).ConfigureAwait(false)) { yield return new Common.IO.FileEntry( obj.Key, diff --git a/Duplicati/Library/Backend/SSHv2/Duplicati.Library.Backend.SSHv2.csproj b/Duplicati/Library/Backend/SSHv2/Duplicati.Library.Backend.SSHv2.csproj index 9182306df..7c7bbd948 100644 --- a/Duplicati/Library/Backend/SSHv2/Duplicati.Library.Backend.SSHv2.csproj +++ b/Duplicati/Library/Backend/SSHv2/Duplicati.Library.Backend.SSHv2.csproj @@ -12,6 +12,7 @@ + diff --git a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs index 05a1bfb7d..d8057f651 100644 --- a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs +++ b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs @@ -22,6 +22,7 @@ using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; +using Duplicati.Library.SourceProvider; using Renci.SshNet; using Renci.SshNet.Common; using System; @@ -33,7 +34,7 @@ using System.Threading.Tasks; namespace Duplicati.Library.Backend { - public class SSHv2 : IStreamingBackend, IRenameEnabledBackend + public class SSHv2 : IStreamingBackend, IRenameEnabledBackend, IFolderEnabledBackend { private const string SSH_KEYFILE_OPTION = "ssh-keyfile"; private const string SSH_KEYFILE_INLINE = "ssh-key"; @@ -56,6 +57,15 @@ namespace Duplicati.Library.Backend private readonly int m_port = 22; + /// + /// The source provider for this backend + /// + private readonly BackendSourceProvider m_sourceProvider; + /// + /// The path key for this backend if used as a source + /// + private readonly string m_sourcePathKey; + private SftpClient m_con; public SSHv2() @@ -107,6 +117,9 @@ namespace Duplicati.Library.Backend if (!string.IsNullOrWhiteSpace(timeoutstr)) m_keepaliveinterval = Library.Utility.Timeparser.ParseTimeSpan(timeoutstr); + + m_sourcePathKey = $"{ProtocolKey}://{m_server}:{m_port}/~{m_username}/{m_path}/"; + m_sourceProvider = new BackendSourceProvider(this); } #region IBackend Members @@ -429,5 +442,35 @@ namespace Duplicati.Library.Backend } public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new[] { m_server }); + + /// + public string PathKey => m_sourcePathKey; + + /// + public IAsyncEnumerable ListAsync(string path, CancellationToken cancellationToken) + { + var prefixPath = m_path; + if (!prefixPath.EndsWith("/")) + prefixPath += "/"; + + var filterPath = prefixPath + path; + if (!filterPath.EndsWith("/")) + filterPath += "/"; + + return Client.ListDirectoryAsync(filterPath, cancellationToken) + .Select(x => new BackendSourceFileEntry( + m_sourceProvider, + BackendSourceFileEntry.ConcatPaths(path, x.Name), + x.Attributes.IsDirectory, + false, + new DateTime(0), + x.Attributes.LastWriteTimeUtc, + x.Attributes.Size + )); + } + + /// + public Task GetEntryAsync(string path, CancellationToken cancellationToken) + => Task.FromResult(null); } } \ No newline at end of file diff --git a/Duplicati/Library/Backend/Storj/StorjFile.cs b/Duplicati/Library/Backend/Storj/StorjFile.cs index f7a0d6e51..6a4b18076 100644 --- a/Duplicati/Library/Backend/Storj/StorjFile.cs +++ b/Duplicati/Library/Backend/Storj/StorjFile.cs @@ -33,6 +33,7 @@ namespace Duplicati.Library.Backend.Storj public DateTime LastAccess { get; set; } public DateTime LastModification { get; set; } + public DateTime Created { get; set; } public string Name { get; set; } diff --git a/Duplicati/Library/Backend/pCloud/Model/pCloudFileEntry.cs b/Duplicati/Library/Backend/pCloud/Model/pCloudFileEntry.cs index bbe68ef6d..179c277c6 100644 --- a/Duplicati/Library/Backend/pCloud/Model/pCloudFileEntry.cs +++ b/Duplicati/Library/Backend/pCloud/Model/pCloudFileEntry.cs @@ -33,22 +33,27 @@ internal class pCloudFileEntry : IFileEntry /// Indicates it is a folder on the remote filesystem /// public bool IsFolder { get; set; } - + /// /// Last access date of the file /// public DateTime LastAccess { get; set; } - + /// /// Last modification of the file /// public DateTime LastModification { get; set; } - + + /// + /// Last modification of the file + /// + public DateTime Created { get; set; } + /// /// Filename or folder name /// public string Name { get; set; } - + /// /// Size of the object in bytes /// diff --git a/Duplicati/Library/Common/IO/FileEntry.cs b/Duplicati/Library/Common/IO/FileEntry.cs index fc1b18b49..b603edd0d 100644 --- a/Duplicati/Library/Common/IO/FileEntry.cs +++ b/Duplicati/Library/Common/IO/FileEntry.cs @@ -1,23 +1,23 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. using System; using Duplicati.Library.Interface; @@ -32,6 +32,7 @@ namespace Duplicati.Library.Common.IO private string m_name; private DateTime m_lastAccess; private DateTime m_lastModification; + private DateTime m_created; private long m_size; private bool m_isFolder; @@ -62,6 +63,15 @@ namespace Duplicati.Library.Common.IO set { m_lastModification = value; } } + /// + /// Gets or sets the time the file or folder was created + /// + public DateTime Created + { + get { return m_created; } + set { m_created = value; } + } + /// /// Gets or sets the size of the file or folder /// diff --git a/Duplicati/Library/Interface/IFileEntry.cs b/Duplicati/Library/Interface/IFileEntry.cs index 3a7f9e104..2e5fb1f01 100644 --- a/Duplicati/Library/Interface/IFileEntry.cs +++ b/Duplicati/Library/Interface/IFileEntry.cs @@ -1,23 +1,23 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. using System; @@ -41,6 +41,10 @@ namespace Duplicati.Library.Interface /// DateTime LastModification { get; } /// + /// The time the file or folder was created + /// + DateTime Created { get; } + /// /// The name of the file or folder /// string Name { get; } diff --git a/Duplicati/Library/Interface/IFolderEnabledBackend.cs b/Duplicati/Library/Interface/IFolderEnabledBackend.cs index 2ccd3f44c..4f83178ad 100644 --- a/Duplicati/Library/Interface/IFolderEnabledBackend.cs +++ b/Duplicati/Library/Interface/IFolderEnabledBackend.cs @@ -34,12 +34,22 @@ namespace Duplicati.Library.Interface; /// public interface IFolderEnabledBackend : IBackend { + /// + /// Returns the key for this backend instance's path + /// + string PathKey { get; } + /// /// Enumerates a list of files and folders found on the remote location /// /// The path to list, empty or null for the root folder /// The cancellation token - /// The list of files + /// The list of files, where paths are scoped to the backend. + /// + /// The paths returned should be relative to the backend root, and should not include the root path. + /// For example, if the backend is defined as proto://host/rootpath, then listing files should + /// return paths like "/subfolder/file.txt" and not "/rootpath/subfolder/file.txt". + /// IAsyncEnumerable ListAsync(string? path, CancellationToken cancellationToken); /// diff --git a/Duplicati/Library/Interface/ISourceProvider.cs b/Duplicati/Library/Interface/ISourceProvider.cs index 4b5725739..11c61d4a4 100644 --- a/Duplicati/Library/Interface/ISourceProvider.cs +++ b/Duplicati/Library/Interface/ISourceProvider.cs @@ -33,6 +33,11 @@ namespace Duplicati.Library.Interface; /// public interface ISourceProvider : IDisposable { + /// + /// Returns the key for this provider instance's path + /// + string PathKey { get; } + /// /// Gets the root entries /// @@ -47,5 +52,5 @@ public interface ISourceProvider : IDisposable /// True if the path is a folder /// The cancellation token /// The file entry - Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken); + Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken); } diff --git a/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs b/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs index 7283da48e..a1ef2ec65 100644 --- a/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs @@ -19,6 +19,8 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +#nullable enable + using System; using CoCoL; using System.Threading.Tasks; @@ -49,17 +51,17 @@ namespace Duplicati.Library.Main.Operation.Backup public static Task Run( Channels channels, ISourceProvider sourceProvider, - UsnJournalService journalService, + UsnJournalService? journalService, FileAttributes fileAttributeFilter, Library.Utility.IFilter emitfilter, Options.SymlinkStrategy symlinkPolicy, Options.HardlinkStrategy hardlinkPolicy, bool excludeemptyfolders, - string[] ignorenames, + string[]? ignorenames, HashSet blacklistPaths, - IEnumerable changedfilelist, + IEnumerable? changedfilelist, ITaskReader taskreader, - Action onStopRequested, + Action? onStopRequested, CancellationToken token) { return AutomationExtensions.RunTask( @@ -98,7 +100,11 @@ namespace Duplicati.Library.Main.Operation.Backup async IAsyncEnumerable ExpandSources(IEnumerable list) { foreach (var s in list) - yield return await sourceProvider.GetEntry(s, false, token).ConfigureAwait(false); + { + var r = await sourceProvider.GetEntry(s, false, token).ConfigureAwait(false); + if (r != null) + yield return r; + } } IAsyncEnumerable worklist; @@ -113,7 +119,7 @@ namespace Duplicati.Library.Main.Operation.Backup throw new NotSupportedException("USN is only supported on Windows"); var fileProviders = (sourceProvider is Combiner c ? c.Providers.AsEnumerable() : [sourceProvider]) - .OfType() + .OfType() .ToList(); if (fileProviders.Count <= 0) @@ -179,11 +185,11 @@ namespace Duplicati.Library.Main.Operation.Backup /// /// The item being tracked /// - public ISourceFileEntry Item; + public required ISourceFileEntry Item; /// /// A flag indicating if any items are found in this folder /// - public bool AnyEntries; + public required bool AnyEntries; } @@ -219,7 +225,7 @@ namespace Duplicati.Library.Main.Operation.Backup if (pathstack.Count == 0 || s.Path.StartsWith(pathstack.Peek().Item.Path, Library.Utility.Utility.ClientFilenameStringComparison)) { - pathstack.Push(new DirectoryStackEntry() { Item = s }); + pathstack.Push(new DirectoryStackEntry() { Item = s, AnyEntries = false }); continue; } } @@ -381,7 +387,7 @@ namespace Duplicati.Library.Main.Operation.Backup /// The ignore names. /// The mixin queue. /// True if the path should be returned, false otherwise. - private static async ValueTask SourceFileEntryFilter(ISourceFileEntry entry, HashSet blacklistPaths, Options.HardlinkStrategy hardlinkPolicy, Options.SymlinkStrategy symlinkPolicy, Dictionary hardlinkmap, FileAttributes fileAttributeFilter, Duplicati.Library.Utility.IFilter enumeratefilter, string[] ignorenames, Queue mixinqueue, CancellationToken cancellationToken) + private static async ValueTask SourceFileEntryFilter(ISourceFileEntry entry, HashSet blacklistPaths, Options.HardlinkStrategy hardlinkPolicy, Options.SymlinkStrategy symlinkPolicy, Dictionary hardlinkmap, FileAttributes fileAttributeFilter, Duplicati.Library.Utility.IFilter enumeratefilter, string[]? ignorenames, Queue mixinqueue, CancellationToken cancellationToken) { // Do the course pre-filtering first if (!PreFilterSourceEntry(entry, blacklistPaths)) @@ -406,8 +412,7 @@ namespace Duplicati.Library.Main.Operation.Backup } else if (hardlinkPolicy == Options.HardlinkStrategy.First) { - string prevPath; - if (hardlinkmap.TryGetValue(id, out prevPath)) + if (hardlinkmap.TryGetValue(id, out var prevPath)) { Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingDuplicateHardlink", "Excluding hardlink ({1}) for: {0}, previous hardlink: {2}", entry.Path, id, prevPath); return false; @@ -481,7 +486,7 @@ namespace Duplicati.Library.Main.Operation.Backup } // If the file is a symlink, apply special handling - string symlinkTarget = null; + string? symlinkTarget = null; try { symlinkTarget = entry.SymlinkTarget; diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 4c3171f6c..84fc89b69 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -132,9 +132,9 @@ namespace Duplicati.Library.Main.Operation foreach (var entry in sourceTypes) { if ("file".Equals(entry.Key, StringComparison.OrdinalIgnoreCase)) - results.Add(new SourceProvider.File(GetFileSnapshotService(entry, options))); + results.Add(new LocalFileSource(GetFileSnapshotService(entry, options))); else if ("vss".Equals(entry.Key, StringComparison.OrdinalIgnoreCase) || "lvm".Equals(entry.Key, StringComparison.OrdinalIgnoreCase)) - results.Add(new SourceProvider.File(SnapshotUtility.CreateSnapshot(entry, options.RawOptions, options.SymlinkPolicy == Options.SymlinkStrategy.Follow))); + results.Add(new LocalFileSource(SnapshotUtility.CreateSnapshot(entry, options.RawOptions, options.SymlinkPolicy == Options.SymlinkStrategy.Follow))); else { foreach (var url in entry) @@ -147,14 +147,15 @@ namespace Duplicati.Library.Main.Operation continue; } - // See if we can implement the same with a backend + // See if there is a backend that can also be a source var backend = BackendLoader.GetBackend(url, options.RawOptions); - if (backend != null) + if (backend is IFolderEnabledBackend folderBackend) { - results.Add(new SourceProvider.BackendSourceProvider(backend, url)); + results.Add(new BackendSourceProvider(folderBackend)); continue; } + backend?.Dispose(); throw new UserInformationException(string.Format("The source \"{0}\" is not supported", url), "SourceNotSupported"); } } @@ -179,14 +180,14 @@ namespace Duplicati.Library.Main.Operation var providers = (provider is Combiner c ? c.Providers : [provider]) .Select((x, i) => new { Provider = x, Index = i }) - .Where(x => x.Provider is SourceProvider.File) + .Where(x => x.Provider is LocalFileSource) .ToList(); if (providers.Count == 0) return null; if (providers.Count > 1) throw new UserInformationException("Multiple USN journal services are not supported", "MultipleUSNJournalServices"); - var fileProvider = providers.First().Provider as SourceProvider.File; + var fileProvider = providers.First().Provider as LocalFileSource; var journalData = m_database.GetChangeJournalData(lastfilesetid); var service = new UsnJournalService(fileProvider.SnapshotService, filter, m_options.FileAttributeFilter, m_options.SkipFilesLargerThan, diff --git a/Duplicati/Library/Main/Operation/FilelistProcessor.cs b/Duplicati/Library/Main/Operation/FilelistProcessor.cs index b3c6405c4..f1d10fdfb 100644 --- a/Duplicati/Library/Main/Operation/FilelistProcessor.cs +++ b/Duplicati/Library/Main/Operation/FilelistProcessor.cs @@ -26,7 +26,6 @@ using System.Data; using System.Linq; using Duplicati.Library.Interface; using System.Threading; -using Duplicati.Library.Utility; using System.Threading.Tasks; namespace Duplicati.Library.Main.Operation diff --git a/Duplicati/Library/Main/Operation/TestFilterHandler.cs b/Duplicati/Library/Main/Operation/TestFilterHandler.cs index 4a06bbc89..b2468738f 100644 --- a/Duplicati/Library/Main/Operation/TestFilterHandler.cs +++ b/Duplicati/Library/Main/Operation/TestFilterHandler.cs @@ -19,6 +19,8 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +#nullable enable + using System; using System.IO; using CoCoL; @@ -71,7 +73,7 @@ namespace Duplicati.Library.Main.Operation catch (Exception ex) { Logging.Log.WriteVerboseMessage(LOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", entry.Path, ex.Message); } // Analyze symlinks - string symlinkTarget = null; + string? symlinkTarget = null; try { symlinkTarget = entry.SymlinkTarget; } catch (Exception ex) { Logging.Log.WriteExplicitMessage(LOGTAG, "SymlinkTargetReadFailure", ex, "Failed to read symlink target for path: {0}", entry.Path); } @@ -84,14 +86,22 @@ namespace Duplicati.Library.Main.Operation // Go for the symlink target, as we know we follow symlinks if (!string.IsNullOrWhiteSpace(symlinkTarget)) { - entry = await provider.GetEntry(symlinkTarget, false, stopToken).ConfigureAwait(false); + var targetEntry = await provider.GetEntry(symlinkTarget, false, stopToken).ConfigureAwait(false); fa = FileAttributes.Normal; - try { fa = entry.Attributes; } + try { fa = targetEntry!.Attributes; } catch (Exception ex) { Logging.Log.WriteVerboseMessage(LOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", entry.Path, ex.Message); } + // If we guessed wrong and the symlink target is a folder, we need to fetch it with the correct flag if (fa.HasFlag(FileAttributes.Directory)) - entry = await provider.GetEntry(symlinkTarget, true, stopToken).ConfigureAwait(false); + targetEntry = await provider.GetEntry(symlinkTarget, true, stopToken).ConfigureAwait(false); + + // No such target + if (targetEntry == null) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "SymlinkTargetNotFound", "Symlink target not found: {0}", symlinkTarget); + continue; + } } // Proceed with non-folders @@ -110,7 +120,6 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteVerboseMessage(LOGTAG, "SizeReadFailed", "Failed to read length of file {0}: {1}", entry.Path, ex.Message); } - if (m_options.SkipFilesLargerThan == long.MaxValue || m_options.SkipFilesLargerThan == 0 || size < m_options.SkipFilesLargerThan) Logging.Log.WriteVerboseMessage(LOGTAG, "IncludeFile", "Including file: {0} ({1})", entry.Path, size < 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(size)); else diff --git a/Duplicati/Library/Snapshots/ISnapshotService.cs b/Duplicati/Library/Snapshots/ISnapshotService.cs index 4deea290f..fa95438b8 100644 --- a/Duplicati/Library/Snapshots/ISnapshotService.cs +++ b/Duplicati/Library/Snapshots/ISnapshotService.cs @@ -57,7 +57,7 @@ namespace Duplicati.Library.Snapshots /// The path to get /// True if the path is a folder /// The filesystem entry - ISourceFileEntry GetFilesystemEntry(string path, bool isFolder); + ISourceFileEntry? GetFilesystemEntry(string path, bool isFolder); /// /// Gets the last write time of a given file in UTC diff --git a/Duplicati/Library/Snapshots/LinuxSnapshot.cs b/Duplicati/Library/Snapshots/LinuxSnapshot.cs index 472324299..5df6084d6 100644 --- a/Duplicati/Library/Snapshots/LinuxSnapshot.cs +++ b/Duplicati/Library/Snapshots/LinuxSnapshot.cs @@ -112,15 +112,6 @@ namespace Duplicati.Library.Snapshots /// public override IEnumerable SourceFolders => m_snapShots.Select(e => e.LocalPath); - /// - /// Gets a filesystem entry for a given path - /// - /// The path to get the entry for - /// A flag indicating if the path is a folder - /// The filesystem entry - public override ISourceFileEntry GetFilesystemEntry(string path, bool isFolder) - => new SnapshotSourceFileEntry(this, isFolder ? Util.AppendDirSeparator(path) : path, isFolder, false); - /// /// Enumerates the root source files and folders /// diff --git a/Duplicati/Library/Snapshots/SnapshotBase.cs b/Duplicati/Library/Snapshots/SnapshotBase.cs index 0c1516790..7fcab2761 100644 --- a/Duplicati/Library/Snapshots/SnapshotBase.cs +++ b/Duplicati/Library/Snapshots/SnapshotBase.cs @@ -73,7 +73,14 @@ namespace Duplicati.Library.Snapshots /// A flag indicating if the path is a folder /// The filesystem entry public virtual ISourceFileEntry GetFilesystemEntry(string path, bool isFolder) - => new SnapshotSourceFileEntry(this, isFolder ? Util.AppendDirSeparator(path) : path, isFolder, false); + { + if (isFolder && !DirectoryExists(path)) + return null; + else if (!isFolder && !FileExists(path)) + return null; + + return new SnapshotSourceFileEntry(this, isFolder ? Util.AppendDirSeparator(path) : path, isFolder, false); + } /// /// Enumerates files and folders in the given folder diff --git a/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs b/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs new file mode 100644 index 000000000..740c51e00 --- /dev/null +++ b/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs @@ -0,0 +1,216 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using Duplicati.Library.Interface; +using Duplicati.Library.Utility; + +namespace Duplicati.Library.SourceProvider; + +/// +/// Creates a new backend source entry +/// +/// The parent backend +/// The path of the entry +/// True if the entry is a folder +/// True if the entry is a meta entry +/// The creation time of the entry +/// The last modification time of the entry +/// The size of the entry +public class BackendSourceFileEntry(BackendSourceProvider parent, string path, bool isFolder, bool isMetaEntry, DateTime createdUtc, DateTime lastModificationUtc, long size) + : ISourceFileEntry +{ + /// + public bool IsFolder => isFolder; + + /// + public bool IsMetaEntry => isMetaEntry; + + /// + public bool IsRootEntry => isMetaEntry; + + /// + public DateTime CreatedUtc => createdUtc; + + /// + public DateTime LastModificationUtc => lastModificationUtc; + + /// + public string Path => ConcatPaths(parent.PathKey, NormalizePath(path)); + + /// + /// The local path of the entry + /// + public string LocalPath => path; + + /// + public long Size => size; + /// + public bool IsSymlink => false; + + /// + public string? SymlinkTarget => null; + + /// + public FileAttributes Attributes => IsFolder + ? FileAttributes.Directory + : FileAttributes.Normal; + + /// + public Dictionary MinorMetadata => new Dictionary(); + + /// + public bool IsBlockDevice => false; + + /// + public bool IsCharacterDevice => false; + + /// + public bool IsAlternateStream => false; + + /// + public string? HardlinkTargetId => null; + + /// + public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) + { + if (!isFolder) + throw new InvalidOperationException("Enumerate can only be called on folders"); + + if (parent.WrappedBackend is IFolderEnabledBackend folderEnabledBackend) + { + return folderEnabledBackend.ListAsync(path, cancellationToken) + .Select(x => new BackendSourceFileEntry(parent, x.Path, x.IsFolder, false, x.CreatedUtc, x.LastModificationUtc, x.Size)); + } + else + { + // If we do not support folders, we can only list the root + if (this.IsMetaEntry) + return parent.WrappedBackend.List() + .Select(x => FromFileEntry(parent, "", x)) + .ToAsyncEnumerable(); + + return Array.Empty().ToAsyncEnumerable(); + } + } + + /// + public async Task FileExists(string path, CancellationToken cancellationToken) + { + if (!isFolder) + throw new InvalidOperationException("FileExists can only be called on folders"); + + if (parent.WrappedBackend is IFolderEnabledBackend folderEnabledBackend) + { + try + { + var entry = await folderEnabledBackend.GetEntryAsync(path, cancellationToken); + return entry != null && !entry.IsFolder; + } + catch (FileNotFoundException) + { + return false; + } + } + + return false; + } + + /// + public Task OpenMetadataRead(CancellationToken cancellationToken) + => Task.FromResult(null); + + /// + public async Task OpenRead(CancellationToken cancellationToken) + { + if (isFolder) + throw new InvalidOperationException("OpenRead can only be called on files"); + + if (parent.WrappedBackend is IStreamingBackend streamingBackend) + { + var mapStream = new MapStream(); + mapStream.CopyTask = streamingBackend.GetAsync(this.Path, mapStream, cancellationToken); + return mapStream; + } + else + { + TempFileStream? file = null; + try + { + file = TempFileStream.Create(); + await parent.WrappedBackend.GetAsync(this.Path, file.Path, cancellationToken); + file.Position = 0; + return file; + } + catch + { + file?.Dispose(); + throw; + } + } + } + + /// + /// Normalizes the path, turning backslashes into forward slashes + /// + /// The path to normalize + /// The normalized path + public static string NormalizePath(string path) + { + if (System.IO.Path.DirectorySeparatorChar == '/') + return path; + + if (string.IsNullOrEmpty(path)) + return path; + + return path.Replace('\\', '/'); + } + + /// + /// Concatenates two paths + /// + /// The first path + /// The second path + /// The concatenated path + public static string ConcatPaths(string path1, string path2) + { + if (string.IsNullOrEmpty(path1)) + return path2; + + if (string.IsNullOrEmpty(path2)) + return path1; + + if (path1.EndsWith('/') || path1.EndsWith('\\')) + return path1 + path2; + + return path1 + "/" + path2; + } + + /// + /// Creates a new backend source entry from a file entry + /// + /// The parent backend + /// The file entry + /// The prefix to add to the path + /// The new backend source entry + public static BackendSourceFileEntry FromFileEntry(BackendSourceProvider parent, string prefix, IFileEntry entry) + => new BackendSourceFileEntry(parent, ConcatPaths(prefix, entry.Name), entry.IsFolder, false, entry.Created, entry.LastModification, entry.Size); +} + diff --git a/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs index 9b144a7ab..815c7e48a 100644 --- a/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs +++ b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs @@ -20,7 +20,6 @@ // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Interface; -using Duplicati.Library.Utility; namespace Duplicati.Library.SourceProvider; @@ -28,179 +27,30 @@ namespace Duplicati.Library.SourceProvider; /// A source provider that wraps a backend /// /// The backend to wrap -public class BackendSourceProvider : ISourceProvider +public class BackendSourceProvider(IFolderEnabledBackend backend) : ISourceProvider { /// /// The wrapped backend /// - public IBackend WrappedBackend { get; init; } + public IFolderEnabledBackend WrappedBackend => backend; /// /// The path key unique for the destination /// - public readonly string PathKey; - - /// - /// Creates a new backend source provider - /// - /// The backend to wrap - /// The URL of the backend - public BackendSourceProvider(IBackend backend, string url) - { - WrappedBackend = backend; - var uri = new Utility.Uri(url); - // TODO: The username may be provided in other ways to the backend... - PathKey = $"{uri.Scheme}://{uri.Host}:{uri.Port}/~{uri.Username}/{uri.Path}/"; - } - - /// - /// Creates a new backend source entry - /// - /// The parent backend - /// The path of the entry - /// True if the entry is a folder - /// True if the entry is a meta entry - /// The creation time of the entry - /// The last modification time of the entry - /// The size of the entry - private class BackendSourceFileEntry(BackendSourceProvider parent, string path, bool isFolder, bool isMetaEntry, DateTime createdUtc, DateTime lastModificationUtc, long size) - : ISourceFileEntry - { - /// - public bool IsFolder => isFolder; - - /// - public bool IsMetaEntry => isMetaEntry; - - /// - public bool IsRootEntry => isMetaEntry; - - /// - public DateTime CreatedUtc => createdUtc; - - /// - public DateTime LastModificationUtc => lastModificationUtc; - - /// - public string Path => $"{parent.PathKey}{path}"; - - /// - public long Size => size; - /// - public bool IsSymlink => false; - - /// - public string? SymlinkTarget => null; - - /// - public FileAttributes Attributes => IsFolder - ? FileAttributes.Directory - : FileAttributes.Normal; - - /// - public Dictionary MinorMetadata => new Dictionary(); - - /// - public bool IsBlockDevice => false; - - /// - public bool IsCharacterDevice => false; - - /// - public bool IsAlternateStream => false; - - /// - public string? HardlinkTargetId => null; - - /// - public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) - { - if (!isFolder) - throw new InvalidOperationException("Enumerate can only be called on folders"); - - if (parent.WrappedBackend is IFolderEnabledBackend folderEnabledBackend) - { - return folderEnabledBackend.ListAsync(path, cancellationToken) - .Select(x => new BackendSourceFileEntry(parent, x.Path, x.IsFolder, false, new DateTime(0), x.LastModificationUtc, x.Size)); - } - else - { - - // If we do not support folders, we can only list the root - if (this.IsMetaEntry) - return parent.WrappedBackend.List() - .Select(x => new BackendSourceFileEntry(parent, x.Name, x.IsFolder, false, new DateTime(0), x.LastModification, x.Size)) - .ToAsyncEnumerable(); - - return Array.Empty().ToAsyncEnumerable(); - } - } - - /// - public async Task FileExists(string path, CancellationToken cancellationToken) - { - if (!isFolder) - throw new InvalidOperationException("FileExists can only be called on folders"); - - if (parent.WrappedBackend is IFolderEnabledBackend folderEnabledBackend) - { - try - { - var entry = await folderEnabledBackend.GetEntryAsync(path, cancellationToken); - return entry != null && !entry.IsFolder; - } - catch (FileNotFoundException) - { - return false; - } - } - - return false; - } - - /// - public Task OpenMetadataRead(CancellationToken cancellationToken) - => Task.FromResult(null); - - /// - public async Task OpenRead(CancellationToken cancellationToken) - { - if (isFolder) - throw new InvalidOperationException("OpenRead can only be called on files"); - - if (parent.WrappedBackend is IStreamingBackend streamingBackend) - { - var mapStream = new MapStream(); - mapStream.CopyTask = streamingBackend.GetAsync(this.Path, mapStream, cancellationToken); - return mapStream; - } - else - { - TempFileStream? file = null; - try - { - file = TempFileStream.Create(); - await parent.WrappedBackend.GetAsync(this.Path, file.Path, cancellationToken); - file.Position = 0; - return file; - } - catch - { - file?.Dispose(); - throw; - } - } - } - } + public string PathKey => backend.PathKey; /// public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) - => new[] { new BackendSourceFileEntry(this, "", true, true, new DateTime(0), new DateTime(0), 0) }.ToAsyncEnumerable(); + => new[] { new BackendSourceFileEntry(this, PathKey, true, true, new DateTime(0), new DateTime(0), 0) }.ToAsyncEnumerable(); /// - public Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) - // TODO: If we cache the enumerate result, we could check if the path exists in the cache and return it - => Task.FromResult(new BackendSourceFileEntry(this, path, isFolder, false, new DateTime(0), new DateTime(0), -1)); + public async Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) + { + if (this.WrappedBackend is IFolderEnabledBackend folderEnabledBackend) + return await folderEnabledBackend.GetEntryAsync(path, cancellationToken); + + return null; + } /// public void Dispose() diff --git a/Duplicati/Library/SourceProvider/Builtin/Combiner.cs b/Duplicati/Library/SourceProvider/Builtin/Combiner.cs index 1dcb01fa8..7f027847f 100644 --- a/Duplicati/Library/SourceProvider/Builtin/Combiner.cs +++ b/Duplicati/Library/SourceProvider/Builtin/Combiner.cs @@ -29,6 +29,9 @@ namespace Duplicati.Library.SourceProvider; /// public class Combiner(IEnumerable providers) : ISourceProvider { + /// + public string PathKey => string.Empty; + /// /// The providers to combine /// @@ -56,10 +59,23 @@ public class Combiner(IEnumerable providers) : ISourceProvider /// True if the path is a folder /// The cancellation token /// The filesystem entry - public Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) + public async Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) { - // TODO: Map roots to the paths so we get the right provider - throw new NotImplementedException(); + foreach (var provider in providers) + if (!string.IsNullOrWhiteSpace(PathKey) && provider.PathKey.StartsWith(path)) + { + var subpath = provider.PathKey.Substring(path.Length); + return await provider.GetEntry(subpath, isFolder, cancellationToken).ConfigureAwait(false); + } + + foreach (var provider in providers.Where(x => string.IsNullOrWhiteSpace(x.PathKey))) + { + var res = await provider.GetEntry(path, isFolder, cancellationToken).ConfigureAwait(false); + if (res != null) + return res; + } + + return null; } /// diff --git a/Duplicati/Library/SourceProvider/Builtin/File.cs b/Duplicati/Library/SourceProvider/Builtin/LocalFileSource.cs similarity index 86% rename from Duplicati/Library/SourceProvider/Builtin/File.cs rename to Duplicati/Library/SourceProvider/Builtin/LocalFileSource.cs index 985fc072c..9c75ca3bb 100644 --- a/Duplicati/Library/SourceProvider/Builtin/File.cs +++ b/Duplicati/Library/SourceProvider/Builtin/LocalFileSource.cs @@ -28,8 +28,11 @@ namespace Duplicati.Library.SourceProvider; /// A source provider that wraps a file system through a snapshot service /// /// The snapshot service to use -public class File(ISnapshotService snapshotService) : ISourceProvider +public class LocalFileSource(ISnapshotService snapshotService) : ISourceProvider { + /// + public string PathKey => string.Empty; + /// public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) => snapshotService.EnumerateFilesystemEntries().ToAsyncEnumerable(); @@ -40,7 +43,7 @@ public class File(ISnapshotService snapshotService) : ISourceProvider public ISnapshotService SnapshotService => snapshotService; /// - public Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) + public Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) => Task.FromResult(snapshotService.GetFilesystemEntry(path, isFolder)); /// From 0e8df7ccd88ee6ad405afa5635f968a730fcf55b Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 17 Feb 2025 14:05:21 +0100 Subject: [PATCH 09/28] Removed unsupported code --- Duplicati/Library/Backend/File/FileBackend.cs | 35 +----------------- .../Builtin/BackendSourceFileEntry.cs | 36 +++++-------------- .../Builtin/BackendSourceProvider.cs | 9 ++--- 3 files changed, 12 insertions(+), 68 deletions(-) diff --git a/Duplicati/Library/Backend/File/FileBackend.cs b/Duplicati/Library/Backend/File/FileBackend.cs index 3a8b76ca8..f13f6889a 100644 --- a/Duplicati/Library/Backend/File/FileBackend.cs +++ b/Duplicati/Library/Backend/File/FileBackend.cs @@ -21,12 +21,9 @@ using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; -using Duplicati.Library.Snapshots; using System; using System.Collections.Generic; using System.IO; -using System.Linq; -using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Versioning; using System.Threading; @@ -36,7 +33,7 @@ namespace Duplicati.Library.Backend { // ReSharper disable once UnusedMember.Global // This class is instantiated dynamically in the BackendLoader. - public class File : IBackend, IStreamingBackend, IQuotaEnabledBackend, IRenameEnabledBackend, IFolderEnabledBackend + public class File : IBackend, IStreamingBackend, IQuotaEnabledBackend, IRenameEnabledBackend { private const string OPTION_DESTINATION_MARKER = "alternate-destination-marker"; private const string OPTION_ALTERNATE_PATHS = "alternate-target-paths"; @@ -467,35 +464,5 @@ namespace Duplicati.Library.Backend throw; } } - - /// - public string PathKey => m_path; - - /// - public async IAsyncEnumerable ListAsync(string path, [EnumeratorCancellation] CancellationToken cancellationToken) - { - PreAuthenticate(); - - var remotePath = string.IsNullOrWhiteSpace(path) ? m_path : GetRemoteName(path); - - if (!systemIO.DirectoryExists(remotePath)) - throw new FolderMissingException(Strings.FileBackend.FolderMissingError(m_path)); - - var service = SnapshotUtility.CreateNoSnapshot([remotePath], false, false); - - await foreach (var folderEntry in systemIO.EnumerateDirectories(remotePath).ToAsyncEnumerable().WithCancellation(cancellationToken).ConfigureAwait(false)) - yield return new SnapshotSourceFileEntry(service, folderEntry, true, false); - - await foreach (var fileEntry in systemIO.EnumerateFiles(remotePath).ToAsyncEnumerable().WithCancellation(cancellationToken).ConfigureAwait(false)) - yield return new SnapshotSourceFileEntry(service, fileEntry, false, false); - } - - /// - public Task GetEntryAsync(string path, CancellationToken cancellationToken) - { - var service = SnapshotUtility.CreateNoSnapshot([path], false, false); - var isFolder = service.DirectoryExists(path); - return Task.FromResult(new SnapshotSourceFileEntry(service, path, isFolder, false)); - } } } diff --git a/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs b/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs index 740c51e00..7b6fa496c 100644 --- a/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs +++ b/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs @@ -94,21 +94,8 @@ public class BackendSourceFileEntry(BackendSourceProvider parent, string path, b if (!isFolder) throw new InvalidOperationException("Enumerate can only be called on folders"); - if (parent.WrappedBackend is IFolderEnabledBackend folderEnabledBackend) - { - return folderEnabledBackend.ListAsync(path, cancellationToken) - .Select(x => new BackendSourceFileEntry(parent, x.Path, x.IsFolder, false, x.CreatedUtc, x.LastModificationUtc, x.Size)); - } - else - { - // If we do not support folders, we can only list the root - if (this.IsMetaEntry) - return parent.WrappedBackend.List() - .Select(x => FromFileEntry(parent, "", x)) - .ToAsyncEnumerable(); - - return Array.Empty().ToAsyncEnumerable(); - } + return parent.WrappedBackend.ListAsync(path, cancellationToken) + .Select(x => new BackendSourceFileEntry(parent, x.Path, x.IsFolder, false, x.CreatedUtc, x.LastModificationUtc, x.Size)); } /// @@ -117,20 +104,15 @@ public class BackendSourceFileEntry(BackendSourceProvider parent, string path, b if (!isFolder) throw new InvalidOperationException("FileExists can only be called on folders"); - if (parent.WrappedBackend is IFolderEnabledBackend folderEnabledBackend) + try { - try - { - var entry = await folderEnabledBackend.GetEntryAsync(path, cancellationToken); - return entry != null && !entry.IsFolder; - } - catch (FileNotFoundException) - { - return false; - } + var entry = await parent.WrappedBackend.GetEntryAsync(path, cancellationToken); + return entry != null && !entry.IsFolder; + } + catch (FileNotFoundException) + { + return false; } - - return false; } /// diff --git a/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs index 815c7e48a..6411cc0a7 100644 --- a/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs +++ b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs @@ -44,13 +44,8 @@ public class BackendSourceProvider(IFolderEnabledBackend backend) : ISourceProvi => new[] { new BackendSourceFileEntry(this, PathKey, true, true, new DateTime(0), new DateTime(0), 0) }.ToAsyncEnumerable(); /// - public async Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) - { - if (this.WrappedBackend is IFolderEnabledBackend folderEnabledBackend) - return await folderEnabledBackend.GetEntryAsync(path, cancellationToken); - - return null; - } + public Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) + => backend.GetEntryAsync(path, cancellationToken); /// public void Dispose() From a99aa87dbba84062a0269eb78c31d7ece6a2d198 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 17 Feb 2025 19:51:26 +0100 Subject: [PATCH 10/28] Fixed issue with Windows paths being stored in extended format --- Duplicati/Library/Snapshots/NoSnapshotWindows.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs index 3108484c4..b4a3b16e0 100644 --- a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs +++ b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs @@ -86,7 +86,7 @@ namespace Duplicati.Library.Snapshots /// The source files and folders public override IEnumerable EnumerateFilesystemEntries() { - foreach (var folder in m_folders.Select(SystemIOWindows.AddExtendedDevicePathPrefix)) + foreach (var folder in m_folders.Select(SystemIOWindows.RemoveExtendedDevicePathPrefix)) { if (DirectoryExists(folder) || folder.EndsWith(System.IO.Path.DirectorySeparatorChar)) yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(folder), true, true); From f61a1d0ad8bc398687e8904154a8d86841261cdc Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 18 Feb 2025 15:13:46 +0100 Subject: [PATCH 11/28] Fixed Windows VSS support --- Duplicati/Library/Snapshots/WindowsSnapshot.cs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Snapshots/WindowsSnapshot.cs b/Duplicati/Library/Snapshots/WindowsSnapshot.cs index 876554595..07de392c5 100644 --- a/Duplicati/Library/Snapshots/WindowsSnapshot.cs +++ b/Duplicati/Library/Snapshots/WindowsSnapshot.cs @@ -55,6 +55,11 @@ namespace Duplicati.Library.Snapshots /// private readonly VssBackupComponents _vssBackupComponents; + /// + /// The source folders included in the snapshot + /// + private readonly IReadOnlyList _sourceFolders; + /// /// Constructs a new backup snapshot, using all the required disks /// @@ -65,7 +70,7 @@ namespace Duplicati.Library.Snapshots : base(followSymlinks) { // For Windows, ensure we don't store paths with extended device path prefixes (i.e., @"\\?\" or @"\\?\UNC\") - sources = sources.Select(SystemIOWindows.RemoveExtendedDevicePathPrefix); + _sourceFolders = sources.Select(SystemIOWindows.RemoveExtendedDevicePathPrefix).ToList(); try { _vssBackupComponents = new VssBackupComponents(); @@ -82,7 +87,7 @@ namespace Duplicati.Library.Snapshots } _vssBackupComponents.SetupWriters(null, excludedWriters); - _vssBackupComponents.InitShadowVolumes(sources); + _vssBackupComponents.InitShadowVolumes(_sourceFolders); _vssBackupComponents.MapVolumesToSnapShots(); @@ -122,7 +127,7 @@ namespace Duplicati.Library.Snapshots /// The source files and folders public override IEnumerable EnumerateFilesystemEntries() { - foreach (var folder in _vssBackupComponents.SnapshotLocalPaths) + foreach (var folder in _sourceFolders) { if (folder.EndsWith(Path.DirectorySeparatorChar) || DirectoryExists(folder)) yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(folder), true, true); From 12d0c9710d038062c4905e7f2e6426403b302a93 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 18 Feb 2025 17:17:16 +0100 Subject: [PATCH 12/28] Added additional fix for VSS --- Duplicati/Library/Snapshots/WindowsSnapshot.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Snapshots/WindowsSnapshot.cs b/Duplicati/Library/Snapshots/WindowsSnapshot.cs index 07de392c5..8d14674ec 100644 --- a/Duplicati/Library/Snapshots/WindowsSnapshot.cs +++ b/Duplicati/Library/Snapshots/WindowsSnapshot.cs @@ -119,7 +119,7 @@ namespace Duplicati.Library.Snapshots /// /// Gets the source folders /// - public override IEnumerable SourceFolders => _vssBackupComponents.SnapshotLocalPaths; + public override IEnumerable SourceFolders => _sourceFolders; /// /// Enumerates the root source files and folders From 73d0ad905f099854a9b1722dd4a3e200dd47bf5f Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 18 Feb 2025 22:12:37 +0100 Subject: [PATCH 13/28] Fixed USN support --- .../Backup/FileEnumerationProcess.cs | 53 ++-- .../Snapshots/RecursiveFilterHelper.cs | 229 --------------- .../Library/Snapshots/UsnJournalService.cs | 278 +++++++++++++++++- 3 files changed, 307 insertions(+), 253 deletions(-) delete mode 100644 Duplicati/Library/Snapshots/RecursiveFilterHelper.cs diff --git a/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs b/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs index a1ef2ec65..a7db2b2ce 100644 --- a/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs @@ -33,6 +33,7 @@ using Duplicati.Library.Interface; using System.Runtime.CompilerServices; using Duplicati.Library.Snapshots; using Duplicati.Library.SourceProvider; +using Duplicati.Library.Common.IO; namespace Duplicati.Library.Main.Operation.Backup { @@ -96,22 +97,30 @@ namespace Duplicati.Library.Main.Operation.Backup if (ignorenames != null && ignorenames.Length == 0) ignorenames = null; - // If we have a specific list, use that instead of enumerating the filesystem - async IAsyncEnumerable ExpandSources(IEnumerable list) - { - foreach (var s in list) - { - var r = await sourceProvider.GetEntry(s, false, token).ConfigureAwait(false); - if (r != null) - yield return r; - } - } + // Shared filter function with bound variables + ValueTask FilterEntry(ISourceFileEntry entry) + => SourceFileEntryFilter(entry, blacklistPaths, hardlinkPolicy, symlinkPolicy, hardlinkmap, fileAttributeFilter, enumeratefilter, ignorenames, mixinqueue, token); + + // Prepare the work list IAsyncEnumerable worklist; + + // If we have a specific list, use that instead of enumerating the filesystem if (changedfilelist != null && changedfilelist.Any()) { - worklist = ExpandSources(changedfilelist).WhereAwait(x => - SourceFileEntryFilter(x, blacklistPaths, hardlinkPolicy, symlinkPolicy, hardlinkmap, fileAttributeFilter, enumeratefilter, ignorenames, mixinqueue, token)); + async IAsyncEnumerable ExpandSources(IEnumerable list) + { + foreach (var s in list) + { + var r = await sourceProvider.GetEntry(s, s.EndsWith(Path.DirectorySeparatorChar), token).ConfigureAwait(false); + if (r != null) + { + //TODO: Set r.IsRoot = true for source elements + yield return r; + } + } + } + worklist = ExpandSources(changedfilelist).WhereAwait(FilterEntry); } else if (journalService != null) { @@ -127,17 +136,27 @@ namespace Duplicati.Library.Main.Operation.Backup if (fileProviders.Count > 1) throw new InvalidOperationException("Multiple file providers found, but USN only supports one"); - worklist = RecursiveFilterHelper.GetModifiedSources( - ExpandSources(journalService.VolumeDataList.SelectMany(x => x.Folders.Concat(x.Files))), - fileProviders.First().SnapshotService, - x => SourceFileEntryFilter(x, blacklistPaths, hardlinkPolicy, symlinkPolicy, hardlinkmap, fileAttributeFilter, enumeratefilter, ignorenames, mixinqueue, token), + // TODO: This is not as effecient as possible. + // If the root folder is marked changed by USN, the expansion with RecurseEntries + // will cause a full regular scan. It should be possible to *only* process the + // changed elements as returned from the USN journal. + // It should be possible to remove RecurseEntries from the GetModifiedSources() + // enumeration result. + // Such a change requires significant testing as there are many pitfalls with USN. + worklist = RecurseEntries(journalService.GetModifiedSources(FilterEntry, token), + FilterEntry, token + ) + .Concat( + RecurseEntries(journalService.GetFullScanSources(token), + FilterEntry, + token) ); } else { worklist = RecurseEntries(sourceProvider.Enumerate(token), - x => SourceFileEntryFilter(x, blacklistPaths, hardlinkPolicy, symlinkPolicy, hardlinkmap, fileAttributeFilter, enumeratefilter, ignorenames, mixinqueue, token), + FilterEntry, token ); } diff --git a/Duplicati/Library/Snapshots/RecursiveFilterHelper.cs b/Duplicati/Library/Snapshots/RecursiveFilterHelper.cs deleted file mode 100644 index 1e8fb9f02..000000000 --- a/Duplicati/Library/Snapshots/RecursiveFilterHelper.cs +++ /dev/null @@ -1,229 +0,0 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - -#nullable enable - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using Duplicati.Library.Interface; -using System.Threading.Tasks; -using System.Runtime.CompilerServices; - -namespace Duplicati.Library.Snapshots; - -public static class RecursiveFilterHelper -{ - /// - /// The log tag to use - /// - private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(RecursiveFilterHelper)); - - /// - /// Filters sources, returning sub-set having been modified since last - /// change, as specified by journalData. - /// - /// List of sources - /// Snapshot service - /// Filter callback to exclude filtered items - /// Cancellation token - /// Filtered sources - public static async IAsyncEnumerable GetModifiedSources(IAsyncEnumerable entries, ISnapshotService snapshot, Func> filter, [EnumeratorCancellation] CancellationToken token) - { - // prepare cache for includes (value = true) and excludes (value = false, will be populated - // on-demand) - var cache = new Dictionary(); - await foreach (var source in entries) - { - if (token.IsCancellationRequested) - break; - - cache[source.Path] = true; - } - - // Check the simplified folders, and their parent folders against the exclusion filter. - // This is needed because the filter may exclude "C:\A\", but this won't match the more - // specific "C:\A\B\" in our list, even though it's meant to be excluded. - // The reason why the filter doesn't exclude it is because during a regular (non-USN) full scan, - // FilterHandler.EnumerateFilesAndFolders() works top-down, and won't even enumerate child - // folders. - // The sources are needed to stop evaluating parent folders above the specified source folders - await foreach (var folder in FilterExcludedFolders(entries.Where(x => x.IsFolder), snapshot, filter, cache, token)) - { - if (token.IsCancellationRequested) - break; - - if (!snapshot.DirectoryExists(folder.Path)) - continue; - - yield return folder; - } - - // The simplified file list also needs to be checked against the exclusion filter, as it - // may contain entries excluded due to attributes, but also because they are below excluded - // folders, which themselves aren't in the folder list from step 1. - // Note that the simplified file list may contain entries that have been deleted! They need to - // be kept in the list (unless excluded by the filter) in order for the backup handler to record their - // deletion. - await foreach (var files in FilterExcludedFiles(entries.Where(x => !x.IsFolder), snapshot, filter, cache, token)) - { - if (token.IsCancellationRequested) - break; - - if (!snapshot.FileExists(files.Path)) - continue; - - yield return files; - } - } - - /// - /// Filter supplied files, removing any files which itself, or one - /// of its parent folders, is excluded by the filter. - /// - /// Files to filter - /// Snapshot service - /// Exclusion filter - /// Cache of included and excluded files / folders - /// Cancellation token - /// Filtered files - private static async IAsyncEnumerable FilterExcludedFiles( - IAsyncEnumerable files, - ISnapshotService snapshot, - Func> filter, - IDictionary cache, - [EnumeratorCancellation] CancellationToken token) - { - await foreach (var file in files.WithCancellation(token).ConfigureAwait(false)) - { - try - { - if (!await filter(file)) - continue; - - var parentPath = Utility.Utility.GetParent(file.Path, true); - if (!string.IsNullOrWhiteSpace(parentPath)) - { - var parent = snapshot.GetFilesystemEntry(parentPath, true); - if (await IsFolderOrAncestorsExcluded(parent, snapshot, filter, cache, token)) - continue; - } - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "FilterExcludedFilesError", ex, "Error while filtering file: {0}", ex.Message); - continue; - } - - yield return file; - } - } - - /// - /// Filter supplied folders, removing any folder which itself, or one - /// of its ancestors, is excluded by the filter. - /// - /// Folder to filter - /// Snapshot service - /// Exclusion filter - /// Cache of excluded folders - /// Cancellation token - /// Filtered folders - private static async IAsyncEnumerable FilterExcludedFolders( - IAsyncEnumerable folders, - ISnapshotService snapshot, - Func> filter, - IDictionary cache, - [EnumeratorCancellation] CancellationToken token) - { - await foreach (var folder in folders.WithCancellation(token).ConfigureAwait(false)) - { - try - { - if (await IsFolderOrAncestorsExcluded(folder, snapshot, filter, cache, token)) - continue; - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "FilterExcludedFoldersError", ex, "Error while filtering folder: {0}", ex.Message); - continue; - } - - yield return folder; - } - } - - /// - /// Tests if specified folder, or any of its ancestors, is excluded by the filter - /// - /// Folder to test - /// Filter - /// Cache of excluded folders (optional) - /// True if excluded, false otherwise - private static async ValueTask IsFolderOrAncestorsExcluded( - ISourceFileEntry folder, - ISnapshotService snapshot, - Func> filter, - IDictionary cache, - CancellationToken token) - { - List? parents = null; - while (folder != null) - { - if (token.IsCancellationRequested) - break; - - // first check cache - if (cache.TryGetValue(folder.Path, out var include)) - { - if (include) - return false; - - break; // hit! - } - - parents ??= []; // create on-demand - - // remember folder for cache - parents.Add(folder.Path); - - - if (!await filter(folder)) - break; // excluded - - var parentPath = Utility.Utility.GetParent(folder.Path, true); - if (string.IsNullOrWhiteSpace(parentPath)) - break; - - folder = snapshot.GetFilesystemEntry(parentPath, true); - } - - if (folder != null) - { - // update cache - parents?.ForEach(p => cache[p] = false); - } - - return folder != null; - } -} - diff --git a/Duplicati/Library/Snapshots/UsnJournalService.cs b/Duplicati/Library/Snapshots/UsnJournalService.cs index 47cd08baa..09583cef8 100644 --- a/Duplicati/Library/Snapshots/UsnJournalService.cs +++ b/Duplicati/Library/Snapshots/UsnJournalService.cs @@ -19,6 +19,7 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +#nullable enable using System; using System.Collections.Generic; @@ -29,6 +30,8 @@ using System.Threading; using Duplicati.Library.Interface; using Duplicati.Library.Common.IO; using Duplicati.Library.Utility; +using System.Threading.Tasks; +using System.Runtime.CompilerServices; namespace Duplicati.Library.Snapshots { @@ -70,6 +73,9 @@ namespace Duplicati.Library.Snapshots m_token = token; } + /// + /// The volume data lists for the mapped root drives + /// public IEnumerable VolumeDataList => m_volumeDataDict.Select(e => e.Value); /// @@ -90,7 +96,7 @@ namespace Duplicati.Library.Snapshots // get hash identifying current source filter / sources configuration var configHash = Utility.Utility.ByteArrayAsHexString(MD5HashHelper.GetHash(new string[] { - emitFilter == null ? string.Empty : emitFilter.ToString(), + emitFilter?.ToString() ?? string.Empty, string.Join("; ", m_snapshot.SourceFolders), fileAttributeFilter.ToString(), skipFilesLargerThan.ToString() @@ -218,6 +224,261 @@ namespace Duplicati.Library.Snapshots return result; } + /// + /// Returns all sources that should be fully scanned + /// + /// Cancellation token + /// Filtered sources + public async IAsyncEnumerable GetFullScanSources([EnumeratorCancellation] CancellationToken token) + { + // Make the method async enumerable + await Task.CompletedTask; + + foreach(var volumeData in m_volumeDataDict.Values.Where(x => x.IsFullScan)) + { + if (volumeData.Folders != null) + { + foreach(var folderPath in volumeData.Folders) + { + if (token.IsCancellationRequested) + yield break; + + var folder = m_snapshot.GetFilesystemEntry(folderPath, true); + if (folder != null) + yield return folder; + else + Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "FolderLookupError", null, "No entry while locating source folder {0}", folderPath); + } + } + + if (volumeData.Files != null) + { + foreach(var filePath in volumeData.Files) + { + if (token.IsCancellationRequested) + yield break; + + var file = m_snapshot.GetFilesystemEntry(filePath, false); + if (file != null) + yield return file; + else + Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "FolderLookupError", null, "No entry while locating source file {0}", filePath); + } + } + } + } + + + /// + /// Filters sources, returning sub-set having been modified since last + /// change, as specified by journalData. + /// + /// Filter callback to exclude filtered items + /// Cancellation token + /// Filtered sources + public async IAsyncEnumerable GetModifiedSources(Func> filter, [EnumeratorCancellation] CancellationToken token) + { + foreach(var volumeData in m_volumeDataDict.Values.Where(x => !x.IsFullScan)) + { + // prepare cache for includes (value = true) and excludes (value = false, will be populated + // on-demand) + var cache = new Dictionary(); + foreach (var source in m_snapshot.SourceFolders) + { + if (token.IsCancellationRequested) + yield break; + + cache[source] = true; + } + + // Check the simplified folders, and their parent folders against the exclusion filter. + // This is needed because the filter may exclude "C:\A\", but this won't match the more + // specific "C:\A\B\" in our list, even though it's meant to be excluded. + // The reason why the filter doesn't exclude it is because during a regular (non-USN) full scan, + // FilterHandler.EnumerateFilesAndFolders() works top-down, and won't even enumerate child + // folders. + // The sources are needed to stop evaluating parent folders above the specified source folders + await foreach (var folder in FilterExcludedFolders(volumeData.Folders, m_snapshot, filter, cache, token).ConfigureAwait(false)) + { + if (token.IsCancellationRequested) + yield break; + + if (!m_snapshot.DirectoryExists(folder.Path)) + continue; + + yield return folder; + } + + // The simplified file list also needs to be checked against the exclusion filter, as it + // may contain entries excluded due to attributes, but also because they are below excluded + // folders, which themselves aren't in the folder list from step 1. + // Note that the simplified file list may contain entries that have been deleted! They need to + // be kept in the list (unless excluded by the filter) in order for the backup handler to record their + // deletion. + await foreach (var files in FilterExcludedFiles(volumeData.Files, m_snapshot, filter, cache, token).ConfigureAwait(false)) + { + if (token.IsCancellationRequested) + yield break; + + if (!m_snapshot.FileExists(files.Path)) + continue; + + yield return files; + } + } + } + + /// + /// Filter supplied files, removing any files which itself, or one + /// of its parent folders, is excluded by the filter. + /// + /// Files to filter + /// Snapshot service + /// Exclusion filter + /// Cache of included and excluded files / folders + /// Cancellation token + /// Filtered files + private static async IAsyncEnumerable FilterExcludedFiles( + IEnumerable? files, + ISnapshotService snapshot, + Func> filter, + IDictionary cache, + [EnumeratorCancellation] CancellationToken token) + { + if (files == null) + yield break; + + foreach (var filePath in files) + { + ISourceFileEntry? file; + try + { + file = snapshot.GetFilesystemEntry(filePath, false); + if (file == null) + continue; + + if (!await filter(file)) + continue; + + var parentPath = Utility.Utility.GetParent(file.Path, true); + if (!string.IsNullOrWhiteSpace(parentPath)) + { + var parent = snapshot.GetFilesystemEntry(parentPath, true); + if (parent == null) + continue; + + if (await IsFolderOrAncestorsExcluded(parent, snapshot, filter, cache, token)) + continue; + } + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "FilterExcludedFilesError", ex, "Error while filtering file: {0}", ex.Message); + continue; + } + + if (file != null) + yield return file; + } + } + + /// + /// Filter supplied folders, removing any folder which itself, or one + /// of its ancestors, is excluded by the filter. + /// + /// Folder to filter + /// Snapshot service + /// Exclusion filter + /// Cache of excluded folders + /// Cancellation token + /// Filtered folders + private static async IAsyncEnumerable FilterExcludedFolders( + IEnumerable? folders, + ISnapshotService snapshot, + Func> filter, + IDictionary cache, + [EnumeratorCancellation] CancellationToken token) + { + if (folders == null) + yield break; + + foreach (var folderPath in folders) + { + ISourceFileEntry? folder; + try + { + folder = snapshot.GetFilesystemEntry(folderPath, true); + if (folder == null) + continue; + if (await IsFolderOrAncestorsExcluded(folder, snapshot, filter, cache, token)) + continue; + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "FilterExcludedFoldersError", ex, "Error while filtering folder: {0}", ex.Message); + continue; + } + + yield return folder; + } + } + + /// + /// Tests if specified folder, or any of its ancestors, is excluded by the filter + /// + /// Folder to test + /// Filter + /// Cache of excluded folders (optional) + /// True if excluded, false otherwise + private static async ValueTask IsFolderOrAncestorsExcluded( + ISourceFileEntry inputFolder, + ISnapshotService snapshot, + Func> filter, + IDictionary cache, + CancellationToken token) + { + List? parents = null; + ISourceFileEntry? folder = inputFolder; + + while (folder != null) + { + if (token.IsCancellationRequested) + break; + + // first check cache + if (cache.TryGetValue(folder.Path, out var include)) + { + if (include) + return false; + + break; // hit! + } + + parents ??= []; // create on-demand + + // remember folder for cache + parents.Add(folder.Path); + + + if (!await filter(folder)) + break; // excluded + + var parentPath = Utility.Utility.GetParent(folder.Path, true); + if (string.IsNullOrWhiteSpace(parentPath)) + break; + + folder = snapshot.GetFilesystemEntry(parentPath, true); + } + + if (folder != null) + { + // update cache + parents?.ForEach(p => cache[p] = false); + } + + return folder != null; + } + /// /// Sort sources by root volume /// @@ -260,9 +521,12 @@ namespace Duplicati.Library.Snapshots if (volumeData.IsFullScan) return true; // do not append from previous set, already scanned - if (volumeData.Files.Contains(path)) + if (volumeData.Files != null && volumeData.Files.Contains(path)) return true; // do not append from previous set, already scanned + if (volumeData.Folders == null) + return false; + foreach (var folder in volumeData.Folders) { if (m_token.IsCancellationRequested) @@ -289,23 +553,23 @@ namespace Duplicati.Library.Snapshots /// /// Volume /// - public string Volume { get; set; } + public required string Volume { get; set; } /// /// Set of potentially modified files /// - public HashSet Files { get; internal set; } + public HashSet? Files { get; internal set; } /// /// Set of folders that are potentially modified, or whose children /// are potentially modified /// - public List Folders { get; internal set; } + public List? Folders { get; internal set; } /// /// Journal data to use for next backup /// - public USNJournalDataEntry JournalData { get; internal set; } + public USNJournalDataEntry? JournalData { get; internal set; } /// /// If true, a full scan for this volume was required @@ -315,6 +579,6 @@ namespace Duplicati.Library.Snapshots /// /// Optional exception message for volume /// - public Exception Exception { get; internal set; } + public Exception? Exception { get; internal set; } } } From ac603a4d8f707cf8079c29eb1522c7cfe2fb9bbc Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 18 Feb 2025 22:32:37 +0100 Subject: [PATCH 14/28] Renamed SourceFolders to SourceEntries as it can contain files as well --- Duplicati/Library/Snapshots/ISnapshotService.cs | 4 ++-- Duplicati/Library/Snapshots/LinuxSnapshot.cs | 10 +++++----- Duplicati/Library/Snapshots/NoSnapshotLinux.cs | 12 ++++++------ Duplicati/Library/Snapshots/NoSnapshotWindows.cs | 14 +++++++------- Duplicati/Library/Snapshots/SnapshotBase.cs | 4 ++-- Duplicati/Library/Snapshots/UsnJournalService.cs | 14 +++++++------- Duplicati/Library/Snapshots/WindowsSnapshot.cs | 10 +++++----- 7 files changed, 34 insertions(+), 34 deletions(-) diff --git a/Duplicati/Library/Snapshots/ISnapshotService.cs b/Duplicati/Library/Snapshots/ISnapshotService.cs index fa95438b8..72542e465 100644 --- a/Duplicati/Library/Snapshots/ISnapshotService.cs +++ b/Duplicati/Library/Snapshots/ISnapshotService.cs @@ -36,9 +36,9 @@ namespace Duplicati.Library.Snapshots public interface ISnapshotService : IDisposable { /// - /// Gets the source folders + /// Gets the source entries /// - IEnumerable SourceFolders { get; } + IEnumerable SourceEntries { get; } /// /// Enumerates the root source files and folders diff --git a/Duplicati/Library/Snapshots/LinuxSnapshot.cs b/Duplicati/Library/Snapshots/LinuxSnapshot.cs index 5df6084d6..05cf884d7 100644 --- a/Duplicati/Library/Snapshots/LinuxSnapshot.cs +++ b/Duplicati/Library/Snapshots/LinuxSnapshot.cs @@ -110,7 +110,7 @@ namespace Duplicati.Library.Snapshots /// /// Gets the source folders /// - public override IEnumerable SourceFolders => m_snapShots.Select(e => e.LocalPath); + public override IEnumerable SourceEntries => m_entries.Select(x => x.Key); /// /// Enumerates the root source files and folders @@ -118,12 +118,12 @@ namespace Duplicati.Library.Snapshots /// The source files and folders public override IEnumerable EnumerateFilesystemEntries() { - foreach (var snap in m_snapShots) + foreach (var sourceEntry in SourceEntries) { - if (DirectoryExists(snap.LocalPath) || snap.LocalPath.EndsWith(System.IO.Path.DirectorySeparatorChar)) - yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(snap.LocalPath), true, true); + if (DirectoryExists(sourceEntry) || sourceEntry.EndsWith(System.IO.Path.DirectorySeparatorChar)) + yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(sourceEntry), true, true); else - yield return new SnapshotSourceFileEntry(this, snap.LocalPath, false, true); + yield return new SnapshotSourceFileEntry(this, sourceEntry, false, true); } } diff --git a/Duplicati/Library/Snapshots/NoSnapshotLinux.cs b/Duplicati/Library/Snapshots/NoSnapshotLinux.cs index 87fff66d1..6aaa1ecd6 100644 --- a/Duplicati/Library/Snapshots/NoSnapshotLinux.cs +++ b/Duplicati/Library/Snapshots/NoSnapshotLinux.cs @@ -69,25 +69,25 @@ namespace Duplicati.Library.Snapshots /// /// The list of folders to create snapshots of /// - private readonly IEnumerable m_folders; + private readonly IEnumerable m_sources; /// /// Initializes a new instance of the class. /// - /// The list of folders to create snapshots of + /// The list of sources to create snapshots of /// A flag indicating if advisory locks should be ignored /// A flag indicating if symlinks should be followed - public NoSnapshotLinux(IEnumerable folders, bool ignoreAdvisoryLocks, bool followSymlinks) + public NoSnapshotLinux(IEnumerable sources, bool ignoreAdvisoryLocks, bool followSymlinks) : base(followSymlinks) { - m_folders = folders; + m_sources = sources; m_ignoreAdvisoryLocks = ignoreAdvisoryLocks; } /// /// Gets the source folders /// - public override IEnumerable SourceFolders => m_folders; + public override IEnumerable SourceEntries => m_sources; /// /// Enumerates the root source files and folders @@ -95,7 +95,7 @@ namespace Duplicati.Library.Snapshots /// The source files and folders public override IEnumerable EnumerateFilesystemEntries() { - foreach (var folder in m_folders) + foreach (var folder in m_sources) { if (DirectoryExists(folder) || folder.EndsWith(Path.DirectorySeparatorChar)) yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(folder), true, true); diff --git a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs index b4a3b16e0..029ae3791 100644 --- a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs +++ b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs @@ -37,17 +37,17 @@ namespace Duplicati.Library.Snapshots /// /// The list of folders to create snapshots of /// - private readonly IEnumerable m_folders; + private readonly IEnumerable m_sources; /// /// Initializes a new instance of the class. /// - /// The list of folders to create snapshots of + /// The list of entries to create snapshots of /// A flag indicating if symlinks should be followed - public NoSnapshotWindows(IEnumerable folders, bool followSymlinks) + public NoSnapshotWindows(IEnumerable sources, bool followSymlinks) : base(followSymlinks) { - m_folders = folders; + m_sources = sources; } /// @@ -77,8 +77,8 @@ namespace Duplicati.Library.Snapshots /// /// Gets the source folders /// - public override IEnumerable SourceFolders - => m_folders; + public override IEnumerable SourceEntries + => m_sources; /// /// Enumerates the root source files and folders @@ -86,7 +86,7 @@ namespace Duplicati.Library.Snapshots /// The source files and folders public override IEnumerable EnumerateFilesystemEntries() { - foreach (var folder in m_folders.Select(SystemIOWindows.RemoveExtendedDevicePathPrefix)) + foreach (var folder in m_sources.Select(SystemIOWindows.RemoveExtendedDevicePathPrefix)) { if (DirectoryExists(folder) || folder.EndsWith(System.IO.Path.DirectorySeparatorChar)) yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(folder), true, true); diff --git a/Duplicati/Library/Snapshots/SnapshotBase.cs b/Duplicati/Library/Snapshots/SnapshotBase.cs index 7fcab2761..1d8f5a253 100644 --- a/Duplicati/Library/Snapshots/SnapshotBase.cs +++ b/Duplicati/Library/Snapshots/SnapshotBase.cs @@ -56,9 +56,9 @@ namespace Duplicati.Library.Snapshots protected bool FollowSymlinks { get; } /// - /// Gets the source folders + /// Gets the source entries /// - public abstract IEnumerable SourceFolders { get; } + public abstract IEnumerable SourceEntries { get; } /// /// Enuemrates the root source files and folders diff --git a/Duplicati/Library/Snapshots/UsnJournalService.cs b/Duplicati/Library/Snapshots/UsnJournalService.cs index 09583cef8..8f1298472 100644 --- a/Duplicati/Library/Snapshots/UsnJournalService.cs +++ b/Duplicati/Library/Snapshots/UsnJournalService.cs @@ -97,7 +97,7 @@ namespace Duplicati.Library.Snapshots // get hash identifying current source filter / sources configuration var configHash = Utility.Utility.ByteArrayAsHexString(MD5HashHelper.GetHash(new string[] { emitFilter?.ToString() ?? string.Empty, - string.Join("; ", m_snapshot.SourceFolders), + string.Join("; ", m_snapshot.SourceEntries), fileAttributeFilter.ToString(), skipFilesLargerThan.ToString() })); @@ -106,7 +106,7 @@ namespace Duplicati.Library.Snapshots var journalDataDict = prevJournalData.ToDictionary(data => data.Volume); // iterate over volumes - foreach (var sourcesPerVolume in SortByVolume(m_snapshot.SourceFolders)) + foreach (var sourcesPerVolume in SortByVolume(m_snapshot.SourceEntries)) { Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "UsnInitialize", "Reading USN journal for volume: {0}", sourcesPerVolume.Key); @@ -234,11 +234,11 @@ namespace Duplicati.Library.Snapshots // Make the method async enumerable await Task.CompletedTask; - foreach(var volumeData in m_volumeDataDict.Values.Where(x => x.IsFullScan)) + foreach (var volumeData in m_volumeDataDict.Values.Where(x => x.IsFullScan)) { if (volumeData.Folders != null) { - foreach(var folderPath in volumeData.Folders) + foreach (var folderPath in volumeData.Folders) { if (token.IsCancellationRequested) yield break; @@ -253,7 +253,7 @@ namespace Duplicati.Library.Snapshots if (volumeData.Files != null) { - foreach(var filePath in volumeData.Files) + foreach (var filePath in volumeData.Files) { if (token.IsCancellationRequested) yield break; @@ -278,12 +278,12 @@ namespace Duplicati.Library.Snapshots /// Filtered sources public async IAsyncEnumerable GetModifiedSources(Func> filter, [EnumeratorCancellation] CancellationToken token) { - foreach(var volumeData in m_volumeDataDict.Values.Where(x => !x.IsFullScan)) + foreach (var volumeData in m_volumeDataDict.Values.Where(x => !x.IsFullScan)) { // prepare cache for includes (value = true) and excludes (value = false, will be populated // on-demand) var cache = new Dictionary(); - foreach (var source in m_snapshot.SourceFolders) + foreach (var source in m_snapshot.SourceEntries) { if (token.IsCancellationRequested) yield break; diff --git a/Duplicati/Library/Snapshots/WindowsSnapshot.cs b/Duplicati/Library/Snapshots/WindowsSnapshot.cs index 8d14674ec..08bd5e1ae 100644 --- a/Duplicati/Library/Snapshots/WindowsSnapshot.cs +++ b/Duplicati/Library/Snapshots/WindowsSnapshot.cs @@ -58,7 +58,7 @@ namespace Duplicati.Library.Snapshots /// /// The source folders included in the snapshot /// - private readonly IReadOnlyList _sourceFolders; + private readonly IReadOnlyList _sourceEntries; /// /// Constructs a new backup snapshot, using all the required disks @@ -70,7 +70,7 @@ namespace Duplicati.Library.Snapshots : base(followSymlinks) { // For Windows, ensure we don't store paths with extended device path prefixes (i.e., @"\\?\" or @"\\?\UNC\") - _sourceFolders = sources.Select(SystemIOWindows.RemoveExtendedDevicePathPrefix).ToList(); + _sourceEntries = sources.Select(SystemIOWindows.RemoveExtendedDevicePathPrefix).ToList(); try { _vssBackupComponents = new VssBackupComponents(); @@ -87,7 +87,7 @@ namespace Duplicati.Library.Snapshots } _vssBackupComponents.SetupWriters(null, excludedWriters); - _vssBackupComponents.InitShadowVolumes(_sourceFolders); + _vssBackupComponents.InitShadowVolumes(_sourceEntries); _vssBackupComponents.MapVolumesToSnapShots(); @@ -119,7 +119,7 @@ namespace Duplicati.Library.Snapshots /// /// Gets the source folders /// - public override IEnumerable SourceFolders => _sourceFolders; + public override IEnumerable SourceEntries => _sourceEntries; /// /// Enumerates the root source files and folders @@ -127,7 +127,7 @@ namespace Duplicati.Library.Snapshots /// The source files and folders public override IEnumerable EnumerateFilesystemEntries() { - foreach (var folder in _sourceFolders) + foreach (var folder in _sourceEntries) { if (folder.EndsWith(Path.DirectorySeparatorChar) || DirectoryExists(folder)) yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(folder), true, true); From cf5b4bb24dc5617179df78fea57532ff862fcb95 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 19 Feb 2025 14:00:14 +0100 Subject: [PATCH 15/28] Added support for Wmic based Windows VSS snapshots. Refactored code a bit to be easier to understand. --- .../Operation/Backup/CountFilesHandler.cs | 2 +- .../Backup/FileEnumerationProcess.cs | 3 +- .../Library/Main/Operation/BackupHandler.cs | 1 + Duplicati/Library/Main/Options.cs | 19 ++ Duplicati/Library/Main/Strings.cs | 2 + .../Library/Modules/Builtin/HyperVOptions.cs | 43 ++-- .../Library/Modules/Builtin/MSSQLOptions.cs | 45 ++-- Duplicati/Library/Snapshots/Program.cs | 2 - .../Library/Snapshots/SnapshotProvider.cs | 30 +++ .../Library/Snapshots/{ => USN}/USNJournal.cs | 2 +- .../Snapshots/{ => USN}/UsnJournalService.cs | 2 +- .../Library/Snapshots/{ => USN}/Win32USN.cs | 42 ++-- .../Snapshots/Windows/AlphaVssBackup.cs | 200 ++++++++++++++++ .../Snapshots/{ => Windows}/HyperVUtility.cs | 11 +- .../Snapshots/Windows/ISnapshotProvider.cs | 118 ++++++++++ .../Snapshots/{ => Windows}/MSSQLUtility.cs | 49 ++-- .../Windows/SnapshotManager.cs} | 180 ++++++--------- .../Windows/WmicShadowCopyManager.cs | 215 ++++++++++++++++++ .../Snapshots/Windows/WmicVssBackup.cs | 139 +++++++++++ .../Library/Snapshots/WindowsSnapshot.cs | 21 +- .../WebserverCore/Endpoints/V1/HyperV.cs | 5 +- Duplicati/WebserverCore/Endpoints/V1/MSSql.cs | 3 +- 22 files changed, 914 insertions(+), 220 deletions(-) create mode 100644 Duplicati/Library/Snapshots/SnapshotProvider.cs rename Duplicati/Library/Snapshots/{ => USN}/USNJournal.cs (97%) rename Duplicati/Library/Snapshots/{ => USN}/UsnJournalService.cs (97%) rename Duplicati/Library/Snapshots/{ => USN}/Win32USN.cs (97%) create mode 100644 Duplicati/Library/Snapshots/Windows/AlphaVssBackup.cs rename Duplicati/Library/Snapshots/{ => Windows}/HyperVUtility.cs (95%) create mode 100644 Duplicati/Library/Snapshots/Windows/ISnapshotProvider.cs rename Duplicati/Library/Snapshots/{ => Windows}/MSSQLUtility.cs (94%) rename Duplicati/Library/{Common/IO/VssBackupComponents.cs => Snapshots/Windows/SnapshotManager.cs} (61%) create mode 100644 Duplicati/Library/Snapshots/Windows/WmicShadowCopyManager.cs create mode 100644 Duplicati/Library/Snapshots/Windows/WmicVssBackup.cs diff --git a/Duplicati/Library/Main/Operation/Backup/CountFilesHandler.cs b/Duplicati/Library/Main/Operation/Backup/CountFilesHandler.cs index ffefa0abc..1d16f29d9 100644 --- a/Duplicati/Library/Main/Operation/Backup/CountFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/Backup/CountFilesHandler.cs @@ -24,7 +24,7 @@ using System.Threading.Tasks; using Duplicati.Library.Utility; using CoCoL; using Duplicati.Library.Interface; -using Duplicati.Library.Snapshots; +using Duplicati.Library.Snapshots.USN; namespace Duplicati.Library.Main.Operation.Backup { diff --git a/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs b/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs index a7db2b2ce..a181907ff 100644 --- a/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs @@ -31,9 +31,8 @@ using System.Threading; using Duplicati.Library.Main.Operation.Common; using Duplicati.Library.Interface; using System.Runtime.CompilerServices; -using Duplicati.Library.Snapshots; using Duplicati.Library.SourceProvider; -using Duplicati.Library.Common.IO; +using Duplicati.Library.Snapshots.USN; namespace Duplicati.Library.Main.Operation.Backup { diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 84fc89b69..059bbc8df 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -36,6 +36,7 @@ using Duplicati.Library.Main.Operation.Common; using System.Data; using Duplicati.Library.DynamicLoader; using Duplicati.Library.SourceProvider; +using Duplicati.Library.Snapshots.USN; namespace Duplicati.Library.Main.Operation { diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index c0d681aaa..4d32c8ae3 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -345,6 +345,7 @@ namespace Duplicati.Library.Main new CommandLineArgument("upload-unchanged-backups", CommandLineArgument.ArgumentType.Boolean, Strings.Options.UploadUnchangedBackupsShort, Strings.Options.UploadUnchangedBackupsLong, "false"), new CommandLineArgument("snapshot-policy", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.SnapshotpolicyShort, Strings.Options.SnapshotpolicyLong, "off", null, Enum.GetNames(typeof(OptimizationStrategy))), + new CommandLineArgument("snapshot-provider", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.SnapshotproviderShort, Strings.Options.SnapshotproviderLong, OperatingSystem.IsWindows() ? Snapshots.SnapshotProvider.AlphaVSS.ToString() : Snapshots.SnapshotProvider.LVM.ToString(), null, (OperatingSystem.IsWindows() ? [Snapshots.SnapshotProvider.AlphaVSS, Snapshots.SnapshotProvider.Wmic] : new [] { Snapshots.SnapshotProvider.LVM }).Select(x => x.ToString()).ToArray()), new CommandLineArgument("vss-exclude-writers", CommandLineArgument.ArgumentType.String, Strings.Options.VssexcludewritersShort, Strings.Options.VssexcludewritersLong, "{e8132975-6f93-4464-a53e-1050253ae220}"), new CommandLineArgument("vss-use-mapping", CommandLineArgument.ArgumentType.Boolean, Strings.Options.VssusemappingShort, Strings.Options.VssusemappingLong, "false"), new CommandLineArgument("usn-policy", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.UsnpolicyShort, Strings.Options.UsnpolicyLong, "off", null, Enum.GetNames(typeof(OptimizationStrategy))), @@ -1025,6 +1026,24 @@ namespace Duplicati.Library.Main } } + /// + /// Gets the snapshot strategy to use + /// + public Snapshots.SnapshotProvider SnapShotProvider + { + get + { + if (!m_options.TryGetValue("snapshot-provider", out var provider)) + provider = ""; + + Snapshots.SnapshotProvider r; + if (!Enum.TryParse(provider, true, out r)) + r = OperatingSystem.IsWindows() ? Snapshots.SnapshotProvider.AlphaVSS : Snapshots.SnapshotProvider.LVM; + + return r; + } + } + /// /// Gets a flag indicating if advisory locking should be ignored /// diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs index 6ee298520..862d064c2 100644 --- a/Duplicati/Library/Main/Strings.cs +++ b/Duplicati/Library/Main/Strings.cs @@ -116,6 +116,8 @@ namespace Duplicati.Library.Main.Strings public static string EnablemoduleShort { get { return LC.L(@"Enable one or more modules"); } } public static string SnapshotpolicyLong { get { return LC.L(@"This setting controls the usage of snapshots, which allows Duplicati to backup files that are locked by other programs. If this is set to ""off"", Duplicati will not attempt to create a disk snapshot. Setting this to ""auto"" makes Duplicati attempt to create a snapshot, and fail silently if that was not allowed or supported (note that the OS may still log system warnings). A setting of ""on"" will also make Duplicati attempt to create a snapshot, but will produce a warning message in the log if it fails. Setting it to ""required"" will make Duplicati abort the backup if the snapshot creation fails. On windows this uses the Volume Shadow Copy Services (VSS) and requires administrative privileges. On Linux this uses Logical Volume Management (LVM) and requires root privileges."); } } public static string SnapshotpolicyShort { get { return LC.L(@"Control the use of disk snapshots"); } } + public static string SnapshotproviderLong { get { return LC.L(@"The snapshot provider implementation for Windows. The AlphaVSS is the most feature complete, but is not supported on Arm64. The WMIC based snapshot has less features, but is more portable. On Linux, only LVM is supported"); } } + public static string SnapshotproviderShort { get { return LC.L(@"The snapshot provider implementation to use"); } } public static string AsynchronousuploadfolderLong { get { return LC.L(@"The pre-generated volumes will be placed into the temporary folder by default. This option can set a different folder for placing the temporary volumes. Despite the name, this also works for synchronous runs."); } } public static string AsynchronousuploadfolderShort { get { return LC.L(@"The path where ready volumes are placed until uploaded"); } } public static string AsynchronousuploadlimitLong { get { return LC.L(@"When performing asynchronous uploads, Duplicati will create volumes that can be uploaded. To prevent Duplicati from generating too many volumes, this option limits the number of pending uploads. Set to zero to disable the limit. The volume(s) that are being created are not counted in this limit. Use the option --concurrency-compressors=1 to limit the number of volumes being created."); } } diff --git a/Duplicati/Library/Modules/Builtin/HyperVOptions.cs b/Duplicati/Library/Modules/Builtin/HyperVOptions.cs index 2df703c2e..67bda3d9e 100644 --- a/Duplicati/Library/Modules/Builtin/HyperVOptions.cs +++ b/Duplicati/Library/Modules/Builtin/HyperVOptions.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -24,8 +24,8 @@ using System.Collections.Generic; using System.Linq; using System.Runtime.Versioning; using System.Text.RegularExpressions; -using Duplicati.Library.Common; using Duplicati.Library.Snapshots; +using Duplicati.Library.Snapshots.Windows; namespace Duplicati.Library.Modules.Builtin { @@ -149,7 +149,8 @@ namespace Duplicati.Library.Modules.Builtin Logging.Log.WriteWarningMessage(LOGTAG, "HyperVOnServerOnly", null, "This is client version of Windows. Hyper-V VSS writer is present only on Server version. Backup will continue, but will be crash consistent only in opposite to application consistent in Server version"); Logging.Log.WriteInformationMessage(LOGTAG, "StartingHyperVQuery", "Starting to gather Hyper-V information"); - hypervUtility.QueryHyperVGuestsInfo(true); + var provider = Utility.Utility.ParseEnumOption(changedOptions.AsReadOnly(), "snapshot-provider", SnapshotProvider.AlphaVSS); + hypervUtility.QueryHyperVGuestsInfo(provider, true); Logging.Log.WriteInformationMessage(LOGTAG, "HyperVMachineCount", "Found {0} virtual machines on Hyper-V", hypervUtility.Guests.Count); foreach (var guest in hypervUtility.Guests) diff --git a/Duplicati/Library/Modules/Builtin/MSSQLOptions.cs b/Duplicati/Library/Modules/Builtin/MSSQLOptions.cs index d41f84009..058b119c5 100644 --- a/Duplicati/Library/Modules/Builtin/MSSQLOptions.cs +++ b/Duplicati/Library/Modules/Builtin/MSSQLOptions.cs @@ -1,31 +1,31 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; using System.Linq; using System.Runtime.Versioning; using System.Text.RegularExpressions; -using Duplicati.Library.Common; using Duplicati.Library.Snapshots; +using Duplicati.Library.Snapshots.Windows; namespace Duplicati.Library.Modules.Builtin { @@ -144,7 +144,8 @@ namespace Duplicati.Library.Modules.Builtin } Logging.Log.WriteInformationMessage(LOGTAG, "StartingMsSqlQuery", "Starting to gather Microsoft SQL Server information", Logging.LogMessageType.Information); - mssqlUtility.QueryDBsInfo(); + var provider = Utility.Utility.ParseEnumOption(changedOptions.AsReadOnly(), "snapshot-provider", SnapshotProvider.AlphaVSS); + mssqlUtility.QueryDBsInfo(provider); Logging.Log.WriteInformationMessage(LOGTAG, "MsSqlDatabaseCount", "Found {0} databases on Microsoft SQL Server", mssqlUtility.DBs.Count); foreach(var db in mssqlUtility.DBs) diff --git a/Duplicati/Library/Snapshots/Program.cs b/Duplicati/Library/Snapshots/Program.cs index 6afd2647d..8e48df343 100644 --- a/Duplicati/Library/Snapshots/Program.cs +++ b/Duplicati/Library/Snapshots/Program.cs @@ -21,9 +21,7 @@ using System; using System.Collections.Generic; -using System.Linq; using Duplicati.Library.AutoUpdater; -using Duplicati.Library.Common; using Duplicati.Library.Utility; namespace Duplicati.Library.Snapshots diff --git a/Duplicati/Library/Snapshots/SnapshotProvider.cs b/Duplicati/Library/Snapshots/SnapshotProvider.cs new file mode 100644 index 000000000..c1e8984fa --- /dev/null +++ b/Duplicati/Library/Snapshots/SnapshotProvider.cs @@ -0,0 +1,30 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +namespace Duplicati.Library.Snapshots +{ + public enum SnapshotProvider + { + AlphaVSS, + Wmic, + LVM + } +} diff --git a/Duplicati/Library/Snapshots/USNJournal.cs b/Duplicati/Library/Snapshots/USN/USNJournal.cs similarity index 97% rename from Duplicati/Library/Snapshots/USNJournal.cs rename to Duplicati/Library/Snapshots/USN/USNJournal.cs index 2608395d5..73346ae59 100644 --- a/Duplicati/Library/Snapshots/USNJournal.cs +++ b/Duplicati/Library/Snapshots/USN/USNJournal.cs @@ -31,7 +31,7 @@ using System.Runtime.Versioning; using Duplicati.Library.Common.IO; using Microsoft.Win32.SafeHandles; -namespace Duplicati.Library.Snapshots +namespace Duplicati.Library.Snapshots.USN { /// /// Class that encapsulates USN journal access to a single volume diff --git a/Duplicati/Library/Snapshots/UsnJournalService.cs b/Duplicati/Library/Snapshots/USN/UsnJournalService.cs similarity index 97% rename from Duplicati/Library/Snapshots/UsnJournalService.cs rename to Duplicati/Library/Snapshots/USN/UsnJournalService.cs index 8f1298472..d44bad907 100644 --- a/Duplicati/Library/Snapshots/UsnJournalService.cs +++ b/Duplicati/Library/Snapshots/USN/UsnJournalService.cs @@ -33,7 +33,7 @@ using Duplicati.Library.Utility; using System.Threading.Tasks; using System.Runtime.CompilerServices; -namespace Duplicati.Library.Snapshots +namespace Duplicati.Library.Snapshots.USN { [SupportedOSPlatform("windows")] public class UsnJournalService diff --git a/Duplicati/Library/Snapshots/Win32USN.cs b/Duplicati/Library/Snapshots/USN/Win32USN.cs similarity index 97% rename from Duplicati/Library/Snapshots/Win32USN.cs rename to Duplicati/Library/Snapshots/USN/Win32USN.cs index ed40e8c3f..3a9ebf7c2 100644 --- a/Duplicati/Library/Snapshots/Win32USN.cs +++ b/Duplicati/Library/Snapshots/USN/Win32USN.cs @@ -1,30 +1,30 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. using System; using System.Runtime.InteropServices; using System.Runtime.Versioning; using Microsoft.Win32.SafeHandles; -namespace Duplicati.Library.Snapshots +namespace Duplicati.Library.Snapshots.USN { //The signatures in this file are from http://pinvoke.net diff --git a/Duplicati/Library/Snapshots/Windows/AlphaVssBackup.cs b/Duplicati/Library/Snapshots/Windows/AlphaVssBackup.cs new file mode 100644 index 000000000..85d65f49b --- /dev/null +++ b/Duplicati/Library/Snapshots/Windows/AlphaVssBackup.cs @@ -0,0 +1,200 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Alphaleonis.Win32.Vss; +using Duplicati.Library.Common.IO; + +namespace Duplicati.Library.Snapshots.Windows; + +/// +/// Implementation of a snapshot, using AlphaVSS +/// +internal class AlphaVssBackup : ISnapshotProvider +{ + /// + /// The implementation that is being wrapped + /// + private IVssBackupComponents _components; + /// + /// Creates a new snapshot implementation + /// + /// The components to use + private AlphaVssBackup(IVssBackupComponents components) + { + _components = components; + } + + /// + public void EnableWriterClasses(Guid[] guids) + => _components.EnableWriterClasses(guids); + + /// + public void DisableWriterClasses(Guid[] guids) + => _components.DisableWriterClasses(guids); + + /// + public void GatherWriterMetadata() + => _components.GatherWriterMetadata(); + /// + public void FreeWriterMetadata() + => _components.FreeWriterMetadata(); + + /// + public void VerifyWriters(Guid[] guids) + { + foreach (var writerGUID in guids) + { + if (!_components.WriterMetadata.Any(o => o.WriterId.Equals(writerGUID))) + { + throw new Exception(string.Format("Writer with GUID {0} was not added to VSS writer set.", writerGUID.ToString())); + } + } + } + + /// + /// Stub for returning the snapshot info + /// + /// The path where the snapshot is located + private sealed record SnapshotInfo(string SnapshotDeviceObject) : ISnapshotInfo; + + /// + public ISnapshotInfo GetSnapshotProperties(Guid shadowId) + => new SnapshotInfo(_components.GetSnapshotProperties(shadowId).SnapshotDeviceObject); + + /// + public void StartSnapshotSet() + => _components.StartSnapshotSet(); + + /// + public void PrepareForBackup() + => _components.PrepareForBackup(); + + /// + public void DoSnapshotSet() + => _components.DoSnapshotSet(); + + /// + public bool IsVolumeSupported(string drive) + => _components.IsVolumeSupported(drive); + + /// + public Guid AddToSnapshotSet(string drive) + => _components.AddToSnapshotSet(drive); + + /// + public void BackupComplete() + => _components.BackupComplete(); + + /// + public void DeleteSnapshot(Guid shadowId, bool forceDelete) + => _components.DeleteSnapshot(shadowId, forceDelete); + + /// + public static ISnapshotProvider Create() + => new AlphaVssBackup(GetVssBackupComponents()); + + /// + /// Creates a new AlphaVssBackupComponents instance and adds the context + /// + /// The configured instance + private static IVssBackupComponents GetVssBackupComponents() + { + //Prepare the backup + IVssBackupComponents vssBackupComponents = CreateVssBackupComponents(); + vssBackupComponents.InitializeForBackup(null); + vssBackupComponents.SetContext(VssSnapshotContext.Backup); + vssBackupComponents.SetBackupState(false, true, VssBackupType.Full, false); + + return vssBackupComponents; + } + + /// + /// Loads the AlphaVSS implementation for the current OS + /// + /// The implementation + private static IVssBackupComponents CreateVssBackupComponents() + { + var vss = VssFactoryProvider.Default.GetVssFactory(); + if (vss == null) + throw new InvalidOperationException(); + + return vss.CreateVssBackupComponents(); + } + + /// + public IEnumerable ParseWriterMetaData(Guid[] writers) + { + // check if writers got enabled + foreach (var writerGUID in writers) + { + var writer = _components.WriterMetadata.First(o => o.WriterId.Equals(writerGUID)); + foreach (var component in writer.Components) + { + yield return new WriterMetaData + { + Guid = writerGUID, + Name = component.ComponentName, + LogicalPath = component.LogicalPath, + Paths = GetPathsFromComponent(component) + }; + } + } + } + + /// + /// Returns the paths from the writer metadata + /// + /// The component to get the paths from + /// The list of paths + private List GetPathsFromComponent(IVssWMComponent component) + { + var paths = new List(); + + foreach (var file in component.Files) + { + if (file.FileSpecification.Contains("*")) + { + if (Directory.Exists(Util.AppendDirSeparator(file.Path))) + paths.Add(Util.AppendDirSeparator(file.Path)); + } + else + { + var fileWithSpec = SystemIO.IO_OS.PathCombine(file.Path, file.FileSpecification); + if (File.Exists(fileWithSpec)) + paths.Add(fileWithSpec); + } + } + return paths; + + } + + /// + public void Dispose() + { + _components.Dispose(); + } +} \ No newline at end of file diff --git a/Duplicati/Library/Snapshots/HyperVUtility.cs b/Duplicati/Library/Snapshots/Windows/HyperVUtility.cs similarity index 95% rename from Duplicati/Library/Snapshots/HyperVUtility.cs rename to Duplicati/Library/Snapshots/Windows/HyperVUtility.cs index 4e7b268a6..9e0f16524 100644 --- a/Duplicati/Library/Snapshots/HyperVUtility.cs +++ b/Duplicati/Library/Snapshots/Windows/HyperVUtility.cs @@ -26,7 +26,7 @@ using System.Management; using System.Runtime.Versioning; using Duplicati.Library.Common.IO; -namespace Duplicati.Library.Snapshots +namespace Duplicati.Library.Snapshots.Windows { public class HyperVGuest : IEquatable { @@ -160,8 +160,9 @@ namespace Duplicati.Library.Snapshots /// Query Hyper-V for all Virtual Machines info /// /// Specify if returned data should contain VM paths + /// The provider to use for VSS /// List of Hyper-V Machines - public void QueryHyperVGuestsInfo(bool bIncludePaths = false) + public void QueryHyperVGuestsInfo(SnapshotProvider provider, bool bIncludePaths = false) { if (!IsHyperVInstalled) return; @@ -178,7 +179,7 @@ namespace Duplicati.Library.Snapshots if (bIncludePaths) { - foreach (var o in GetAllVMsPathsVSS()) + foreach (var o in GetAllVMsPathsVSS(provider)) { foreach (var mObject in moCollection) { @@ -218,9 +219,9 @@ namespace Duplicati.Library.Snapshots /// For all Hyper-V guests it enumerate all associated paths using VSS data /// /// A collection of VMs and paths - private static IEnumerable GetAllVMsPathsVSS() + private static IEnumerable GetAllVMsPathsVSS(SnapshotProvider provider) { - using (var vssBackupComponents = new VssBackupComponents()) + using (var vssBackupComponents = new SnapshotManager(provider)) { var writerGUIDS = new[] { HyperVWriterGuid }; diff --git a/Duplicati/Library/Snapshots/Windows/ISnapshotProvider.cs b/Duplicati/Library/Snapshots/Windows/ISnapshotProvider.cs new file mode 100644 index 000000000..db6a0965a --- /dev/null +++ b/Duplicati/Library/Snapshots/Windows/ISnapshotProvider.cs @@ -0,0 +1,118 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +using System; +using System.Collections.Generic; + +namespace Duplicati.Library.Snapshots.Windows; + +/// +/// Interface for the snapshot information +/// +internal interface ISnapshotInfo +{ + /// + /// The path where the snapshot is located + /// + string SnapshotDeviceObject { get; } +} + +/// +/// Interface for a snapshot provider +/// +internal interface ISnapshotProvider : IDisposable +{ + /// + /// Explicitly enables the selected writer GUIDs + /// + /// The GUIDs to enable + void EnableWriterClasses(Guid[] guids); + /// + /// Explicitly disable the selected writer GUIDs + /// + /// The GUIDs to disable + void DisableWriterClasses(Guid[] guids); + /// + /// Gather writer metadata prior to the snapshot + /// + void GatherWriterMetadata(); + /// + /// Release any allocated metadata + /// + void FreeWriterMetadata(); + /// + /// Verify that the writes are enabled on the snapshot + /// + /// The writer GUIDs to verify as enabled + void VerifyWriters(Guid[] guids); + + /// + /// Gets information of a snapshot + /// + /// The shadow copy id + /// The snapshot information + ISnapshotInfo GetSnapshotProperties(Guid shadowId); + + /// + /// Creates a snapshot + /// + void StartSnapshotSet(); + /// + /// Starts the snapshot as a backup snapshot + /// + void PrepareForBackup(); + /// + /// Finalizes the snapshot creation + /// + void DoSnapshotSet(); + /// + /// Checks if the volume is supported by the snapshot + /// + /// The drive to check + /// true if the volume is supported; false otherwise + bool IsVolumeSupported(string drive); + /// + /// Adds a drive to the snapshot set + /// + /// The drive to add + /// The shadow id + Guid AddToSnapshotSet(string drive); + + /// + /// Completes the snapshot after the backup has completed + /// + void BackupComplete(); + /// + /// Releases all resources held by the snapshot + /// + /// The snapshot shadow id + /// Flag to choose force deletion + void DeleteSnapshot(Guid shadowId, bool forceDelete); + + /// + /// Parses write metadata + /// + /// The writers to return information from + /// The writer metadata + IEnumerable ParseWriterMetaData(Guid[] writers); +} \ No newline at end of file diff --git a/Duplicati/Library/Snapshots/MSSQLUtility.cs b/Duplicati/Library/Snapshots/Windows/MSSQLUtility.cs similarity index 94% rename from Duplicati/Library/Snapshots/MSSQLUtility.cs rename to Duplicati/Library/Snapshots/Windows/MSSQLUtility.cs index 896c24686..7cbf8444e 100644 --- a/Duplicati/Library/Snapshots/MSSQLUtility.cs +++ b/Duplicati/Library/Snapshots/Windows/MSSQLUtility.cs @@ -1,31 +1,30 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. -using Duplicati.Library.Common; -using Duplicati.Library.Common.IO; +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + using System; using System.Collections.Generic; using System.Linq; using System.Runtime.Versioning; -namespace Duplicati.Library.Snapshots +namespace Duplicati.Library.Snapshots.Windows { public class MSSQLDB : IEquatable { @@ -152,14 +151,14 @@ namespace Duplicati.Library.Snapshots /// For all MS SQL databases it enumerate all associated paths using VSS data /// /// A collection of DBs and paths - public void QueryDBsInfo() + public void QueryDBsInfo(SnapshotProvider provider) { if (!IsMSSQLInstalled) return; m_DBs.Clear(); - using (var vssBackupComponents = new VssBackupComponents()) + using (var vssBackupComponents = new SnapshotManager(provider)) { var writerGUIDS = new [] { MSSQLWriterGuid }; try diff --git a/Duplicati/Library/Common/IO/VssBackupComponents.cs b/Duplicati/Library/Snapshots/Windows/SnapshotManager.cs similarity index 61% rename from Duplicati/Library/Common/IO/VssBackupComponents.cs rename to Duplicati/Library/Snapshots/Windows/SnapshotManager.cs index 8d8619a2a..78d21fa91 100644 --- a/Duplicati/Library/Common/IO/VssBackupComponents.cs +++ b/Duplicati/Library/Snapshots/Windows/SnapshotManager.cs @@ -21,13 +21,16 @@ using System; using System.Collections.Generic; -using System.IO; using System.Linq; using System.Runtime.Versioning; using Alphaleonis.Win32.Vss; +using Duplicati.Library.Common.IO; -namespace Duplicati.Library.Common.IO +namespace Duplicati.Library.Snapshots.Windows { + /// + /// Metadata from a writer instance + /// public class WriterMetaData { public string Name { get; set; } @@ -36,16 +39,21 @@ namespace Duplicati.Library.Common.IO public List Paths { get; set; } } + /// + /// The manager for the Windows snapshot + /// [SupportedOSPlatform("windows")] - public class VssBackupComponents : IDisposable + public class SnapshotManager : IDisposable { /// /// The tag used for logging messages /// - public static readonly string LOGTAG = Logging.Log.LogTagFromType(); + private static readonly string LOGTAG = Logging.Log.LogTagFromType(); - - private IVssBackupComponents _vssBackupComponents; + /// + /// The snapshot implementation + /// + private ISnapshotProvider _snapshotProvider; /// /// The list of snapshot ids for each volume, key is the path root, e.g. C:\. @@ -69,26 +77,39 @@ namespace Duplicati.Library.Common.IO /// private List _mappedDrives; - public VssBackupComponents() + /// + /// Creates a new snapshot manager + /// + /// The provider to use + public SnapshotManager(SnapshotProvider provider) { - _vssBackupComponents = VssBackupComponentsHelper.GetVssBackupComponents(); + _snapshotProvider = provider switch { + SnapshotProvider.AlphaVSS => AlphaVssBackup.Create(), + SnapshotProvider.Wmic => new WmicVssBackup(), + _ => throw new ArgumentException($"Invalid provider: {provider}") + }; } + /// + /// Setup writers on the snapshot + /// + /// The explicitly included writers + /// The explicitly excluded writers public void SetupWriters(Guid[] includedWriters, Guid[] excludedWriters) { if (includedWriters != null && includedWriters.Length > 0) - _vssBackupComponents.EnableWriterClasses(includedWriters); + _snapshotProvider.EnableWriterClasses(includedWriters); if (excludedWriters != null && excludedWriters.Length > 0) - _vssBackupComponents.DisableWriterClasses(excludedWriters); + _snapshotProvider.DisableWriterClasses(excludedWriters); try { - _vssBackupComponents.GatherWriterMetadata(); + _snapshotProvider.GatherWriterMetadata(); } finally { - _vssBackupComponents.FreeWriterMetadata(); + _snapshotProvider.FreeWriterMetadata(); } if (includedWriters == null) @@ -97,15 +118,12 @@ namespace Duplicati.Library.Common.IO } // check if writers got enabled - foreach (var writerGUID in includedWriters) - { - if (!_vssBackupComponents.WriterMetadata.Any(o => o.WriterId.Equals(writerGUID))) - { - throw new Exception(string.Format("Writer with GUID {0} was not added to VSS writer set.", writerGUID.ToString())); - } - } + _snapshotProvider.VerifyWriters(includedWriters); } + /// + /// Maps drives from snapshot paths + /// public void MapDrives() { _mappedDrives = new List(); @@ -124,6 +142,9 @@ namespace Duplicati.Library.Common.IO } } + /// + /// Makes a map of the snapshots for easy lookup + /// public void MapVolumesToSnapShots() { _volumeMap = new Dictionary(StringComparer.OrdinalIgnoreCase); @@ -133,43 +154,17 @@ namespace Duplicati.Library.Common.IO // Attempting to get attributes on this path will fail, so we need to append a double backslash // Strangely, the double backslash is not needed when acessing a subfolder, here a single backslash is enough, // but double backslash works for both cases due to path normalization. - _volumeMap.Add(kvp.Key, Util.AppendDirSeparator(_vssBackupComponents.GetSnapshotProperties(kvp.Value).SnapshotDeviceObject) + "\\"); + _volumeMap.Add(kvp.Key, Util.AppendDirSeparator(_snapshotProvider.GetSnapshotProperties(kvp.Value).SnapshotDeviceObject) + "\\"); } _volumeReverseMap = _volumeMap.ToDictionary(x => x.Value, x => x.Key); } - public Dictionary SnapshotDeviceAndVolumes - { - get - { - return _volumeReverseMap; - } - } - - public IEnumerable SnapshotLocalPaths => _volumeMap.Keys; - - private List GetPathsFromComponent(IVssWMComponent component) - { - var paths = new List(); - - foreach (var file in component.Files) - { - if (file.FileSpecification.Contains("*")) - { - if (Directory.Exists(Util.AppendDirSeparator(file.Path))) - paths.Add(Util.AppendDirSeparator(file.Path)); - } - else - { - var fileWithSpec = SystemIO.IO_OS.PathCombine(file.Path, file.FileSpecification); - if (File.Exists(fileWithSpec)) - paths.Add(fileWithSpec); - } - } - return paths; - - } + /// + /// Returns the map of snapshot path and logical path + /// + public Dictionary SnapshotDeviceAndVolumes + => _volumeReverseMap; /// /// Enumerable to iterate over components of the given writers. @@ -178,38 +173,30 @@ namespace Duplicati.Library.Common.IO /// Anonymous object containing GUID, component name, logical path and paths of the component. /// Writers. public IEnumerable ParseWriterMetaData(Guid[] writers) - { - // check if writers got enabled - foreach (var writerGUID in writers) - { - var writer = _vssBackupComponents.WriterMetadata.First(o => o.WriterId.Equals(writerGUID)); - foreach (var component in writer.Components) - { - yield return new WriterMetaData - { - Guid = writerGUID, - Name = component.ComponentName, - LogicalPath = component.LogicalPath, - Paths = GetPathsFromComponent(component) - }; - } - } - } - + => _snapshotProvider.ParseWriterMetaData(writers); + /// + /// Prepares the snapshot and notifies all writers + /// + /// The soruces to include public void InitShadowVolumes(IEnumerable sources) { - _vssBackupComponents.StartSnapshotSet(); + _snapshotProvider.StartSnapshotSet(); - CheckSupportedVolumes(sources); + CheckAndAddSupportedVolumes(sources); //Make all writers aware that we are going to do the backup - _vssBackupComponents.PrepareForBackup(); + _snapshotProvider.PrepareForBackup(); //Create the shadow volumes - _vssBackupComponents.DoSnapshotSet(); + _snapshotProvider.DoSnapshotSet(); } + /// + /// Gets the snapshot path from the local path + /// + /// The local path + /// The snapshot path public string GetVolumeFromCache(string path) { if (!_volumeMap.TryGetValue(path, out var volumePath)) @@ -218,7 +205,11 @@ namespace Duplicati.Library.Common.IO return volumePath; } - public void CheckSupportedVolumes(IEnumerable sources) + /// + /// Checks if the volumes can be added and creates snapshots as needed + /// + /// The sources to create snapshots for + private void CheckAndAddSupportedVolumes(IEnumerable sources) { //Figure out which volumes are in the set _volumes = new Dictionary(StringComparer.OrdinalIgnoreCase); @@ -227,16 +218,17 @@ namespace Duplicati.Library.Common.IO var drive = SystemIO.IO_OS.GetPathRoot(s); if (!_volumes.ContainsKey(drive)) { - if (!_vssBackupComponents.IsVolumeSupported(drive)) + if (!_snapshotProvider.IsVolumeSupported(drive)) { throw new VssVolumeNotSupportedException(drive); } - _volumes.Add(drive, _vssBackupComponents.AddToSnapshotSet(drive)); + _volumes.Add(drive, _snapshotProvider.AddToSnapshotSet(drive)); } } } + /// public void Dispose() { try @@ -258,7 +250,7 @@ namespace Duplicati.Library.Common.IO try { - _vssBackupComponents?.BackupComplete(); + _snapshotProvider?.BackupComplete(); } catch (Exception ex) { @@ -267,13 +259,13 @@ namespace Duplicati.Library.Common.IO try { - if (_vssBackupComponents != null) + if (_snapshotProvider != null) { foreach (var g in _volumes.Values) { try { - _vssBackupComponents.DeleteSnapshot(g, false); + _snapshotProvider.DeleteSnapshot(g, false); } catch (Exception ex) { @@ -287,35 +279,11 @@ namespace Duplicati.Library.Common.IO Logging.Log.WriteVerboseMessage(LOGTAG, "VSSSnapShotDeleteCleanError", ex, "Failed during VSS esnapshot closing"); } - if (_vssBackupComponents != null) + if (_snapshotProvider != null) { - _vssBackupComponents.Dispose(); - _vssBackupComponents = null; + _snapshotProvider.Dispose(); + _snapshotProvider = null; } } } - - - public static class VssBackupComponentsHelper - { - public static IVssBackupComponents GetVssBackupComponents() - { - //Prepare the backup - IVssBackupComponents vssBackupComponents = CreateVssBackupComponents(); - vssBackupComponents.InitializeForBackup(null); - vssBackupComponents.SetContext(VssSnapshotContext.Backup); - vssBackupComponents.SetBackupState(false, true, VssBackupType.Full, false); - - return vssBackupComponents; - } - - public static IVssBackupComponents CreateVssBackupComponents() - { - var vss = VssFactoryProvider.Default.GetVssFactory(); - if (vss == null) - throw new InvalidOperationException(); - - return vss.CreateVssBackupComponents(); - } - } } diff --git a/Duplicati/Library/Snapshots/Windows/WmicShadowCopyManager.cs b/Duplicati/Library/Snapshots/Windows/WmicShadowCopyManager.cs new file mode 100644 index 000000000..45a1ddd6b --- /dev/null +++ b/Duplicati/Library/Snapshots/Windows/WmicShadowCopyManager.cs @@ -0,0 +1,215 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Runtime.Versioning; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +#nullable enable + +namespace Duplicati.Library.Snapshots.Windows; + +/// +/// A shadow copy manager using the wmic commandline tool +/// +[SupportedOSPlatform("windows")] +internal class WmicShadowCopyManager : IDisposable +{ + /// + /// The tag used for logging messages + /// + public static readonly string LOGTAG = Logging.Log.LogTagFromType(); + + /// + /// A single shadow copy + /// + /// The shadow ID as a string + /// The shadow ID as a GUID + /// The drive that the snapshot is for + /// The path that contains the snapshot + public class WmicShadowCopy(string shadowId, Guid parsedId, string originalDrive, string mappedPath) : IDisposable + { + /// + /// Gets the shadow ID + /// + public string ShadowID { get; } = shadowId; + /// + /// Gets the shadow ID + /// + public Guid ParsedId { get; } = parsedId; + /// + /// Gets the drive that was originally mapped + /// + public string OriginalDrive { get; } = originalDrive; + /// + /// Gets the path where the snapshot is found + /// + public string MappedPath { get; } = mappedPath; + + /// + /// Flag keeping track of the snapshot deletion state + /// + private bool _snapshotDeleted; + + /// + public void Dispose() + { + DeleteShadowCopy(); + } + + /// + /// Deletes the shadow copy + /// + private void DeleteShadowCopy() + { + if (_snapshotDeleted) + return; + if (!string.IsNullOrEmpty(ShadowID)) + { + _snapshotDeleted = true; + Logging.Log.WriteVerboseMessage(LOGTAG, "DeleteShadowCopy", $"Deleting Shadow Copy: {ShadowID}"); + DeleteShadow(ShadowID); + } + } + } + + /// + /// The list of the currently registered shadow copies + /// + private List _shadowCopies = new List(); + + /// + /// Gets the list of the currently registered shadow copies + /// + public IEnumerable ShadowCopies => _shadowCopies; + + /// + /// Creates a new snapshot for the given drive + /// + /// The drive to create the snapshot for + /// The created snapshot + public WmicShadowCopy Add(string drive) + { + var shadowId = CreateShadowCopy(drive); + if (string.IsNullOrEmpty(shadowId)) + throw new InvalidOperationException("Failed to create shadow copy"); + + var shadowPath = GetShadowPath(shadowId); + if (string.IsNullOrEmpty(shadowPath)) + { + DeleteShadow(shadowId); + throw new InvalidOperationException("Failed to get shadow copy path"); + } + + var snapshot = new WmicShadowCopy(shadowId, Guid.Parse(shadowId), drive, shadowPath); + _shadowCopies.Add(snapshot); + + return snapshot; + } + + /// + public void Dispose() + { + foreach (var shadow in ShadowCopies) + { + shadow.Dispose(); + } + _shadowCopies.Clear(); + } + + /// + /// Creates a shadow copy + /// + /// The drive to create the snapshot for + /// The shadow id + private static string? CreateShadowCopy(string drive) + { + string output = ExecuteCommand("wmic", $"shadowcopy call create Volume='{drive}'", 10000); + + // Extract ShadowID using regex + Match match = Regex.Match(output, @"ShadowID\s*=\s*""({[0-9A-F\-]+})"""); + if (match.Success) + { + return match.Groups[1].Value; + } + + Logging.Log.WriteErrorMessage(LOGTAG, "ShadowCopyFailed", null, "Failed to create shadow copy for {0}: {1}", drive, output); + return null; + } + + /// + /// Gets the path where the shadow copy is mounted + /// + /// The shadow copy id + /// The path where the copy is mounted + private static string? GetShadowPath(string shadowId) + { + string output = ExecuteCommand("wmic", "shadowcopy get ID, DeviceObject", 5000); + + // Extract DeviceObject corresponding to the ShadowID + string pattern = $@"(\\\\\?\\GLOBALROOT\\Device\\HarddiskVolumeShadowCopy\d+)\s+{shadowId}"; + Match match = Regex.Match(output, pattern); + + if (match.Success) + { + return match.Groups[1].Value; + } + + Logging.Log.WriteErrorMessage(LOGTAG, "ShadowCopyFailed", null, "Failed to get shadow copy path for {0}: {1}", shadowId, output); + return null; + } + + /// + /// Delete a shadow copy + /// + /// The shadow id + private static void DeleteShadow(string shadowId) + => ExecuteCommand("wmic", $"shadowcopy where ID=\"{shadowId}\" delete", 5000); + + /// + /// Executes a command + /// + /// The binary to execute + /// The arguments to use + /// The timeout + /// The output of the command + private static string ExecuteCommand(string fileName, string arguments, int timeoutMs) + { + try + { + using (var process = new Process()) + { + process.StartInfo.FileName = fileName; + process.StartInfo.Arguments = arguments; + process.StartInfo.RedirectStandardOutput = true; + process.StartInfo.RedirectStandardError = true; + process.StartInfo.UseShellExecute = false; + process.StartInfo.CreateNoWindow = true; + + process.Start(); + + var outputTask = process.StandardOutput.ReadToEndAsync(); + var errorTask = process.StandardError.ReadToEndAsync(); + + if (!process.WaitForExit(timeoutMs)) + { + process.Kill(); + throw new TimeoutException($"Command '{fileName} {arguments}' timed out after {timeoutMs}ms."); + } + + var output = outputTask.Result; + var error = errorTask.Result; + + if (!string.IsNullOrWhiteSpace(error)) + Logging.Log.WriteWarningMessage(LOGTAG, "ShadowCopyFailed", null, "Failed to execute command: {0} {1}: {2}", fileName, arguments, error); + + return output; + } + } + catch (Exception ex) + { + Logging.Log.WriteErrorMessage(LOGTAG, "ShadowCopyFailed", ex, "Failed to execute command: {0} {1}", fileName, arguments); + return string.Empty; + } + } +} \ No newline at end of file diff --git a/Duplicati/Library/Snapshots/Windows/WmicVssBackup.cs b/Duplicati/Library/Snapshots/Windows/WmicVssBackup.cs new file mode 100644 index 000000000..37d901dd8 --- /dev/null +++ b/Duplicati/Library/Snapshots/Windows/WmicVssBackup.cs @@ -0,0 +1,139 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Management; +using System.Runtime.Versioning; + +namespace Duplicati.Library.Snapshots.Windows; + +/// +/// Snapshot provider based on WMIC +/// +[SupportedOSPlatform("windows")] +internal class WmicVssBackup : ISnapshotProvider +{ + /// + /// The manager that contains the actual interface to WMIC + /// + private WmicShadowCopyManager _manager = new WmicShadowCopyManager(); + + /// + public Guid AddToSnapshotSet(string drive) + => _manager.Add(drive).ParsedId; + + /// + public void BackupComplete() + { + } + + /// + public void DeleteSnapshot(Guid shadowId, bool forceDelete) + { + foreach (var x in _manager.ShadowCopies.Where(x => x.ParsedId == shadowId)) + x.Dispose(); + } + + /// + public void DisableWriterClasses(Guid[] guids) + { + } + + /// + public void Dispose() + { + _manager.Dispose(); + } + + /// + public void DoSnapshotSet() + { + } + + /// + public void EnableWriterClasses(Guid[] guids) + { + } + + /// + public void FreeWriterMetadata() + { + } + + /// + public void GatherWriterMetadata() + { + } + + /// + /// Stub for returning the snapshot info + /// + /// The path where the snapshot is located + private sealed record SnapshotInfo (string SnapshotDeviceObject) : ISnapshotInfo; + + /// + public ISnapshotInfo GetSnapshotProperties(Guid shadowId) + => new SnapshotInfo(_manager.ShadowCopies.First(x => x.ParsedId == shadowId).MappedPath); + + /// + public bool IsVolumeSupported(string drive) + { + if (string.IsNullOrWhiteSpace(drive)) + return false; + + try + { + var query = $"SELECT * FROM Win32_ShadowCopy WHERE VolumeName LIKE '%{drive[0]}%'"; + var searcher = new ManagementObjectSearcher(query); + var results = searcher.Get(); + + return results.Count > 0; + } + catch (Exception ex) + { + Console.WriteLine($"Error checking VSS support: {ex.Message}"); + return false; + } + } + + /// + public IEnumerable ParseWriterMetaData(Guid[] writers) + => Enumerable.Empty(); + + /// + public void PrepareForBackup() + { + } + + /// + public void StartSnapshotSet() + { + } + + /// + public void VerifyWriters(Guid[] guids) + { + } +} \ No newline at end of file diff --git a/Duplicati/Library/Snapshots/WindowsSnapshot.cs b/Duplicati/Library/Snapshots/WindowsSnapshot.cs index 08bd5e1ae..1c1e70416 100644 --- a/Duplicati/Library/Snapshots/WindowsSnapshot.cs +++ b/Duplicati/Library/Snapshots/WindowsSnapshot.cs @@ -19,7 +19,6 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. - using System; using System.Collections.Generic; using System.IO; @@ -27,6 +26,7 @@ using System.Linq; using System.Runtime.Versioning; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; +using Duplicati.Library.Snapshots.Windows; namespace Duplicati.Library.Snapshots { @@ -53,7 +53,7 @@ namespace Duplicati.Library.Snapshots /// /// The main reference to the backup controller /// - private readonly VssBackupComponents _vssBackupComponents; + private readonly SnapshotManager _snapshotManager; /// /// The source folders included in the snapshot @@ -73,7 +73,8 @@ namespace Duplicati.Library.Snapshots _sourceEntries = sources.Select(SystemIOWindows.RemoveExtendedDevicePathPrefix).ToList(); try { - _vssBackupComponents = new VssBackupComponents(); + var provider = Utility.Utility.ParseEnumOption(options.AsReadOnly(), "snapshot-provider", SnapshotProvider.AlphaVSS); + _snapshotManager = new SnapshotManager(provider); // Default to exclude the System State writer var excludedWriters = new Guid[] { new Guid("{e8132975-6f93-4464-a53e-1050253ae220}") }; @@ -85,16 +86,16 @@ namespace Duplicati.Library.Snapshots .Select(x => new Guid(x)) .ToArray(); } - _vssBackupComponents.SetupWriters(null, excludedWriters); + _snapshotManager.SetupWriters(null, excludedWriters); - _vssBackupComponents.InitShadowVolumes(_sourceEntries); + _snapshotManager.InitShadowVolumes(_sourceEntries); - _vssBackupComponents.MapVolumesToSnapShots(); + _snapshotManager.MapVolumesToSnapShots(); //If we should map the drives, we do that now and update the volumeMap if (Utility.Utility.ParseBoolOption(options.AsReadOnly(), "vss-use-mapping")) { - _vssBackupComponents.MapDrives(); + _snapshotManager.MapDrives(); } } catch (Exception ex1) @@ -279,7 +280,7 @@ namespace Duplicati.Library.Snapshots if (!Path.IsPathRooted(snapshotPath)) throw new InvalidOperationException(); - foreach (var kvp in _vssBackupComponents.SnapshotDeviceAndVolumes) + foreach (var kvp in _snapshotManager.SnapshotDeviceAndVolumes) { if (snapshotPath.StartsWith(kvp.Key, Utility.Utility.ClientFilenameStringComparison)) return IO_WIN.PathCombine(kvp.Value, snapshotPath.Substring(kvp.Key.Length)); @@ -298,7 +299,7 @@ namespace Duplicati.Library.Snapshots throw new InvalidOperationException(); var root = IO_WIN.GetPathRoot(localPath); - var volumePath = _vssBackupComponents.GetVolumeFromCache(root); + var volumePath = _snapshotManager.GetVolumeFromCache(root); // Note that using a simple Path.Combine() for the following code // can result in invalid snapshot paths; e.g., if localPath is @@ -333,7 +334,7 @@ namespace Duplicati.Library.Snapshots { if (disposing) { - _vssBackupComponents?.Dispose(); + _snapshotManager?.Dispose(); } base.Dispose(disposing); diff --git a/Duplicati/WebserverCore/Endpoints/V1/HyperV.cs b/Duplicati/WebserverCore/Endpoints/V1/HyperV.cs index cde404bae..98f8fed9e 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/HyperV.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/HyperV.cs @@ -21,6 +21,7 @@ using System.Runtime.Versioning; using System.Security.Principal; using Duplicati.Library.Snapshots; +using Duplicati.Library.Snapshots.Windows; using Duplicati.WebserverCore.Abstractions; using Duplicati.WebserverCore.Exceptions; using Microsoft.AspNetCore.Mvc; @@ -58,7 +59,7 @@ public class HyperV : IEndpointV1 { if (string.IsNullOrEmpty(key)) { - hypervUtility.QueryHyperVGuestsInfo(); + hypervUtility.QueryHyperVGuestsInfo(SnapshotProvider.AlphaVSS); return hypervUtility.Guests.Select(x => new Dto.TreeNodeDto() { id = x.ID.ToString(), @@ -77,7 +78,7 @@ public class HyperV : IEndpointV1 } else { - hypervUtility.QueryHyperVGuestsInfo(true); + hypervUtility.QueryHyperVGuestsInfo(SnapshotProvider.AlphaVSS, true); var foundVMs = hypervUtility.Guests.FindAll(x => x.ID.Equals(new Guid(key))); if (foundVMs.Count != 1) diff --git a/Duplicati/WebserverCore/Endpoints/V1/MSSql.cs b/Duplicati/WebserverCore/Endpoints/V1/MSSql.cs index 0a2b98679..b4f4e6607 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/MSSql.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/MSSql.cs @@ -21,6 +21,7 @@ using System.Runtime.Versioning; using System.Security.Principal; using Duplicati.Library.Snapshots; +using Duplicati.Library.Snapshots.Windows; using Duplicati.WebserverCore.Abstractions; using Duplicati.WebserverCore.Exceptions; using Microsoft.AspNetCore.Mvc; @@ -56,7 +57,7 @@ public class MSSql : IEndpointV1 try { - mssqlUtility.QueryDBsInfo(); + mssqlUtility.QueryDBsInfo(SnapshotProvider.AlphaVSS); if (string.IsNullOrEmpty(key)) { From 36ba547a92b8f98797aa98dd4175ea30c81371c6 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 19 Feb 2025 14:20:03 +0100 Subject: [PATCH 16/28] Added the @ prefix to backend paths --- Duplicati/Library/Backend/CIFS/CIFS.cs | 2 +- Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs | 2 +- Duplicati/Library/Backend/S3/S3Backend.cs | 2 +- Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs | 2 +- Duplicati/Library/Interface/IFolderEnabledBackend.cs | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Duplicati/Library/Backend/CIFS/CIFS.cs b/Duplicati/Library/Backend/CIFS/CIFS.cs index eafcdfc45..549721edf 100644 --- a/Duplicati/Library/Backend/CIFS/CIFS.cs +++ b/Duplicati/Library/Backend/CIFS/CIFS.cs @@ -190,7 +190,7 @@ public class CIFSBackend : IStreamingBackend, IFolderEnabledBackend writeBufferSize ); - _sourcePathKey = $"{ProtocolKey}://{_connectionParameters.ServerName}/~{_connectionParameters.AuthUser}/{_connectionParameters.Path}/"; + _sourcePathKey = $"@{ProtocolKey}://{_connectionParameters.ServerName}/~{_connectionParameters.AuthUser}/{_connectionParameters.Path}/"; _sourceProvider = new BackendSourceProvider(this); } diff --git a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs index f495ddf52..a3c5d47be 100644 --- a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs +++ b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs @@ -90,7 +90,7 @@ namespace Duplicati.Library.Backend m_s3Client = new S3AwsClient(accessKeyId, accessKeySecret, null, host, null, true, false, options); - m_sourcePathKey = $"{ProtocolKey}://{host}/{m_bucket}/{m_prefix}/"; + m_sourcePathKey = $"@{ProtocolKey}://{host}/{m_bucket}/{m_prefix}/"; m_sourceProvider = new BackendSourceProvider(this); } diff --git a/Duplicati/Library/Backend/S3/S3Backend.cs b/Duplicati/Library/Backend/S3/S3Backend.cs index e6d4d99bb..92d4029d1 100644 --- a/Duplicati/Library/Backend/S3/S3Backend.cs +++ b/Duplicati/Library/Backend/S3/S3Backend.cs @@ -256,7 +256,7 @@ namespace Duplicati.Library.Backend throw new UserInformationException(Strings.S3Backend.UnknownS3ClientError(s3ClientOptionValue), "UnknownS3Client"); } - m_sourcePathKey = $"{ProtocolKey}://{hostname}/{m_bucket}/{m_prefix}/"; + m_sourcePathKey = $"@{ProtocolKey}://{hostname}/{m_bucket}/{m_prefix}/"; m_sourceProvider = new BackendSourceProvider(this); } diff --git a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs index d8057f651..e9abbe27e 100644 --- a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs +++ b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs @@ -118,7 +118,7 @@ namespace Duplicati.Library.Backend if (!string.IsNullOrWhiteSpace(timeoutstr)) m_keepaliveinterval = Library.Utility.Timeparser.ParseTimeSpan(timeoutstr); - m_sourcePathKey = $"{ProtocolKey}://{m_server}:{m_port}/~{m_username}/{m_path}/"; + m_sourcePathKey = $"@{ProtocolKey}://{m_server}:{m_port}/~{m_username}/{m_path}/"; m_sourceProvider = new BackendSourceProvider(this); } diff --git a/Duplicati/Library/Interface/IFolderEnabledBackend.cs b/Duplicati/Library/Interface/IFolderEnabledBackend.cs index 4f83178ad..4b9c6047a 100644 --- a/Duplicati/Library/Interface/IFolderEnabledBackend.cs +++ b/Duplicati/Library/Interface/IFolderEnabledBackend.cs @@ -35,7 +35,7 @@ namespace Duplicati.Library.Interface; public interface IFolderEnabledBackend : IBackend { /// - /// Returns the key for this backend instance's path + /// Returns the key for this backend instance's path, should be prefixed with a "@" /// string PathKey { get; } From bf808452923a0950831fd5f76ceed2ebfdfbf298 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 19 Feb 2025 15:46:22 +0100 Subject: [PATCH 17/28] Fixed a rare startup error --- Duplicati/Library/RestAPI/Scheduler.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/RestAPI/Scheduler.cs b/Duplicati/Library/RestAPI/Scheduler.cs index 281efc0fd..037af95cf 100644 --- a/Duplicati/Library/RestAPI/Scheduler.cs +++ b/Duplicati/Library/RestAPI/Scheduler.cs @@ -152,7 +152,7 @@ namespace Duplicati.Server /// public List WorkerQueue { - get { return (from t in m_worker?.CurrentTasks where t != null select t)?.ToList() ?? []; } + get { return m_worker?.CurrentTasks?.Where(t => t != null)?.ToList() ?? []; } } /// From b1488dae0cd2e849fb8c6c7f935dcfff8f1ccecf Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 19 Feb 2025 15:48:00 +0100 Subject: [PATCH 18/28] Switched to `vssadmin` for listing VSS enabled drives --- .../Windows/WmicShadowCopyManager.cs | 27 ++++++++++++++++++- .../Snapshots/Windows/WmicVssBackup.cs | 20 +++++--------- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/Duplicati/Library/Snapshots/Windows/WmicShadowCopyManager.cs b/Duplicati/Library/Snapshots/Windows/WmicShadowCopyManager.cs index 45a1ddd6b..ab54530b8 100644 --- a/Duplicati/Library/Snapshots/Windows/WmicShadowCopyManager.cs +++ b/Duplicati/Library/Snapshots/Windows/WmicShadowCopyManager.cs @@ -18,7 +18,7 @@ internal class WmicShadowCopyManager : IDisposable /// /// The tag used for logging messages /// - public static readonly string LOGTAG = Logging.Log.LogTagFromType(); + private static readonly string LOGTAG = Logging.Log.LogTagFromType(); /// /// A single shadow copy @@ -159,6 +159,31 @@ internal class WmicShadowCopyManager : IDisposable return null; } + /// + /// Returns the drives that are vss enabled + /// + /// + public static HashSet GetVssCapableDrivesViaVssadmin() + { + var vssDrives = new HashSet(StringComparer.OrdinalIgnoreCase); + + try + { + var output = ExecuteCommand("vssadmin", "list volumes", 5000); + + // Regex to match drive letters in output + var matches = Regex.Matches(output, @"Volume\s+path:\s*([A-Z]:)\\?\s", RegexOptions.IgnoreCase); + foreach (Match match in matches) + vssDrives.Add(match.Groups[1].Value.Substring(0, 1)); + } + catch (Exception ex) + { + Logging.Log.WriteErrorMessage(LOGTAG, "ShadowCopyListFailed", ex, "Failed to list volumes"); + } + + return vssDrives; + } + /// /// Delete a shadow copy /// diff --git a/Duplicati/Library/Snapshots/Windows/WmicVssBackup.cs b/Duplicati/Library/Snapshots/Windows/WmicVssBackup.cs index 37d901dd8..bf7568ed0 100644 --- a/Duplicati/Library/Snapshots/Windows/WmicVssBackup.cs +++ b/Duplicati/Library/Snapshots/Windows/WmicVssBackup.cs @@ -24,7 +24,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Management; using System.Runtime.Versioning; namespace Duplicati.Library.Snapshots.Windows; @@ -40,6 +39,11 @@ internal class WmicVssBackup : ISnapshotProvider /// private WmicShadowCopyManager _manager = new WmicShadowCopyManager(); + /// + /// Gets the VSS enabled drives + /// + private Lazy> _vssEnabledDrives = new Lazy>(() => WmicShadowCopyManager.GetVssCapableDrivesViaVssadmin()); + /// public Guid AddToSnapshotSet(string drive) => _manager.Add(drive).ParsedId; @@ -102,20 +106,8 @@ internal class WmicVssBackup : ISnapshotProvider { if (string.IsNullOrWhiteSpace(drive)) return false; - - try - { - var query = $"SELECT * FROM Win32_ShadowCopy WHERE VolumeName LIKE '%{drive[0]}%'"; - var searcher = new ManagementObjectSearcher(query); - var results = searcher.Get(); - return results.Count > 0; - } - catch (Exception ex) - { - Console.WriteLine($"Error checking VSS support: {ex.Message}"); - return false; - } + return _vssEnabledDrives.Value.Contains(drive.Substring(0, 1)); } /// From 5a4fcaa6d77b30d29b4dc78c128d86ece686d546 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 21 Feb 2025 11:51:45 +0100 Subject: [PATCH 19/28] Added tool to work with remote sources --- Duplicati.sln | 14 +++ .../SourceTool/CommandExtensions.cs | 42 +++++++ .../SourceTool/Commands/Download.cs | 110 ++++++++++++++++++ .../CommandLine/SourceTool/Commands/List.cs | 66 +++++++++++ .../Duplicati.CommandLine.SourceTool.csproj | 25 ++++ .../CommandLine/SourceTool/GetProvider.cs | 39 +++++++ .../CommandLine/SourceTool/ListEnumerator.cs | 60 ++++++++++ Duplicati/CommandLine/SourceTool/Program.cs | 68 +++++++++++ .../Library/AutoUpdater/PackageHelper.cs | 7 +- .../DynamicLoader/SourceProviderLoader.cs | 15 ++- .../Library/Main/Operation/BackupHandler.cs | 23 +--- .../Builtin/BackendSourceProvider.cs | 13 ++- .../Duplicati.CommandLine.SourceTool.csproj | 22 ++++ .../Program.cs | 31 +++++ ReleaseBuilder/Build/Command.cs | 1 + 15 files changed, 513 insertions(+), 23 deletions(-) create mode 100644 Duplicati/CommandLine/SourceTool/CommandExtensions.cs create mode 100644 Duplicati/CommandLine/SourceTool/Commands/Download.cs create mode 100644 Duplicati/CommandLine/SourceTool/Commands/List.cs create mode 100644 Duplicati/CommandLine/SourceTool/Duplicati.CommandLine.SourceTool.csproj create mode 100644 Duplicati/CommandLine/SourceTool/GetProvider.cs create mode 100644 Duplicati/CommandLine/SourceTool/ListEnumerator.cs create mode 100644 Duplicati/CommandLine/SourceTool/Program.cs create mode 100644 Executables/net8/Duplicati.CommandLine.SourceTool/Duplicati.CommandLine.SourceTool.csproj create mode 100644 Executables/net8/Duplicati.CommandLine.SourceTool/Program.cs diff --git a/Duplicati.sln b/Duplicati.sln index 36212bbde..e9f8d2405 100644 --- a/Duplicati.sln +++ b/Duplicati.sln @@ -178,6 +178,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RemoteSynchronization", "To EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.CommandLine.SyncTool", "Executables\net8\Duplicati.CommandLine.SyncTool\Duplicati.CommandLine.SyncTool.csproj", "{48D32674-0FE9-4407-B102-4AE46D93595F}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.CommandLine.SourceTool", "Duplicati\CommandLine\SourceTool\Duplicati.CommandLine.SourceTool.csproj", "{2873BCE2-2EA6-419D-BFFA-693FDB34926E}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.CommandLine.SourceTool", "Executables\net8\Duplicati.CommandLine.SourceTool\Duplicati.CommandLine.SourceTool.csproj", "{4003BF90-6681-4155-9D06-639ED23FE3AB}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -500,6 +504,14 @@ Global {48D32674-0FE9-4407-B102-4AE46D93595F}.Debug|Any CPU.Build.0 = Debug|Any CPU {48D32674-0FE9-4407-B102-4AE46D93595F}.Release|Any CPU.ActiveCfg = Release|Any CPU {48D32674-0FE9-4407-B102-4AE46D93595F}.Release|Any CPU.Build.0 = Release|Any CPU + {2873BCE2-2EA6-419D-BFFA-693FDB34926E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2873BCE2-2EA6-419D-BFFA-693FDB34926E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2873BCE2-2EA6-419D-BFFA-693FDB34926E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2873BCE2-2EA6-419D-BFFA-693FDB34926E}.Release|Any CPU.Build.0 = Release|Any CPU + {4003BF90-6681-4155-9D06-639ED23FE3AB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4003BF90-6681-4155-9D06-639ED23FE3AB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4003BF90-6681-4155-9D06-639ED23FE3AB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4003BF90-6681-4155-9D06-639ED23FE3AB}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -563,6 +575,8 @@ Global {6D9742A7-6F32-4571-8276-79716168B525} = {424CEF73-4984-430E-9C8B-E61CC0F22074} {D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D} = {15388C37-9218-4818-972E-738EEA8F1602} {48D32674-0FE9-4407-B102-4AE46D93595F} = {6B46F6B1-1898-49B8-ADA7-5CAF68EB77E3} + {2873BCE2-2EA6-419D-BFFA-693FDB34926E} = {D19A38DD-68F1-4EF5-BF5F-8966CE0D9A5B} + {4003BF90-6681-4155-9D06-639ED23FE3AB} = {6B46F6B1-1898-49B8-ADA7-5CAF68EB77E3} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {8B40BAFE-D862-4397-9495-8F5EAF5CE80C} diff --git a/Duplicati/CommandLine/SourceTool/CommandExtensions.cs b/Duplicati/CommandLine/SourceTool/CommandExtensions.cs new file mode 100644 index 000000000..019672143 --- /dev/null +++ b/Duplicati/CommandLine/SourceTool/CommandExtensions.cs @@ -0,0 +1,42 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +using System.CommandLine; +using System.CommandLine.Invocation; + +namespace Duplicati.CommandLine.SourceTool; + +/// +/// Extensions for . +/// +public static class CommandExtensions +{ + /// + /// Adds the missing WithHandler method to . + /// + /// The command to add the handler to. + /// The handler to add. + /// The command with the handler added. + public static Command WithHandler(this Command command, ICommandHandler handler) + { + command.Handler = handler; + return command; + } +} diff --git a/Duplicati/CommandLine/SourceTool/Commands/Download.cs b/Duplicati/CommandLine/SourceTool/Commands/Download.cs new file mode 100644 index 000000000..7f2f1d046 --- /dev/null +++ b/Duplicati/CommandLine/SourceTool/Commands/Download.cs @@ -0,0 +1,110 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +using System.CommandLine; +using System.CommandLine.NamingConventionBinder; + +namespace Duplicati.CommandLine.SourceTool.Commands; + +/// +/// The download command +/// +public static class Download +{ + /// + /// Creates the download command + /// + /// The download command + public static Command Create() => + new Command("download", "Downloads all files from the remote") + { + new Argument("url", "The source URL") { + Arity = ArgumentArity.ExactlyOne + }, + new Option("--destination", description: "The destination folder", getDefaultValue: () => new DirectoryInfo(Directory.GetCurrentDirectory())), + new Option("--max-depth", description: "The maximum depth to visit", getDefaultValue: () => 0), + new Option("--max-size", description: "The maximum filesize to download", getDefaultValue: () => 0), + new Option("--overwrite", description: "Overwrite existing files", getDefaultValue: () => false), + } + .WithHandler(CommandHandler.Create(async (url, destination, maxdepth, maxsize, overwrite) => + { + var token = new CancellationTokenSource().Token; + + using var source = Common.GetProvider(url); + + string localPath(string path) + { + var relpath = path.Substring(source.PathKey.Length).TrimStart('/'); + return Path.Combine(destination.FullName, relpath); + } + + var started = DateTime.UtcNow; + var fileCount = 0L; + var folderCount = 0L; + var totalSize = 0L; + var downloadCount = 0L; + var downloadSize = 0L; + + await Common.Visit(source, maxdepth, async (entry, level) => + { + if (entry.IsMetaEntry) + return true; + + var path = localPath(entry.Path); + if (entry.IsFolder) + { + folderCount++; + if (!Directory.Exists(path)) + Directory.CreateDirectory(path); + } + else + { + fileCount++; + totalSize += entry.Size; + + if (entry.Size <= maxsize || maxsize <= 0) + { + var folder = Path.GetDirectoryName(path); + if (!string.IsNullOrWhiteSpace(folder) && !Directory.Exists(folder)) + Directory.CreateDirectory(folder); + + if (!overwrite && File.Exists(path)) + { + Console.WriteLine($"Skipping {entry.Path} as it already exists"); + } + else + { + downloadCount++; + downloadSize += entry.Size; + Console.WriteLine($"Downloading {entry.Path} to {path} ({Library.Utility.Utility.FormatSizeString(entry.Size)})"); + await using var stream = await entry.OpenRead(token); + await using var dest = File.OpenWrite(path); + await stream.CopyToAsync(dest, token); + } + } + } + + return true; + }, token); + + Console.WriteLine($"Found {fileCount} files and {folderCount} folders with a total size of {Library.Utility.Utility.FormatSizeString(totalSize)}"); + Console.WriteLine($"Downloaded {downloadCount} files with a total size of {Library.Utility.Utility.FormatSizeString(downloadSize)} in {DateTime.UtcNow - started}"); + })); +} diff --git a/Duplicati/CommandLine/SourceTool/Commands/List.cs b/Duplicati/CommandLine/SourceTool/Commands/List.cs new file mode 100644 index 000000000..a2169b99e --- /dev/null +++ b/Duplicati/CommandLine/SourceTool/Commands/List.cs @@ -0,0 +1,66 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +using System.CommandLine; +using System.CommandLine.NamingConventionBinder; + +namespace Duplicati.CommandLine.SourceTool.Commands; + +/// +/// The list command +/// +public static class List +{ + /// + /// Creates the list command + /// + /// The command + public static Command Create() => + new Command("list", "Lists all paths on the remote") + { + new Argument("url", "The source URL") { + Arity = ArgumentArity.ExactlyOne + }, + new Option("--max-depth", description: "The maximum depth to list", getDefaultValue: () => 0), + } + .WithHandler(CommandHandler.Create(async (url, maxdepth) => + { + var token = new CancellationTokenSource().Token; + var folders = 0L; + var files = 0L; + + using var source = Common.GetProvider(url); + await Common.Visit(source, maxdepth, (entry, level) => + { + if (entry.IsFolder) + folders++; + else + files++; + if (entry.IsFolder && level > 0) + level--; + Console.WriteLine($"{new string(' ', (level + 1) * 2)}{entry.Path}"); + return Task.FromResult(true); + }, token); + + Console.WriteLine($"Found {folders} folders and {files} files"); + }) + ); + +} diff --git a/Duplicati/CommandLine/SourceTool/Duplicati.CommandLine.SourceTool.csproj b/Duplicati/CommandLine/SourceTool/Duplicati.CommandLine.SourceTool.csproj new file mode 100644 index 000000000..99c103755 --- /dev/null +++ b/Duplicati/CommandLine/SourceTool/Duplicati.CommandLine.SourceTool.csproj @@ -0,0 +1,25 @@ + + + + net8.0 + enable + enable + Duplicati.CommandLine.SourceTool.Implementation + Copyright © 2025 Team Duplicati, MIT license + Duplicati.CommandLine.Sourcetool + + + + + + + + + + + + + + + + diff --git a/Duplicati/CommandLine/SourceTool/GetProvider.cs b/Duplicati/CommandLine/SourceTool/GetProvider.cs new file mode 100644 index 000000000..62888b184 --- /dev/null +++ b/Duplicati/CommandLine/SourceTool/GetProvider.cs @@ -0,0 +1,39 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +using Duplicati.Library.DynamicLoader; +using Duplicati.Library.Interface; + +namespace Duplicati.CommandLine.SourceTool; + +/// +/// Common methods for the source tool +/// +public static partial class Common +{ + /// + /// Gets the source provider for the given URL + /// + /// The URL to get the provider for + /// The source provider + public static ISourceProvider GetProvider(string url) => + SourceProviderLoader.GetSourceProvider(url, []) + ?? throw new UserInformationException(string.Format("The source \"{0}\" is not supported", url), "SourceNotSupported"); +} \ No newline at end of file diff --git a/Duplicati/CommandLine/SourceTool/ListEnumerator.cs b/Duplicati/CommandLine/SourceTool/ListEnumerator.cs new file mode 100644 index 000000000..e8899c8b6 --- /dev/null +++ b/Duplicati/CommandLine/SourceTool/ListEnumerator.cs @@ -0,0 +1,60 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +using Duplicati.Library.Interface; + +namespace Duplicati.CommandLine.SourceTool; + +/// +/// Common methods for the source tool +/// +public static partial class Common +{ + /// + /// Visits all entries in the source provider + /// + /// The source provider to visit + /// The maximum depth to visit + /// The visitor function + /// The cancellation token + /// An awaitable task + public static async Task Visit(ISourceProvider source, int maxdepth, Func> visitor, CancellationToken token) + { + var visit = new Stack<(ISourceFileEntry Entry, int Level)>(); + await foreach (var item in source.Enumerate(token)) + visit.Push((item, 0)); + + while (visit.Count() != 0) + { + var item = visit.Pop(); + var process = await visitor(item.Entry, item.Level); + if (item.Entry.IsFolder && process && (item.Level < maxdepth || maxdepth <= 0)) + { + await foreach (var subitem in item.Entry.Enumerate(token)) + { + if (subitem.IsFolder) + visit.Push((subitem, item.Level + 1)); + else + await visitor(subitem, item.Level); + } + } + } + } +} diff --git a/Duplicati/CommandLine/SourceTool/Program.cs b/Duplicati/CommandLine/SourceTool/Program.cs new file mode 100644 index 000000000..c8aea4225 --- /dev/null +++ b/Duplicati/CommandLine/SourceTool/Program.cs @@ -0,0 +1,68 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +using System.CommandLine; +using System.CommandLine.Builder; +using System.CommandLine.Parsing; +using Duplicati.Library.Interface; +using Duplicati.Library.Utility; + +namespace Duplicati.CommandLine.SourceTool; + +/// +/// The entry point of the application +/// +public static class Program +{ + /// + /// Executes the program + /// + /// The arguments + /// The return code + public static Task Main(string[] args) + { + Library.AutoUpdater.PreloadSettingsLoader.ConfigurePreloadSettings(ref args, Library.AutoUpdater.PackageHelper.NamedExecutable.ServerUtil); + + var rootCmd = new RootCommand("Source CLI tool for Duplicati") + { + Commands.List.Create(), + Commands.Download.Create(), + }; + + return new CommandLineBuilder(rootCmd) + .UseDefaults() + .UseExceptionHandler((ex, context) => + { + if (ex is UserInformationException uie) + { + Console.WriteLine(uie.Message); + context.ExitCode = 2; + } + else + { + Console.WriteLine(ex.ToString()); + context.ExitCode = 1; + } + }) + .UseAdditionalHelpAliases() + .Build() + .InvokeAsync(args); + } +} diff --git a/Duplicati/Library/AutoUpdater/PackageHelper.cs b/Duplicati/Library/AutoUpdater/PackageHelper.cs index 314facb04..4ceefc41e 100644 --- a/Duplicati/Library/AutoUpdater/PackageHelper.cs +++ b/Duplicati/Library/AutoUpdater/PackageHelper.cs @@ -92,7 +92,11 @@ namespace Duplicati.Library.AutoUpdater /// /// The sync tool /// - SyncTool + SyncTool, + /// + /// The source tool + /// + SourceTool } @@ -120,6 +124,7 @@ namespace Duplicati.Library.AutoUpdater NamedExecutable.Agent => OperatingSystem.IsWindows() ? "Duplicati.Agent.exe" : "duplicati-agent", NamedExecutable.SecretTool => OperatingSystem.IsWindows() ? "Duplicati.CommandLine.SecretTool.exe" : "duplicati-secret-tool", NamedExecutable.SyncTool => OperatingSystem.IsWindows() ? "Duplicati.CommandLine.SyncTool.exe" : "duplicati-sync-tool", + NamedExecutable.SourceTool => OperatingSystem.IsWindows() ? "Duplicati.CommandLine.Sourcetool.exe" : "duplicati-source-tool", _ => throw new ArgumentException($"Named executable not known: {exe}", nameof(exe)) }; diff --git a/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs b/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs index 23920c594..f73022611 100644 --- a/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs +++ b/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs @@ -25,6 +25,7 @@ using Duplicati.Library.Interface; using System.Linq; using System.Runtime.ExceptionServices; using Duplicati.Library.SourceProviders; +using Duplicati.Library.SourceProvider; namespace Duplicati.Library.DynamicLoader { @@ -164,7 +165,19 @@ namespace Duplicati.Library.DynamicLoader /// The instanciated SourceProvider or null if the url is not supported public static ISourceProviderModule GetSourceProvider(string url, Dictionary options) { - return _SourceProviderLoader.GetSourceProvider(url, options); + // Source providers are preferred over backends + var provider = _SourceProviderLoader.GetSourceProvider(url, options); + if (provider != null) + return provider; + + // See if there is a backend that can also be a source + var backend = BackendLoader.GetBackend(url, options); + if (backend is IFolderEnabledBackend folderBackend) + return new BackendSourceProvider(folderBackend); + + backend?.Dispose(); + return null; + } #endregion diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 059bbc8df..8d09a6f27 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -138,27 +138,10 @@ namespace Duplicati.Library.Main.Operation results.Add(new LocalFileSource(SnapshotUtility.CreateSnapshot(entry, options.RawOptions, options.SymlinkPolicy == Options.SymlinkStrategy.Follow))); else { + // Assume it is a remote source foreach (var url in entry) - { - // Source providers are preferred over backends - var provider = SourceProviderLoader.GetSourceProvider(url, options.RawOptions); - if (provider != null) - { - results.Add(provider); - continue; - } - - // See if there is a backend that can also be a source - var backend = BackendLoader.GetBackend(url, options.RawOptions); - if (backend is IFolderEnabledBackend folderBackend) - { - results.Add(new BackendSourceProvider(folderBackend)); - continue; - } - - backend?.Dispose(); - throw new UserInformationException(string.Format("The source \"{0}\" is not supported", url), "SourceNotSupported"); - } + results.Add(SourceProviderLoader.GetSourceProvider(url, options.RawOptions) + ?? throw new UserInformationException(string.Format("The source \"{0}\" is not supported", url), "SourceNotSupported")); } } } diff --git a/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs index 6411cc0a7..618de97f7 100644 --- a/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs +++ b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs @@ -19,6 +19,8 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +using System.Runtime.CompilerServices; +using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; namespace Duplicati.Library.SourceProvider; @@ -27,7 +29,7 @@ namespace Duplicati.Library.SourceProvider; /// A source provider that wraps a backend /// /// The backend to wrap -public class BackendSourceProvider(IFolderEnabledBackend backend) : ISourceProvider +public class BackendSourceProvider(IFolderEnabledBackend backend) : ISourceProvider, ISourceProviderModule { /// /// The wrapped backend @@ -39,6 +41,15 @@ public class BackendSourceProvider(IFolderEnabledBackend backend) : ISourceProvi /// public string PathKey => backend.PathKey; + /// + public string Key => backend.ProtocolKey; + + /// + public string DisplayName => backend.DisplayName; + + /// + public IList SupportedCommands => backend.SupportedCommands; + /// public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) => new[] { new BackendSourceFileEntry(this, PathKey, true, true, new DateTime(0), new DateTime(0), 0) }.ToAsyncEnumerable(); diff --git a/Executables/net8/Duplicati.CommandLine.SourceTool/Duplicati.CommandLine.SourceTool.csproj b/Executables/net8/Duplicati.CommandLine.SourceTool/Duplicati.CommandLine.SourceTool.csproj new file mode 100644 index 000000000..038a3b98d --- /dev/null +++ b/Executables/net8/Duplicati.CommandLine.SourceTool/Duplicati.CommandLine.SourceTool.csproj @@ -0,0 +1,22 @@ + + + + Exe + net8.0 + SyncTool for Duplicati + Copyright © 2025 Team Duplicati, MIT license + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + diff --git a/Executables/net8/Duplicati.CommandLine.SourceTool/Program.cs b/Executables/net8/Duplicati.CommandLine.SourceTool/Program.cs new file mode 100644 index 000000000..490864a25 --- /dev/null +++ b/Executables/net8/Duplicati.CommandLine.SourceTool/Program.cs @@ -0,0 +1,31 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +using Duplicati.Library.Crashlog; + +namespace Duplicati.CommandLine.SourceTool.Net8 +{ + // Wrapper class to keep code independent + public static class Program + { + public static int Main(string[] args) + => CrashlogHelper.WrapWithCrashLog(() => SourceTool.Program.Main(args).ConfigureAwait(false).GetAwaiter().GetResult()); + } +} \ No newline at end of file diff --git a/ReleaseBuilder/Build/Command.cs b/ReleaseBuilder/Build/Command.cs index 46ea9f935..94cd6a8c3 100644 --- a/ReleaseBuilder/Build/Command.cs +++ b/ReleaseBuilder/Build/Command.cs @@ -87,6 +87,7 @@ public static partial class Command { "Duplicati.CommandLine.ServerUtil", "duplicati-server-util" }, { "Duplicati.CommandLine.SecretTool", "duplicati-secret-tool" }, { "Duplicati.CommandLine.SyncTool", "duplicati-sync-tool" }, + { "Duplicati.CommandLine.SourceTool", "duplicati-source-tool" }, { "Duplicati.Service", "duplicati-service" }, { "Duplicati.Agent", "duplicati-agent" }, { "Duplicati.CommandLine", "duplicati-cli" }, From 1716d09b5175728d478cc2687867bf2c4dfd8e37 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 21 Feb 2025 13:20:52 +0100 Subject: [PATCH 20/28] Fixed source providers for S3, CIFS, and SSH. Made the SMBShareConnection slightly more robust. Introduced v2 List API to S3 backend, currently it retains v1 as the default. Changed S3 backends to not recurse, potentially making smaller listings. This option can be disabled if it breaks things. Added support for relative paths in the SSH backend. --- Duplicati/Library/Backend/CIFS/CIFS.cs | 20 +- .../Backend/CIFS/SMBShareConnection.cs | 7 +- .../Backend/Idrivee2/Idrivee2Backend.cs | 31 +-- Duplicati/Library/Backend/S3/IS3Client.cs | 2 +- Duplicati/Library/Backend/S3/S3AwsClient.cs | 85 +++++++-- Duplicati/Library/Backend/S3/S3Backend.cs | 39 ++-- Duplicati/Library/Backend/S3/S3MinioClient.cs | 7 +- Duplicati/Library/Backend/S3/Strings.cs | 42 +++-- .../Library/Backend/SSHv2/SSHv2Backend.cs | 65 +++++-- Duplicati/Library/Backend/SSHv2/Strings.cs | 40 ++-- Duplicati/Library/Common/IO/Util.cs | 7 + Duplicati/Library/Main/Controller.cs | 14 ++ .../Main/Database/LocalBackupDatabase.cs | 5 +- .../Library/Main/Operation/BackupHandler.cs | 2 +- .../Builtin/BackendSourceFileEntry.cs | 109 +++++++++-- .../Builtin/BackendSourceProvider.cs | 38 +++- ...cati.Library.SourceProvider.Builtin.csproj | 4 + Duplicati/Library/Utility/MapStream.cs | 176 ------------------ .../Library/Utility/OverrideableStream.cs | 136 +++++++------- Duplicati/Library/Utility/TempFileStream.cs | 10 + 20 files changed, 440 insertions(+), 399 deletions(-) delete mode 100644 Duplicati/Library/Utility/MapStream.cs diff --git a/Duplicati/Library/Backend/CIFS/CIFS.cs b/Duplicati/Library/Backend/CIFS/CIFS.cs index 549721edf..34b05ec0a 100644 --- a/Duplicati/Library/Backend/CIFS/CIFS.cs +++ b/Duplicati/Library/Backend/CIFS/CIFS.cs @@ -32,6 +32,7 @@ using Duplicati.Library.Utility; using SMBLibrary; using Duplicati.Library.SourceProvider; using System.Runtime.CompilerServices; +using Duplicati.Library.Common.IO; namespace Duplicati.Library.Backend; @@ -190,7 +191,11 @@ public class CIFSBackend : IStreamingBackend, IFolderEnabledBackend writeBufferSize ); - _sourcePathKey = $"@{ProtocolKey}://{_connectionParameters.ServerName}/~{_connectionParameters.AuthUser}/{_connectionParameters.Path}/"; + _sourcePathKey = $"{Util.RemotePathPrefix}{ProtocolKey}://{_connectionParameters.ServerName}/{_connectionParameters.ShareName}/"; + if (!string.IsNullOrEmpty(_connectionParameters.AuthUser)) + _sourcePathKey += $"~{_connectionParameters.AuthUser}/"; + if (!string.IsNullOrEmpty(_connectionParameters.Path)) + _sourcePathKey += $"{_connectionParameters.Path}/"; _sourceProvider = new BackendSourceProvider(this); } @@ -360,15 +365,16 @@ public class CIFSBackend : IStreamingBackend, IFolderEnabledBackend public async IAsyncEnumerable ListAsync(string path, [EnumeratorCancellation] CancellationToken cancellationToken) { var sourcePath = _connectionParameters.Path; - if (!sourcePath.EndsWith('\\')) - sourcePath += '\\'; + if (!string.IsNullOrWhiteSpace(sourcePath) && !sourcePath.EndsWith('\\')) + sourcePath += '/'; - var list = (await GetConnection().ListAsync(sourcePath + path, cancellationToken).ConfigureAwait(false)) + foreach (var v in (await GetConnection().ListAsync(sourcePath + path, cancellationToken).ConfigureAwait(false)) + .Where(x => x.Name != "." && x.Name != "..") .Select(f => BackendSourceFileEntry.FromFileEntry(_sourceProvider, path, f)) - .Cast(); - - foreach (var v in list) + .Cast()) + { yield return v; + } } /// diff --git a/Duplicati/Library/Backend/CIFS/SMBShareConnection.cs b/Duplicati/Library/Backend/CIFS/SMBShareConnection.cs index 710e36ea2..47e23b742 100644 --- a/Duplicati/Library/Backend/CIFS/SMBShareConnection.cs +++ b/Duplicati/Library/Backend/CIFS/SMBShareConnection.cs @@ -79,7 +79,7 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable if (!_smb2Client.Connect(connectionParameters.ServerName, connectionParameters.TransportType)) throw new UserInformationException($"{LC.L("Failed to connect to server")} {connectionParameters.ServerName}", "ConnectionError"); - var status = _smb2Client.Login(connectionParameters.AuthDomain, connectionParameters.AuthUser, connectionParameters.AuthPassword); + var status = _smb2Client.Login(connectionParameters.AuthDomain ?? "", connectionParameters.AuthUser ?? "", connectionParameters.AuthPassword ?? ""); if (status != NTStatus.STATUS_SUCCESS) throw new UserInformationException($"{LC.L("Failed to authenticate to server")} {connectionParameters.ServerName} with status {status}", "ConnectionError"); @@ -207,7 +207,6 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable { try { - var status = _smbFileStore.CreateFile( out directoryHandle, out fileStatus, @@ -223,7 +222,7 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable if (status == NTStatus.STATUS_OBJECT_PATH_NOT_FOUND || status == NTStatus.STATUS_OBJECT_NAME_NOT_FOUND) throw new FolderMissingException(); else - throw new UserInformationException($"{LC.L("Failed to open directory")} {NormalizeSlashes(_connectionParameters.Path)} with status {status.ToString()}", "DirectoryOpenError"); + throw new UserInformationException($"{LC.L("Failed to open directory")} {NormalizeSlashes(path)} with status {status}", "DirectoryOpenError"); List fileList; status = _smbFileStore.QueryDirectory( @@ -233,7 +232,7 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable FileInformationClass.FileDirectoryInformation); if (status != NTStatus.STATUS_NO_MORE_FILES) - throw new UserInformationException($"{LC.L("Failed to query directory contents")} with status {status.ToString()}", "DirectoryQueryError"); + throw new UserInformationException($"{LC.L("Failed to query directory contents")} with status {status}", "DirectoryQueryError"); return [ diff --git a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs index a3c5d47be..2d5afdbab 100644 --- a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs +++ b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs @@ -90,7 +90,9 @@ namespace Duplicati.Library.Backend m_s3Client = new S3AwsClient(accessKeyId, accessKeySecret, null, host, null, true, false, options); - m_sourcePathKey = $"@{ProtocolKey}://{host}/{m_bucket}/{m_prefix}/"; + m_sourcePathKey = $"{Util.RemotePathPrefix}{ProtocolKey}://{host}/{m_bucket}/"; + if (!string.IsNullOrWhiteSpace(m_prefix)) + m_sourcePathKey += m_prefix; m_sourceProvider = new BackendSourceProvider(this); } @@ -142,7 +144,7 @@ namespace Duplicati.Library.Backend public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { - await foreach (IFileEntry file in Connection.ListBucketAsync(m_bucket, m_prefix, cancelToken).ConfigureAwait(false)) + await foreach (IFileEntry file in Connection.ListBucketAsync(m_bucket, m_prefix, false, cancelToken).ConfigureAwait(false)) { ((FileEntry)file).Name = file.Name.Substring(m_prefix.Length); if (file.Name.StartsWith("/", StringComparison.Ordinal) && !m_prefix.StartsWith("/", StringComparison.Ordinal)) @@ -249,30 +251,9 @@ namespace Duplicati.Library.Backend public string PathKey => m_sourcePathKey; /// - public async IAsyncEnumerable ListAsync(string path, [EnumeratorCancellation] CancellationToken cancellationToken) - { - var prefixPath = m_prefix; - if (!prefixPath.EndsWith("/", StringComparison.Ordinal)) - prefixPath += "/"; + public IAsyncEnumerable ListAsync(string path, CancellationToken cancellationToken) + => m_sourceProvider.ListFromFileEntryAsync(m_prefix, path, (filter, token) => m_s3Client.ListBucketAsync(m_bucket, filter, false, token), cancellationToken); - var filterPath = prefixPath + path; - if (!filterPath.EndsWith("/", StringComparison.Ordinal)) - filterPath += "/"; - - await foreach (var f in m_s3Client.ListBucketAsync(m_bucket, filterPath, cancellationToken).ConfigureAwait(false)) - { - if (!f.Name.StartsWith(prefixPath, StringComparison.Ordinal)) - continue; - - ((FileEntry)f).Name = f.Name.Substring(prefixPath.Length); - - // Skip sub-folder items - if (f.Name.Contains("/")) - continue; - - yield return BackendSourceFileEntry.FromFileEntry(m_sourceProvider, path, f); - } - } /// public Task GetEntryAsync(string path, CancellationToken cancellationToken) diff --git a/Duplicati/Library/Backend/S3/IS3Client.cs b/Duplicati/Library/Backend/S3/IS3Client.cs index 6d32833ee..3a0cb65b9 100644 --- a/Duplicati/Library/Backend/S3/IS3Client.cs +++ b/Duplicati/Library/Backend/S3/IS3Client.cs @@ -28,7 +28,7 @@ namespace Duplicati.Library.Backend { public interface IS3Client : IDisposable { - IAsyncEnumerable ListBucketAsync(string bucketName, string prefix, CancellationToken cancellationToken); + IAsyncEnumerable ListBucketAsync(string bucketName, string prefix, bool recursive, CancellationToken cancellationToken); Task AddBucketAsync(string bucketName, CancellationToken cancelToken); diff --git a/Duplicati/Library/Backend/S3/S3AwsClient.cs b/Duplicati/Library/Backend/S3/S3AwsClient.cs index 94e578182..5cc424666 100644 --- a/Duplicati/Library/Backend/S3/S3AwsClient.cs +++ b/Duplicati/Library/Backend/S3/S3AwsClient.cs @@ -49,6 +49,7 @@ namespace Duplicati.Library.Backend private readonly bool m_useChunkEncoding; private readonly string m_dnsHost; + private readonly bool m_useV2ListApi; public S3AwsClient(string awsID, string awsKey, string locationConstraint, string servername, string storageClass, bool useSSL, bool disableChunkEncoding, Dictionary options) @@ -61,6 +62,7 @@ namespace Duplicati.Library.Backend m_client = new AmazonS3Client(awsID, awsKey, cfg); + m_useV2ListApi = string.Equals(options.GetValueOrDefault("list-api-version", "v1"), "v2", StringComparison.OrdinalIgnoreCase); m_locationConstraint = locationConstraint; m_storageClass = storageClass; m_dnsHost = string.IsNullOrWhiteSpace(cfg.ServiceURL) ? null : new System.Uri(cfg.ServiceURL).Host; @@ -192,32 +194,65 @@ namespace Duplicati.Library.Backend return m_client.DeleteObjectAsync(objectDeleteRequest, cancellationToken); } - public async IAsyncEnumerable ListBucketAsync(string bucketName, string prefix, [EnumeratorCancellation] CancellationToken cancellationToken) + /// + /// Lists the contents of a bucket, using a plugable API call + /// + /// The bucket to list + /// The prefix to list + /// If true, the list is recursive + /// The cancellation token + /// The list of files + public async IAsyncEnumerable ListBucketAsync(string bucketName, string prefix, bool recursive, [EnumeratorCancellation] CancellationToken cancellationToken) { - bool isTruncated = true; + var isTruncated = true; string filename = null; + var delimiter = recursive ? "" : "/"; //TODO: Figure out if this is the case with AWSSDK too //Unfortunately S3 sometimes reports duplicate values when requesting more than one page of results //So, track the files that have already been returned and skip any duplicates. - HashSet alreadyReturned = new HashSet(); + var alreadyReturned = new HashSet(); //We truncate after ITEM_LIST_LIMIT elements, and then repeat while (isTruncated) { - var listRequest = new ListObjectsRequest { BucketName = bucketName }; - - if (!string.IsNullOrEmpty(filename)) - listRequest.Marker = filename; - - listRequest.MaxKeys = ITEM_LIST_LIMIT; - if (!string.IsNullOrEmpty(prefix)) - listRequest.Prefix = prefix; - - ListObjectsResponse listResponse; + ListObjectsV2Response listResponse; try { - listResponse = await m_client.ListObjectsAsync(listRequest, cancellationToken).ConfigureAwait(false); + if (m_useV2ListApi) + { + var listRequest = new ListObjectsV2Request + { + BucketName = bucketName, + Prefix = prefix, + ContinuationToken = filename, + MaxKeys = ITEM_LIST_LIMIT, + Delimiter = delimiter + }; + listResponse = await m_client.ListObjectsV2Async(listRequest, cancellationToken).ConfigureAwait(false); + } + else + { + // Use the V1 API + var listRequest = new ListObjectsRequest + { + BucketName = bucketName, + Prefix = prefix, + Marker = filename, + MaxKeys = ITEM_LIST_LIMIT, + Delimiter = delimiter + }; + var listResponsev1 = await m_client.ListObjectsAsync(listRequest, cancellationToken).ConfigureAwait(false); + + // Map the V1 response to the V2 response + listResponse = new ListObjectsV2Response + { + CommonPrefixes = listResponsev1.CommonPrefixes, + IsTruncated = listResponsev1.IsTruncated, + NextContinuationToken = listResponsev1.NextMarker, + S3Objects = listResponsev1.S3Objects + }; + } } catch (AmazonS3Exception e) { @@ -231,16 +266,34 @@ namespace Duplicati.Library.Backend } isTruncated = listResponse.IsTruncated; - filename = listResponse.NextMarker; + filename = listResponse.NextContinuationToken; + + foreach (var obj in listResponse.CommonPrefixes) + { + // Because the prefixes are returned, and not the folder objects + // we do not get the folder modification date + yield return new Common.IO.FileEntry( + obj, + -1, + new DateTime(0), + new DateTime(0) + ) + { IsFolder = true }; + } foreach (var obj in listResponse.S3Objects.Where(obj => alreadyReturned.Add(obj.Key))) { + // Skip self-prefix, this discards the folder modification date :/ + if (obj.Key == prefix) + continue; + yield return new Common.IO.FileEntry( obj.Key, obj.Size, obj.LastModified, obj.LastModified - ); + ) + { IsFolder = obj.Key.EndsWith("/") }; } } } diff --git a/Duplicati/Library/Backend/S3/S3Backend.cs b/Duplicati/Library/Backend/S3/S3Backend.cs index 92d4029d1..dffb5035d 100644 --- a/Duplicati/Library/Backend/S3/S3Backend.cs +++ b/Duplicati/Library/Backend/S3/S3Backend.cs @@ -44,6 +44,8 @@ namespace Duplicati.Library.Backend private const string SSL_OPTION = "use-ssl"; private const string S3_CLIENT_OPTION = "s3-client"; private const string S3_DISABLE_CHUNK_ENCODING_OPTION = "s3-disable-chunk-encoding"; + private const string S3_LIST_API_VERSION_OPTION = "s3-list-api-version"; + private const string S3_RECURSIVE_LIST = "s3-recursive-list"; public static readonly Dictionary KNOWN_S3_PROVIDERS = new Dictionary(StringComparer.OrdinalIgnoreCase) { { "Amazon S3", "s3.amazonaws.com" }, @@ -174,6 +176,7 @@ namespace Duplicati.Library.Backend private readonly string m_bucket; private readonly string m_prefix; + private readonly bool m_recurseLists; /// /// Source provider to be used if the backend is a source @@ -235,6 +238,8 @@ namespace Duplicati.Library.Backend if (m_prefix.Length != 0) m_prefix = Util.AppendDirSeparator(m_prefix, "/"); + m_recurseLists = Utility.Utility.ParseBoolOption(options, S3_RECURSIVE_LIST); + // Auto-disable DNS lookup for non-AWS configurations if (!options.ContainsKey("s3-ext-forcepathstyle") && !hostname.EndsWith(".amazonaws.com", StringComparison.OrdinalIgnoreCase)) options["s3-ext-forcepathstyle"] = "true"; @@ -256,7 +261,9 @@ namespace Duplicati.Library.Backend throw new UserInformationException(Strings.S3Backend.UnknownS3ClientError(s3ClientOptionValue), "UnknownS3Client"); } - m_sourcePathKey = $"@{ProtocolKey}://{hostname}/{m_bucket}/{m_prefix}/"; + m_sourcePathKey = $"{Util.RemotePathPrefix}{ProtocolKey}://{hostname}/{m_bucket}/"; + if (!string.IsNullOrWhiteSpace(m_prefix)) + m_sourcePathKey += m_prefix; m_sourceProvider = new BackendSourceProvider(this); } @@ -287,7 +294,7 @@ namespace Duplicati.Library.Backend public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { - await foreach (IFileEntry file in Connection.ListBucketAsync(m_bucket, m_prefix, cancelToken)) + await foreach (IFileEntry file in Connection.ListBucketAsync(m_bucket, m_prefix, m_recurseLists, cancelToken)) { ((FileEntry)file).Name = file.Name.Substring(m_prefix.Length); @@ -349,6 +356,8 @@ namespace Duplicati.Library.Backend new CommandLineArgument(SSL_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.S3Backend.DescriptionUseSSLShort, Strings.S3Backend.DescriptionUseSSLLong), new CommandLineArgument(S3_CLIENT_OPTION, CommandLineArgument.ArgumentType.Enumeration, Strings.S3Backend.S3ClientDescriptionShort, Strings.S3Backend.S3ClientDescriptionLong, "aws", null, new string[] { "aws", "minio" }), new CommandLineArgument(S3_DISABLE_CHUNK_ENCODING_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.S3Backend.DescriptionDisableChunkEncodingShort, Strings.S3Backend.DescriptionDisableChunkEncodingLong, "false"), + new CommandLineArgument(S3_LIST_API_VERSION_OPTION, CommandLineArgument.ArgumentType.Enumeration, Strings.S3Backend.DescriptionListApiVersionShort, Strings.S3Backend.DescriptionListApiVersionLong, "v1", null, ["v1", "v2"]), + new CommandLineArgument(S3_RECURSIVE_LIST, CommandLineArgument.ArgumentType.Boolean, Strings.S3Backend.DescriptionRecursiveListShort, Strings.S3Backend.DescriptionRecursiveListLong, "false"), new CommandLineArgument("auth-password", CommandLineArgument.ArgumentType.Password, Strings.S3Backend.AuthPasswordDescriptionShort, Strings.S3Backend.AuthPasswordDescriptionLong), new CommandLineArgument("auth-username", CommandLineArgument.ArgumentType.String, Strings.S3Backend.AuthUsernameDescriptionShort, Strings.S3Backend.AuthUsernameDescriptionLong), }; @@ -416,30 +425,8 @@ namespace Duplicati.Library.Backend public string PathKey => m_sourcePathKey; /// - public async IAsyncEnumerable ListAsync(string path, [EnumeratorCancellation] CancellationToken cancellationToken) - { - var prefixPath = m_prefix; - if (!prefixPath.EndsWith("/", StringComparison.Ordinal)) - prefixPath += "/"; - - var filterPath = prefixPath + path; - if (!filterPath.EndsWith("/", StringComparison.Ordinal)) - filterPath += "/"; - - await foreach (var f in s3Client.ListBucketAsync(m_bucket, filterPath, cancellationToken).ConfigureAwait(false)) - { - if (!f.Name.StartsWith(prefixPath, StringComparison.Ordinal)) - continue; - - ((FileEntry)f).Name = f.Name.Substring(prefixPath.Length); - - // Skip sub-folder items - if (f.Name.Contains("/")) - continue; - - yield return BackendSourceFileEntry.FromFileEntry(m_sourceProvider, path, f); - } - } + public IAsyncEnumerable ListAsync(string path, CancellationToken cancellationToken) + => m_sourceProvider.ListFromFileEntryAsync(m_prefix, path, (filter, token) => s3Client.ListBucketAsync(m_bucket, filter, m_recurseLists, token), cancellationToken); /// public Task GetEntryAsync(string path, CancellationToken cancellationToken) diff --git a/Duplicati/Library/Backend/S3/S3MinioClient.cs b/Duplicati/Library/Backend/S3/S3MinioClient.cs index cee966473..e48c7b301 100644 --- a/Duplicati/Library/Backend/S3/S3MinioClient.cs +++ b/Duplicati/Library/Backend/S3/S3MinioClient.cs @@ -81,11 +81,11 @@ namespace Duplicati.Library.Backend } } - public async IAsyncEnumerable ListBucketAsync(string bucketName, string prefix, [EnumeratorCancellation] CancellationToken cancellationToken) + public async IAsyncEnumerable ListBucketAsync(string bucketName, string prefix, bool recursive, [EnumeratorCancellation] CancellationToken cancellationToken) { ThrowExceptionIfBucketDoesNotExist(bucketName); - var observable = m_client.ListObjectsAsync(bucketName, prefix, true, cancellationToken); + var observable = m_client.ListObjectsAsync(bucketName, prefix, recursive, cancellationToken); await foreach (var obj in ToAsyncEnumerable(observable, cancellationToken).ConfigureAwait(false)) { yield return new Common.IO.FileEntry( @@ -93,7 +93,8 @@ namespace Duplicati.Library.Backend (long)obj.Size, Convert.ToDateTime(obj.LastModified), Convert.ToDateTime(obj.LastModified) - ); + ) + { IsFolder = obj.Key.EndsWith("/") }; } } diff --git a/Duplicati/Library/Backend/S3/Strings.cs b/Duplicati/Library/Backend/S3/Strings.cs index 49062dc97..c7721bcdd 100644 --- a/Duplicati/Library/Backend/S3/Strings.cs +++ b/Duplicati/Library/Backend/S3/Strings.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Localization.Short; namespace Duplicati.Library.Backend.Strings @@ -49,6 +49,10 @@ namespace Duplicati.Library.Backend.Strings public static string DescriptionDisableChunkEncodingShort { get { return LC.L(@"Disable chunk encoding (aws client only)"); } } public static string S3StorageclassDescriptionLong { get { return LC.L(@"Use this option to specify a storage class. If this option is not used, the server will choose a default storage class."); } } public static string S3StorageclassDescriptionShort { get { return LC.L(@"Specify storage class"); } } + public static string DescriptionListApiVersionShort { get { return LC.L(@"Specify the S3 list API version to use"); } } + public static string DescriptionListApiVersionLong { get { return LC.L(@"Use this option to specify the S3 list API version to use. This can be used to work around issues with some S3 providers."); } } + public static string DescriptionRecursiveListShort { get { return LC.L(@"Use this option to list all files in the bucket"); } } + public static string DescriptionRecursiveListLong { get { return LC.L(@"To reduce the number of objects listed, the default is to only list the first level of objects. Use this option to list all objects in the bucket."); } } public static string UnknownS3ClientError(string client) { return LC.L(@"Unknown S3 client: {0}", client); } } } diff --git a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs index e9abbe27e..8f3c1e846 100644 --- a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs +++ b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs @@ -39,6 +39,7 @@ namespace Duplicati.Library.Backend private const string SSH_KEYFILE_OPTION = "ssh-keyfile"; private const string SSH_KEYFILE_INLINE = "ssh-key"; private const string SSH_FINGERPRINT_OPTION = "ssh-fingerprint"; + private const string SSH_RELATIVE_PATH_OPTION = "ssh-relative-path"; private const string SSH_FINGERPRINT_ACCEPT_ANY_OPTION = "ssh-accept-any-fingerprints"; public const string KEYFILE_URI = "sshkey://"; private const string SSH_TIMEOUT_OPTION = "ssh-operation-timeout"; @@ -56,6 +57,8 @@ namespace Duplicati.Library.Backend private readonly TimeSpan m_keepaliveinterval; private readonly int m_port = 22; + private readonly bool m_relativePath; + private string m_initialDirectory; /// /// The source provider for this backend @@ -91,6 +94,7 @@ namespace Duplicati.Library.Backend m_password = uri.Password; m_fingerprintallowall = Utility.Utility.ParseBoolOption(options, SSH_FINGERPRINT_ACCEPT_ANY_OPTION); + m_relativePath = Utility.Utility.ParseBoolOption(options, SSH_RELATIVE_PATH_OPTION); m_path = uri.Path; @@ -99,8 +103,15 @@ namespace Duplicati.Library.Backend m_path = Util.AppendDirSeparator(m_path, "/"); } - if (!m_path.StartsWith("/", StringComparison.Ordinal)) - m_path = "/" + m_path; + if (m_relativePath) + { + m_path = m_path.TrimStart('/'); + } + else + { + if (!m_path.StartsWith("/", StringComparison.Ordinal)) + m_path = "/" + m_path; + } m_server = uri.Host; @@ -118,7 +129,9 @@ namespace Duplicati.Library.Backend if (!string.IsNullOrWhiteSpace(timeoutstr)) m_keepaliveinterval = Library.Utility.Timeparser.ParseTimeSpan(timeoutstr); - m_sourcePathKey = $"@{ProtocolKey}://{m_server}:{m_port}/~{m_username}/{m_path}/"; + m_sourcePathKey = $"{Util.RemotePathPrefix}{ProtocolKey}://{m_server}:{m_port}/~{m_username}/"; + if (string.IsNullOrWhiteSpace(m_path)) + m_sourcePathKey += $"{m_path}/"; m_sourceProvider = new BackendSourceProvider(this); } @@ -179,7 +192,7 @@ namespace Duplicati.Library.Backend try { CreateConnection(); - ChangeDirectory(m_path); + SetWorkingDirectory(); await m_con.DeleteFileAsync(remotename, cancelToken).ConfigureAwait(false); } catch (SftpPathNotFoundException ex) @@ -218,6 +231,9 @@ namespace Duplicati.Library.Backend new CommandLineArgument(SSH_KEEPALIVE_OPTION, CommandLineArgument.ArgumentType.Timespan, Strings.SSHv2Backend.DescriptionSshkeepaliveShort, Strings.SSHv2Backend.DescriptionSshkeepaliveLong, "0"), + new CommandLineArgument(SSH_RELATIVE_PATH_OPTION, CommandLineArgument.ArgumentType.Boolean, + Strings.SSHv2Backend.DescriptionRelativePathShort, + Strings.SSHv2Backend.DescriptionRelativePathLong) }); } } @@ -255,7 +271,7 @@ namespace Duplicati.Library.Backend public Task PutAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) { CreateConnection(); - ChangeDirectory(m_path); + SetWorkingDirectory(); return Task.Factory.FromAsync( (cb, state) => m_con.BeginUploadFile(stream, remotename, cb, state, _ => cancelToken.ThrowIfCancellationRequested()), m_con.EndUploadFile, @@ -265,7 +281,7 @@ namespace Duplicati.Library.Backend public async Task GetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) { CreateConnection(); - ChangeDirectory(m_path); + SetWorkingDirectory(); try { @@ -287,7 +303,7 @@ namespace Duplicati.Library.Backend public Task RenameAsync(string source, string target, CancellationToken cancelToken) { CreateConnection(); - ChangeDirectory(m_path); + SetWorkingDirectory(); return m_con.RenameFileAsync(source, target, cancelToken); } @@ -358,6 +374,9 @@ namespace Duplicati.Library.Backend con.KeepAliveInterval = m_keepaliveinterval; this.TryConnect(con); + if (m_relativePath && (string.IsNullOrWhiteSpace(con.WorkingDirectory) || !con.WorkingDirectory.StartsWith("/"))) + throw new UserInformationException("Server does not report absolute initial directory, please switch to absolute paths", "RelativePathNotSupported"); + m_initialDirectory = con.WorkingDirectory; m_con = con; } @@ -367,23 +386,27 @@ namespace Duplicati.Library.Backend client.Connect(); } - private void ChangeDirectory(string path) + private void SetWorkingDirectory() { - if (string.IsNullOrEmpty(path)) + if (string.IsNullOrEmpty(m_path)) return; + var targetPath = m_relativePath + ? Util.AppendDirSeparator(m_initialDirectory, "/") + m_path + : m_path; + var workingDir = Util.AppendDirSeparator(m_con.WorkingDirectory, "/"); - if (workingDir == path) + if (workingDir == targetPath) return; try { - m_con.ChangeDirectory(path); + m_con.ChangeDirectory(targetPath); } catch (Exception ex) { throw new Interface.FolderMissingException( - Strings.SSHv2Backend.FolderNotFoundManagedError(path, ex.Message), ex); + Strings.SSHv2Backend.FolderNotFoundManagedError(m_path, ex.Message), ex); } } @@ -392,7 +415,7 @@ namespace Duplicati.Library.Backend string path = "."; CreateConnection(); - ChangeDirectory(m_path); + SetWorkingDirectory(); foreach (var ls in m_con.ListDirectory(path)) { @@ -449,13 +472,19 @@ namespace Duplicati.Library.Backend /// public IAsyncEnumerable ListAsync(string path, CancellationToken cancellationToken) { - var prefixPath = m_path; - if (!prefixPath.EndsWith("/")) - prefixPath += "/"; + CreateConnection(); + SetWorkingDirectory(); + + var prefixPath = m_relativePath ? "" : m_path; + if (!string.IsNullOrWhiteSpace(prefixPath)) + prefixPath = Util.AppendDirSeparator(prefixPath, "/"); var filterPath = prefixPath + path; - if (!filterPath.EndsWith("/")) - filterPath += "/"; + if (!string.IsNullOrWhiteSpace(filterPath)) + filterPath = Util.AppendDirSeparator(filterPath, "/"); + + if (string.IsNullOrWhiteSpace(filterPath)) + filterPath = "."; return Client.ListDirectoryAsync(filterPath, cancellationToken) .Select(x => new BackendSourceFileEntry( diff --git a/Duplicati/Library/Backend/SSHv2/Strings.cs b/Duplicati/Library/Backend/SSHv2/Strings.cs index 95360cdd8..a182b771a 100644 --- a/Duplicati/Library/Backend/SSHv2/Strings.cs +++ b/Duplicati/Library/Backend/SSHv2/Strings.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Localization.Short; namespace Duplicati.Library.Backend.Strings @@ -61,6 +61,8 @@ namespace Duplicati.Library.Backend.Strings public static string DescriptionSshtimeoutShort { get { return LC.L(@"Set the operation timeout value"); } } public static string DescriptionSshkeepaliveLong { get { return LC.L(@"Use this option to enable the keep-alive interval for the SSH connection. If the connection is idle, aggressive firewalls might close the connection. Using keep-alive will keep the connection open in this scenario. If this value is set to zero, the keep-alive is disabled."); } } public static string DescriptionSshkeepaliveShort { get { return LC.L(@"Set a keepalive value"); } } + public static string DescriptionRelativePathShort { get { return LC.L(@"Treat source path as relative to the initial path"); } } + public static string DescriptionRelativePathLong { get { return LC.L(@"Use this option to treat the source path as relative to the initial path. This is useful when the full path of the system is not known."); } } public static string FolderNotFoundManagedError(string foldername, string message) { return LC.L(@"Unable to set folder to {0}, error message: {1}", foldername, message); } public static string FingerprintNotMatchManagedError(string fingerprint) { return LC.L(@"Validation of server fingerprint failed. Server returned fingerprint ""{0}"". Cause of this message is either not correct configuration or Man-in-the-middle attack!", fingerprint); } public static string FingerprintNotSpecifiedManagedError(string fingerprint, string hostkeyoption, string allkeysoptions) { return LC.L(@"Please add --{1}=""{0}"" to trust this host. Optionally you can use --{2} (NOT SECURE) for testing!", fingerprint, hostkeyoption, allkeysoptions); } diff --git a/Duplicati/Library/Common/IO/Util.cs b/Duplicati/Library/Common/IO/Util.cs index 80b247cf6..460c6fb2c 100644 --- a/Duplicati/Library/Common/IO/Util.cs +++ b/Duplicati/Library/Common/IO/Util.cs @@ -36,6 +36,11 @@ namespace Duplicati.Library.Common.IO /// public static readonly string AltDirectorySeparatorString = Path.AltDirectorySeparatorChar.ToString(); + /// + /// The prefix marker for a remote path + /// + public const string RemotePathPrefix = "@"; + /// /// Filename of a marker file that can be put inside the data folder to prevent Duplicati from fixing lax permissions /// @@ -72,6 +77,8 @@ namespace Duplicati.Library.Common.IO /// The guessed directory separator public static string GuessDirSeparator(string path) { + if (path != null && path.StartsWith(RemotePathPrefix, StringComparison.Ordinal)) + throw new ArgumentException("Path cannot start with @, most likely this is a remote path", nameof(path)); return string.IsNullOrWhiteSpace(path) || path.StartsWith("/", StringComparison.Ordinal) ? "/" : "\\"; } diff --git a/Duplicati/Library/Main/Controller.cs b/Duplicati/Library/Main/Controller.cs index d391b7fb3..312df9f8f 100644 --- a/Duplicati/Library/Main/Controller.cs +++ b/Duplicati/Library/Main/Controller.cs @@ -948,6 +948,20 @@ namespace Duplicati.Library.Main string source; try { + // Check if this is not a file source + var scheme = Library.Utility.Utility.GuessScheme(expandedSource) ?? "file"; + if (scheme != "file") + { + // TODO: If the remote source fails to load, + // this will be an enumeration warning, but will result + // in the backup being recorded without files from the source + // Eventually, this could lead to retention causing the last backup + // with the data from the source to be deleted + foundAnyPaths = true; + sources.Add(expandedSource); + continue; + } + // TODO: This expands "C:" to CWD, but not C:\ source = System.IO.Path.GetFullPath(expandedSource); } diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index 888b4fea5..f558c6e8a 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -22,6 +22,7 @@ using System; using System.Collections.Generic; using System.Data; using System.Linq; +using Duplicati.Library.Common.IO; namespace Duplicati.Library.Main.Database { @@ -877,11 +878,11 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist } } - public string GetFirstPath() + public string GetFirstLocalPath() { using (var cmd = m_connection.CreateCommand()) { - cmd.CommandText = @"SELECT ""Path"" FROM ""File"" ORDER BY LENGTH(""Path"") DESC LIMIT 1"; + cmd.CommandText = @$"SELECT ""Path"" FROM ""File"" WHERE ""Path"" NOT LIKE '{Util.RemotePathPrefix}%' ORDER BY LENGTH(""Path"") DESC LIMIT 1"; var v0 = cmd.ExecuteScalar(); if (v0 == null || v0 == DBNull.Value) return null; diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 8d09a6f27..de97b7449 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -269,7 +269,7 @@ namespace Duplicati.Library.Main.Operation Utility.UpdateOptionsFromDb(database, options); Utility.VerifyOptionsAndUpdateDatabase(database, options); - var probe_path = database.GetFirstPath(); + var probe_path = database.GetFirstLocalPath(); if (probe_path != null && Util.GuessDirSeparator(probe_path) != Util.DirectorySeparatorString) throw new UserInformationException(string.Format("The backup contains files that belong to another operating system. Proceeding with a backup would cause the database to contain paths from two different operation systems, which is not supported. To proceed without losing remote data, delete all filesets and make sure the --{0} option is set, then run the backup again to re-use the existing data on the remote store.", "no-auto-compact"), "CrossOsDatabaseReuseNotSupported"); diff --git a/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs b/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs index 7b6fa496c..535ec7d5d 100644 --- a/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs +++ b/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs @@ -19,6 +19,7 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +using System.IO.Pipelines; using Duplicati.Library.Interface; using Duplicati.Library.Utility; @@ -37,6 +38,11 @@ namespace Duplicati.Library.SourceProvider; public class BackendSourceFileEntry(BackendSourceProvider parent, string path, bool isFolder, bool isMetaEntry, DateTime createdUtc, DateTime lastModificationUtc, long size) : ISourceFileEntry { + /// + /// The log tag for this instance + /// + private static string LOGTAG = Logging.Log.LogTagFromType(); + /// public bool IsFolder => isFolder; @@ -55,11 +61,6 @@ public class BackendSourceFileEntry(BackendSourceProvider parent, string path, b /// public string Path => ConcatPaths(parent.PathKey, NormalizePath(path)); - /// - /// The local path of the entry - /// - public string LocalPath => path; - /// public long Size => size; /// @@ -95,7 +96,13 @@ public class BackendSourceFileEntry(BackendSourceProvider parent, string path, b throw new InvalidOperationException("Enumerate can only be called on folders"); return parent.WrappedBackend.ListAsync(path, cancellationToken) - .Select(x => new BackendSourceFileEntry(parent, x.Path, x.IsFolder, false, x.CreatedUtc, x.LastModificationUtc, x.Size)); + .Select(x => + { + if (x is BackendSourceFileEntry) + return x; + + return new BackendSourceFileEntry(parent, x.Path, x.IsFolder, false, x.CreatedUtc, x.LastModificationUtc, x.Size); + }); } /// @@ -119,6 +126,59 @@ public class BackendSourceFileEntry(BackendSourceProvider parent, string path, b public Task OpenMetadataRead(CancellationToken cancellationToken) => Task.FromResult(null); + /// + /// Helper class for reporting th length of a stream + /// + /// The stream to wrap + /// The length to report + private class LengthReportingStream(Stream stream, long reportedLength) : OverrideableStream(stream) + { + /// + /// Track the position + /// + private long position = 0; + + /// + public override long Length => reportedLength; + + /// + public override long Position + { + get => position; + set => position = base.Position = value; + } + + /// + public override void Write(byte[] buffer, int offset, int count) + { + base.Write(buffer, offset, count); + position += count; + } + + /// + public override int Read(byte[] buffer, int offset, int count) + { + var read = base.Read(buffer, offset, count); + position += read; + return read; + } + + /// + public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + var read = await base.ReadAsync(buffer, offset, count, cancellationToken); + position += read; + return read; + } + + /// + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + await base.WriteAsync(buffer, offset, count, cancellationToken); + position += count; + } + } + /// public async Task OpenRead(CancellationToken cancellationToken) { @@ -127,23 +187,48 @@ public class BackendSourceFileEntry(BackendSourceProvider parent, string path, b if (parent.WrappedBackend is IStreamingBackend streamingBackend) { - var mapStream = new MapStream(); - mapStream.CopyTask = streamingBackend.GetAsync(this.Path, mapStream, cancellationToken); - return mapStream; + var pipe = new Pipe(); + + // Start writing data to the pipe asynchronously + _ = Task.Run(async () => + { + try + { + await streamingBackend.GetAsync(path, pipe.Writer.AsStream(), cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + await pipe.Writer.CompleteAsync(ex).ConfigureAwait(false); + } + finally + { + await pipe.Writer.CompleteAsync().ConfigureAwait(false); + } + }) + .ContinueWith(t => + { + if (t.IsFaulted) + Logging.Log.WriteWarningMessage(LOGTAG, "ErrorPipingStream", t.Exception, "Error piping stream for {0}", this.Path); + }); + + // Return the readable stream so the caller can read data as it's produced + return new LengthReportingStream(pipe.Reader.AsStream(), size); } else { + TempFile? tempFile = null; TempFileStream? file = null; try { - file = TempFileStream.Create(); - await parent.WrappedBackend.GetAsync(this.Path, file.Path, cancellationToken); - file.Position = 0; + tempFile = new TempFile(); + await parent.WrappedBackend.GetAsync(path, tempFile, cancellationToken); + file = TempFileStream.Create(tempFile); return file; } catch { file?.Dispose(); + tempFile?.Dispose(); throw; } } diff --git a/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs index 618de97f7..3943fb1b0 100644 --- a/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs +++ b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs @@ -52,7 +52,7 @@ public class BackendSourceProvider(IFolderEnabledBackend backend) : ISourceProvi /// public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) - => new[] { new BackendSourceFileEntry(this, PathKey, true, true, new DateTime(0), new DateTime(0), 0) }.ToAsyncEnumerable(); + => new[] { new BackendSourceFileEntry(this, "", true, true, new DateTime(0), new DateTime(0), 0) }.ToAsyncEnumerable(); /// public Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) @@ -64,4 +64,40 @@ public class BackendSourceProvider(IFolderEnabledBackend backend) : ISourceProvi WrappedBackend.Dispose(); GC.SuppressFinalize(this); } + + /// + /// Helper function to use a List function that returns IFileEntry and convert it to ISourceFileEntry + /// + /// The prefix to hide from the caller + /// The logical path to list + /// The function that returns the IFileEntry items + /// The cancellation token + /// The list of source file entries + public async IAsyncEnumerable ListFromFileEntryAsync(string prefix, string path, Func> listFunction, [EnumeratorCancellation] CancellationToken cancellationToken) + { + if (!string.IsNullOrWhiteSpace(prefix) && !prefix.EndsWith("/", StringComparison.Ordinal)) + prefix += "/"; + + // Logic here is that the prefix is invisible to the caller, so we add it to the prefix to filter the results + // and remove it from the name to make the prefix invisible. + // The input path is preserved so the output looks like a "full path" to the caller. + + var filterPath = prefix + path; + if (filterPath != "" && !filterPath.EndsWith("/", StringComparison.Ordinal)) + filterPath += "/"; + + await foreach (var f in listFunction(filterPath, cancellationToken).ConfigureAwait(false)) + { + if (!f.Name.StartsWith(filterPath, StringComparison.Ordinal)) + continue; + + ((FileEntry)f).Name = f.Name.Substring(filterPath.Length); + + // Skip self and sub-folder items + if (f.Name.Length == 0 || f.Name[0..^1].Contains("/")) + continue; + + yield return BackendSourceFileEntry.FromFileEntry(this, path, f); + } + } } diff --git a/Duplicati/Library/SourceProvider/Builtin/Duplicati.Library.SourceProvider.Builtin.csproj b/Duplicati/Library/SourceProvider/Builtin/Duplicati.Library.SourceProvider.Builtin.csproj index 4ba5377b7..e48c06ee4 100644 --- a/Duplicati/Library/SourceProvider/Builtin/Duplicati.Library.SourceProvider.Builtin.csproj +++ b/Duplicati/Library/SourceProvider/Builtin/Duplicati.Library.SourceProvider.Builtin.csproj @@ -24,4 +24,8 @@ + + + + diff --git a/Duplicati/Library/Utility/MapStream.cs b/Duplicati/Library/Utility/MapStream.cs deleted file mode 100644 index b2c717739..000000000 --- a/Duplicati/Library/Utility/MapStream.cs +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - -#nullable enable - -using System; -using System.IO; -using System.Threading; -using System.Threading.Channels; -using System.Threading.Tasks; - -namespace Duplicati.Library.Utility; - -/// -/// A stream that maps data with an internal buffer. -/// This allows a reader and a writer to share a stream without blocking. -/// -public class MapStream : Stream -{ - /// - /// The channel used to communicate between the reader and writer. - /// - private readonly Channel channel = Channel.CreateBounded(0); - - /// - /// The buffer used to store data that has been written but not consumed by the reader. - /// - private Memory readBuffer = Array.Empty(); - - /// - /// The task that is using the map stream. - /// - public Task? CopyTask { get; set; } - - /// - public override bool CanRead => true; - - /// - public override bool CanSeek => false; - - /// - public override bool CanWrite => true; - - /// - public override long Length => throw new NotImplementedException(); - - /// - public override long Position { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } - - /// - public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - int read = 0; - // If we have left-over data from the previous read, return it - if (readBuffer.Length > 0) - { - // Return as much as possible - var requested = Math.Min(readBuffer.Length, count); - readBuffer.Span.Slice(0, requested).CopyTo(buffer.AsSpan(offset, count)); - // Adjust the remaining buffer - if (readBuffer.Length > requested) - readBuffer = readBuffer.Slice(requested); - else - readBuffer = Array.Empty(); - - // Adjust the counters - offset += requested; - count -= requested; - read += requested; - - // Return if we have read all that was requested - if (count == 0) - return read; - } - - // Handle external task - var ct = CopyTask; - - // If the task is completed, clear it - if (ct != null && ct.IsCompletedSuccessfully) - CopyTask = ct = null; - // If the task failed, stop reading - else if (ct != null && (ct.IsCanceled || ct.IsFaulted)) - { - // If we have any data in the buffer, return it - if (read > 0) - return read; - - // Otherwise, throw the exception - await ct; - } - - if (!await channel.Reader.WaitToReadAsync(cancellationToken)) - { - // If we have any data in the buffer, return it - if (read > 0) - return read; - - // If the copy crashed, throw the exception - ct = CopyTask; - if (ct != null && (ct.IsCanceled || ct.IsFaulted)) - await ct; - - // Otherwise, return 0 - return read; - } - - // Read the data from the channel - var res = await channel.Reader.ReadAsync(cancellationToken); - - // Consume as much as was requested - var consumed = Math.Min(res.Length, count); - - // Keep the rest in the buffer - if (res.Length > count) - readBuffer = res.AsMemory(count); - - res.AsSpan(0, consumed).CopyTo(buffer.AsSpan(offset, count)); - read += consumed; - return read; - } - - /// - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - var ct = CopyTask; - // Throw if the task failed - if (ct != null && (ct.IsCanceled || ct.IsFaulted)) - await ct; - await channel.Writer.WriteAsync(buffer.AsSpan(offset, count).ToArray(), cancellationToken); - } - - public override void Close() - { - base.Close(); - channel.Writer.Complete(); - } - - protected override void Dispose(bool disposing) - { - channel.Writer.Complete(); - base.Dispose(disposing); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) - => ReadAsync(buffer, offset, count, CancellationToken.None).Await(); - - public override long Seek(long offset, SeekOrigin origin) - => throw new NotImplementedException(); - - public override void SetLength(long value) - => throw new NotImplementedException(); - - public override void Write(byte[] buffer, int offset, int count) - => WriteAsync(buffer, offset, count, CancellationToken.None).Await(); -} diff --git a/Duplicati/Library/Utility/OverrideableStream.cs b/Duplicati/Library/Utility/OverrideableStream.cs index dd65cbbd0..35e8a5273 100644 --- a/Duplicati/Library/Utility/OverrideableStream.cs +++ b/Duplicati/Library/Utility/OverrideableStream.cs @@ -1,28 +1,27 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. using System; -using System.Collections.Generic; -using System.Text; using System.IO; +using System.Threading.Tasks; namespace Duplicati.Library.Utility { @@ -33,79 +32,78 @@ namespace Duplicati.Library.Utility /// public class OverrideableStream : Stream { - protected System.IO.Stream m_basestream; + /// + /// The base stream that is wrapped + /// + protected Stream m_basestream; + /// + /// Creates a new instance + /// + /// The stream to wrap + /// public OverrideableStream(Stream basestream) { - if (basestream == null) - throw new ArgumentNullException(nameof(basestream)); - m_basestream = basestream; + m_basestream = basestream ?? throw new ArgumentNullException(nameof(basestream)); } - public override bool CanRead - { - get { return m_basestream.CanRead; } - } + /// + public override bool CanRead => m_basestream.CanRead; - public override bool CanSeek - { - get { return m_basestream.CanSeek; } - } + /// + public override bool CanSeek => m_basestream.CanSeek; - public override bool CanWrite - { - get { return m_basestream.CanWrite; } - } + /// + public override bool CanWrite => m_basestream.CanWrite; - public override void Flush() - { - m_basestream.Flush(); - } + /// + public override void Flush() => m_basestream.Flush(); - public override long Length - { - get { return m_basestream.Length; } - } + /// + public override long Length => m_basestream.Length; + /// public override long Position { - get - { - return m_basestream.Position; - } - set - { - m_basestream.Position = value; - } + get => m_basestream.Position; + set => m_basestream.Position = value; } + /// public override int Read(byte[] buffer, int offset, int count) - { - return m_basestream.Read(buffer, offset, count); - } + => m_basestream.Read(buffer, offset, count); + /// public override long Seek(long offset, System.IO.SeekOrigin origin) - { - return m_basestream.Seek(offset, origin); - } + => m_basestream.Seek(offset, origin); + /// public override void SetLength(long value) - { - m_basestream.SetLength(value); - } + => m_basestream.SetLength(value); + /// public override void Write(byte[] buffer, int offset, int count) - { - m_basestream.Write(buffer, offset, count); - } + => m_basestream.Write(buffer, offset, count); + /// + public override Task ReadAsync(byte[] buffer, int offset, int count, System.Threading.CancellationToken cancellationToken) + => m_basestream.ReadAsync(buffer, offset, count, cancellationToken); + + /// + public override Task WriteAsync(byte[] buffer, int offset, int count, System.Threading.CancellationToken cancellationToken) + => m_basestream.WriteAsync(buffer, offset, count, cancellationToken); + + /// + public override Task FlushAsync(System.Threading.CancellationToken cancellationToken) + => m_basestream.FlushAsync(cancellationToken); + + /// protected override void Dispose(bool disposing) { - if (m_basestream != null) - m_basestream.Dispose(); + m_basestream?.Dispose(); m_basestream = null; base.Dispose(disposing); } - + } } diff --git a/Duplicati/Library/Utility/TempFileStream.cs b/Duplicati/Library/Utility/TempFileStream.cs index 104eab261..85b8efcd2 100644 --- a/Duplicati/Library/Utility/TempFileStream.cs +++ b/Duplicati/Library/Utility/TempFileStream.cs @@ -59,6 +59,16 @@ public class TempFileStream : StreamUtil.WrappingAsyncStream this.stream = stream; } + /// + /// Creates a new TempFileStream instance. + /// + /// The temporary file instance. + /// The new TempFileStream instance. + public static TempFileStream Create(TempFile tempFile) + { + return new TempFileStream(tempFile, File.Open(tempFile.Name, FileMode.Open, FileAccess.ReadWrite, FileShare.None)); + } + /// /// Creates a new TempFileStream instance. /// From cd512ea7dd06e19d87d283235861d2a3ea921038 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 21 Feb 2025 13:25:00 +0100 Subject: [PATCH 21/28] Changed remote path prefix to `>` to avoid clash with literal string prefix `@` --- Duplicati/Library/Common/IO/Util.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Common/IO/Util.cs b/Duplicati/Library/Common/IO/Util.cs index 460c6fb2c..f982b7568 100644 --- a/Duplicati/Library/Common/IO/Util.cs +++ b/Duplicati/Library/Common/IO/Util.cs @@ -39,7 +39,7 @@ namespace Duplicati.Library.Common.IO /// /// The prefix marker for a remote path /// - public const string RemotePathPrefix = "@"; + public const string RemotePathPrefix = ">"; /// /// Filename of a marker file that can be put inside the data folder to prevent Duplicati from fixing lax permissions From 9dfe977d3f67b04f07303aca41d9e93a821a010d Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 21 Feb 2025 16:06:55 +0100 Subject: [PATCH 22/28] Changed logic so the source providers are now logically mounted into the source directory structure. This fixes the restore issues as all sources will now appear as if they were originally part of the source file system. The backend implementation has now been simplified so the backends only return names of entries with no path information. Most backends already do this, but the S3 backends did not, so this was fixed as well. This also clears up the interfaces, so the backend does not know about the `ISourceProviderEntry`. All mapping is done by the `BackendSourceProvider`, making it simpler to support other backends as sources. --- Duplicati/CommandLine/CLI/Commands.cs | 2 +- .../SourceTool/Commands/Download.cs | 2 +- .../CommandLine/SourceTool/Commands/List.cs | 3 +- .../CommandLine/SourceTool/GetProvider.cs | 2 +- .../CommandLine/SourceTool/ListEnumerator.cs | 4 +- Duplicati/Library/Backend/CIFS/CIFS.cs | 40 +++-------- .../Backend/Idrivee2/Idrivee2Backend.cs | 24 ++----- Duplicati/Library/Backend/S3/S3AwsClient.cs | 12 +++- Duplicati/Library/Backend/S3/S3Backend.cs | 35 ++++----- Duplicati/Library/Backend/S3/S3MinioClient.cs | 11 ++- .../Library/Backend/SSHv2/SSHv2Backend.cs | 42 ++++------- Duplicati/Library/Common/IO/Util.cs | 7 -- .../DynamicLoader/SourceProviderLoader.cs | 11 +-- .../Library/Interface/IDestinationProvider.cs | 43 +++++++++++ .../Interface/IFolderEnabledBackend.cs | 18 +++-- .../Library/Interface/ISourceProvider.cs | 8 +-- ...ceFileEntry.cs => ISourceProviderEntry.cs} | 6 +- Duplicati/Library/Main/Controller.cs | 9 ++- .../Main/Database/LocalBackupDatabase.cs | 4 +- .../Library/Main/Operation/Backup/Channels.cs | 2 +- .../Backup/FileEnumerationProcess.cs | 22 +++--- .../Operation/Backup/MetadataGenerator.cs | 2 +- .../Operation/Backup/MetadataPreProcess.cs | 4 +- .../Library/Main/Operation/BackupHandler.cs | 31 ++++++-- .../Library/Snapshots/ISnapshotService.cs | 6 +- Duplicati/Library/Snapshots/LinuxSnapshot.cs | 2 +- .../Library/Snapshots/NoSnapshotLinux.cs | 2 +- .../Library/Snapshots/NoSnapshotWindows.cs | 2 +- Duplicati/Library/Snapshots/SnapshotBase.cs | 6 +- .../Snapshots/SnapshotSourceFileEntry.cs | 4 +- .../Snapshots/USN/UsnJournalService.cs | 22 +++--- .../Library/Snapshots/WindowsSnapshot.cs | 2 +- .../Builtin/BackendSourceFileEntry.cs | 72 ++++++++++--------- .../Builtin/BackendSourceProvider.cs | 54 +++----------- .../SourceProvider/Builtin/Combiner.cs | 12 ++-- .../SourceProvider/Builtin/LocalFileSource.cs | 6 +- Duplicati/Library/Utility/Utility.cs | 3 + 37 files changed, 256 insertions(+), 281 deletions(-) create mode 100644 Duplicati/Library/Interface/IDestinationProvider.cs rename Duplicati/Library/Interface/{ISourceFileEntry.cs => ISourceProviderEntry.cs} (96%) diff --git a/Duplicati/CommandLine/CLI/Commands.cs b/Duplicati/CommandLine/CLI/Commands.cs index b5834a95d..76425af0c 100644 --- a/Duplicati/CommandLine/CLI/Commands.cs +++ b/Duplicati/CommandLine/CLI/Commands.cs @@ -622,7 +622,7 @@ namespace Duplicati.CommandLine switch (phase) { case Duplicati.Library.Main.OperationPhase.Backup_ProcessingFiles: - output.MessageEvent("Scanning local files ..."); + output.MessageEvent("Scanning source files ..."); periodicOutput.SetReady(); break; case Duplicati.Library.Main.OperationPhase.Backup_Finalize: diff --git a/Duplicati/CommandLine/SourceTool/Commands/Download.cs b/Duplicati/CommandLine/SourceTool/Commands/Download.cs index 7f2f1d046..295545ccd 100644 --- a/Duplicati/CommandLine/SourceTool/Commands/Download.cs +++ b/Duplicati/CommandLine/SourceTool/Commands/Download.cs @@ -51,7 +51,7 @@ public static class Download string localPath(string path) { - var relpath = path.Substring(source.PathKey.Length).TrimStart('/'); + var relpath = path.Substring(source.MountedPath.Length).TrimStart(Path.DirectorySeparatorChar); return Path.Combine(destination.FullName, relpath); } diff --git a/Duplicati/CommandLine/SourceTool/Commands/List.cs b/Duplicati/CommandLine/SourceTool/Commands/List.cs index a2169b99e..fd8cb6c25 100644 --- a/Duplicati/CommandLine/SourceTool/Commands/List.cs +++ b/Duplicati/CommandLine/SourceTool/Commands/List.cs @@ -55,7 +55,8 @@ public static class List files++; if (entry.IsFolder && level > 0) level--; - Console.WriteLine($"{new string(' ', (level + 1) * 2)}{entry.Path}"); + if (!entry.IsRootEntry) + Console.WriteLine($"{new string(' ', (level + 1) * 2)}{entry.Path}"); return Task.FromResult(true); }, token); diff --git a/Duplicati/CommandLine/SourceTool/GetProvider.cs b/Duplicati/CommandLine/SourceTool/GetProvider.cs index 62888b184..1f8b30e01 100644 --- a/Duplicati/CommandLine/SourceTool/GetProvider.cs +++ b/Duplicati/CommandLine/SourceTool/GetProvider.cs @@ -34,6 +34,6 @@ public static partial class Common /// The URL to get the provider for /// The source provider public static ISourceProvider GetProvider(string url) => - SourceProviderLoader.GetSourceProvider(url, []) + SourceProviderLoader.GetSourceProvider(url, "", []) ?? throw new UserInformationException(string.Format("The source \"{0}\" is not supported", url), "SourceNotSupported"); } \ No newline at end of file diff --git a/Duplicati/CommandLine/SourceTool/ListEnumerator.cs b/Duplicati/CommandLine/SourceTool/ListEnumerator.cs index e8899c8b6..5c6ca8845 100644 --- a/Duplicati/CommandLine/SourceTool/ListEnumerator.cs +++ b/Duplicati/CommandLine/SourceTool/ListEnumerator.cs @@ -35,9 +35,9 @@ public static partial class Common /// The visitor function /// The cancellation token /// An awaitable task - public static async Task Visit(ISourceProvider source, int maxdepth, Func> visitor, CancellationToken token) + public static async Task Visit(ISourceProvider source, int maxdepth, Func> visitor, CancellationToken token) { - var visit = new Stack<(ISourceFileEntry Entry, int Level)>(); + var visit = new Stack<(ISourceProviderEntry Entry, int Level)>(); await foreach (var item in source.Enumerate(token)) visit.Push((item, 0)); diff --git a/Duplicati/Library/Backend/CIFS/CIFS.cs b/Duplicati/Library/Backend/CIFS/CIFS.cs index 34b05ec0a..8af6a8494 100644 --- a/Duplicati/Library/Backend/CIFS/CIFS.cs +++ b/Duplicati/Library/Backend/CIFS/CIFS.cs @@ -61,16 +61,6 @@ public class CIFSBackend : IStreamingBackend, IFolderEnabledBackend /// private string _DnsName; - /// - /// Source provider to be used if the backend is a source - /// - private readonly BackendSourceProvider _sourceProvider; - - /// - /// Key to be used in the path for the source provider - /// - private readonly string _sourcePathKey; - /// /// Path separators (both Windows \ and unix /) to be used in path manipulation /// @@ -190,13 +180,6 @@ public class CIFSBackend : IStreamingBackend, IFolderEnabledBackend readBufferSize, writeBufferSize ); - - _sourcePathKey = $"{Util.RemotePathPrefix}{ProtocolKey}://{_connectionParameters.ServerName}/{_connectionParameters.ShareName}/"; - if (!string.IsNullOrEmpty(_connectionParameters.AuthUser)) - _sourcePathKey += $"~{_connectionParameters.AuthUser}/"; - if (!string.IsNullOrEmpty(_connectionParameters.Path)) - _sourcePathKey += $"{_connectionParameters.Path}/"; - _sourceProvider = new BackendSourceProvider(this); } /// @@ -359,25 +342,18 @@ public class CIFSBackend : IStreamingBackend, IFolderEnabledBackend } /// - public string PathKey => _sourcePathKey; - - /// - public async IAsyncEnumerable ListAsync(string path, [EnumeratorCancellation] CancellationToken cancellationToken) + public async IAsyncEnumerable ListAsync(string path, [EnumeratorCancellation] CancellationToken cancellationToken) { var sourcePath = _connectionParameters.Path; - if (!string.IsNullOrWhiteSpace(sourcePath) && !sourcePath.EndsWith('\\')) - sourcePath += '/'; + if (!string.IsNullOrWhiteSpace(sourcePath)) + sourcePath = Util.AppendDirSeparator(sourcePath, "/"); - foreach (var v in (await GetConnection().ListAsync(sourcePath + path, cancellationToken).ConfigureAwait(false)) - .Where(x => x.Name != "." && x.Name != "..") - .Select(f => BackendSourceFileEntry.FromFileEntry(_sourceProvider, path, f)) - .Cast()) - { - yield return v; - } + foreach (var v in await GetConnection().ListAsync(sourcePath + BackendSourceFileEntry.NormalizePathTo(path, '/'), cancellationToken).ConfigureAwait(false)) + if (v.Name != "." && v.Name != "..") + yield return v; } /// - public Task GetEntryAsync(string path, CancellationToken cancellationToken) - => Task.FromResult(null); + public Task GetEntryAsync(string path, CancellationToken cancellationToken) + => Task.FromResult(null); } \ No newline at end of file diff --git a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs index 2d5afdbab..db797a927 100644 --- a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs +++ b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs @@ -44,15 +44,6 @@ namespace Duplicati.Library.Backend private readonly string m_prefix; private readonly string m_bucket; - /// - /// The source provider for this backend - /// - private readonly BackendSourceProvider m_sourceProvider; - /// - /// The key the backend is identified by if it is a source - /// - private readonly string m_sourcePathKey; - private IS3Client m_s3Client; public Idrivee2Backend() @@ -90,10 +81,6 @@ namespace Duplicati.Library.Backend m_s3Client = new S3AwsClient(accessKeyId, accessKeySecret, null, host, null, true, false, options); - m_sourcePathKey = $"{Util.RemotePathPrefix}{ProtocolKey}://{host}/{m_bucket}/"; - if (!string.IsNullOrWhiteSpace(m_prefix)) - m_sourcePathKey += m_prefix; - m_sourceProvider = new BackendSourceProvider(this); } public string GetRegionEndpoint(string url) @@ -248,15 +235,12 @@ namespace Duplicati.Library.Backend } /// - public string PathKey => m_sourcePathKey; - - /// - public IAsyncEnumerable ListAsync(string path, CancellationToken cancellationToken) - => m_sourceProvider.ListFromFileEntryAsync(m_prefix, path, (filter, token) => m_s3Client.ListBucketAsync(m_bucket, filter, false, token), cancellationToken); + public IAsyncEnumerable ListAsync(string path, CancellationToken cancellationToken) + => m_s3Client.ListBucketAsync(m_bucket, path, false, cancellationToken); /// - public Task GetEntryAsync(string path, CancellationToken cancellationToken) - => Task.FromResult(null); + public Task GetEntryAsync(string path, CancellationToken cancellationToken) + => Task.FromResult(null); } } diff --git a/Duplicati/Library/Backend/S3/S3AwsClient.cs b/Duplicati/Library/Backend/S3/S3AwsClient.cs index 5cc424666..4527826dd 100644 --- a/Duplicati/Library/Backend/S3/S3AwsClient.cs +++ b/Duplicati/Library/Backend/S3/S3AwsClient.cs @@ -22,6 +22,7 @@ using Amazon.S3; using Amazon.S3.Model; +using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using Duplicati.Library.Utility; using System; @@ -207,6 +208,8 @@ namespace Duplicati.Library.Backend var isTruncated = true; string filename = null; var delimiter = recursive ? "" : "/"; + if (!string.IsNullOrWhiteSpace(prefix)) + prefix = Util.AppendDirSeparator(prefix, "/"); //TODO: Figure out if this is the case with AWSSDK too //Unfortunately S3 sometimes reports duplicate values when requesting more than one page of results @@ -270,10 +273,13 @@ namespace Duplicati.Library.Backend foreach (var obj in listResponse.CommonPrefixes) { + if (obj == prefix || !obj.StartsWith(prefix)) + continue; + // Because the prefixes are returned, and not the folder objects // we do not get the folder modification date yield return new Common.IO.FileEntry( - obj, + obj.Substring(prefix.Length), -1, new DateTime(0), new DateTime(0) @@ -284,11 +290,11 @@ namespace Duplicati.Library.Backend foreach (var obj in listResponse.S3Objects.Where(obj => alreadyReturned.Add(obj.Key))) { // Skip self-prefix, this discards the folder modification date :/ - if (obj.Key == prefix) + if (obj.Key == prefix || !obj.Key.StartsWith(prefix)) continue; yield return new Common.IO.FileEntry( - obj.Key, + obj.Key.Substring(prefix.Length), obj.Size, obj.LastModified, obj.LastModified diff --git a/Duplicati/Library/Backend/S3/S3Backend.cs b/Duplicati/Library/Backend/S3/S3Backend.cs index dffb5035d..2136eedbc 100644 --- a/Duplicati/Library/Backend/S3/S3Backend.cs +++ b/Duplicati/Library/Backend/S3/S3Backend.cs @@ -178,15 +178,6 @@ namespace Duplicati.Library.Backend private readonly string m_prefix; private readonly bool m_recurseLists; - /// - /// Source provider to be used if the backend is a source - /// - private readonly BackendSourceProvider m_sourceProvider; - /// - /// The key path for the backend as a source - /// - private readonly string m_sourcePathKey; - private const string DEFAULT_S3_HOST = "s3.amazonaws.com"; private IS3Client s3Client; @@ -260,11 +251,6 @@ namespace Duplicati.Library.Backend { throw new UserInformationException(Strings.S3Backend.UnknownS3ClientError(s3ClientOptionValue), "UnknownS3Client"); } - - m_sourcePathKey = $"{Util.RemotePathPrefix}{ProtocolKey}://{hostname}/{m_bucket}/"; - if (!string.IsNullOrWhiteSpace(m_prefix)) - m_sourcePathKey += m_prefix; - m_sourceProvider = new BackendSourceProvider(this); } public static bool IsValidHostname(string bucketname) @@ -422,14 +408,21 @@ namespace Duplicati.Library.Backend } /// - public string PathKey => m_sourcePathKey; + public IAsyncEnumerable ListAsync(string path, CancellationToken cancellationToken) + { + var prefix = m_prefix; + if (!string.IsNullOrWhiteSpace(prefix)) + prefix = Util.AppendDirSeparator(prefix); + + var filterPath = prefix + path; + if (!string.IsNullOrWhiteSpace(filterPath)) + filterPath = Util.AppendDirSeparator(filterPath); + + return s3Client.ListBucketAsync(m_bucket, filterPath, m_recurseLists, cancellationToken); + } /// - public IAsyncEnumerable ListAsync(string path, CancellationToken cancellationToken) - => m_sourceProvider.ListFromFileEntryAsync(m_prefix, path, (filter, token) => s3Client.ListBucketAsync(m_bucket, filter, m_recurseLists, token), cancellationToken); - - /// - public Task GetEntryAsync(string path, CancellationToken cancellationToken) - => Task.FromResult(null); + public Task GetEntryAsync(string path, CancellationToken cancellationToken) + => Task.FromResult(null); } } diff --git a/Duplicati/Library/Backend/S3/S3MinioClient.cs b/Duplicati/Library/Backend/S3/S3MinioClient.cs index e48c7b301..3f7d48f1c 100644 --- a/Duplicati/Library/Backend/S3/S3MinioClient.cs +++ b/Duplicati/Library/Backend/S3/S3MinioClient.cs @@ -26,6 +26,7 @@ using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Channels; using System.Threading.Tasks; +using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using Duplicati.Library.Utility; using Minio; @@ -85,11 +86,17 @@ namespace Duplicati.Library.Backend { ThrowExceptionIfBucketDoesNotExist(bucketName); + if (!string.IsNullOrWhiteSpace(prefix)) + prefix = Util.AppendDirSeparator(prefix, "/"); + var observable = m_client.ListObjectsAsync(bucketName, prefix, recursive, cancellationToken); await foreach (var obj in ToAsyncEnumerable(observable, cancellationToken).ConfigureAwait(false)) { - yield return new Common.IO.FileEntry( - obj.Key, + if (obj.Key == prefix || !obj.Key.StartsWith(prefix)) + continue; + + yield return new FileEntry( + obj.Key.Substring(prefix.Length), (long)obj.Size, Convert.ToDateTime(obj.LastModified), Convert.ToDateTime(obj.LastModified) diff --git a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs index 8f3c1e846..fa31e6c6f 100644 --- a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs +++ b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs @@ -60,15 +60,6 @@ namespace Duplicati.Library.Backend private readonly bool m_relativePath; private string m_initialDirectory; - /// - /// The source provider for this backend - /// - private readonly BackendSourceProvider m_sourceProvider; - /// - /// The path key for this backend if used as a source - /// - private readonly string m_sourcePathKey; - private SftpClient m_con; public SSHv2() @@ -128,11 +119,6 @@ namespace Duplicati.Library.Backend if (!string.IsNullOrWhiteSpace(timeoutstr)) m_keepaliveinterval = Library.Utility.Timeparser.ParseTimeSpan(timeoutstr); - - m_sourcePathKey = $"{Util.RemotePathPrefix}{ProtocolKey}://{m_server}:{m_port}/~{m_username}/"; - if (string.IsNullOrWhiteSpace(m_path)) - m_sourcePathKey += $"{m_path}/"; - m_sourceProvider = new BackendSourceProvider(this); } #region IBackend Members @@ -467,10 +453,7 @@ namespace Duplicati.Library.Backend public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new[] { m_server }); /// - public string PathKey => m_sourcePathKey; - - /// - public IAsyncEnumerable ListAsync(string path, CancellationToken cancellationToken) + public IAsyncEnumerable ListAsync(string path, CancellationToken cancellationToken) { CreateConnection(); SetWorkingDirectory(); @@ -479,7 +462,7 @@ namespace Duplicati.Library.Backend if (!string.IsNullOrWhiteSpace(prefixPath)) prefixPath = Util.AppendDirSeparator(prefixPath, "/"); - var filterPath = prefixPath + path; + var filterPath = prefixPath + BackendSourceFileEntry.NormalizePathTo(path, '/'); if (!string.IsNullOrWhiteSpace(filterPath)) filterPath = Util.AppendDirSeparator(filterPath, "/"); @@ -487,19 +470,18 @@ namespace Duplicati.Library.Backend filterPath = "."; return Client.ListDirectoryAsync(filterPath, cancellationToken) - .Select(x => new BackendSourceFileEntry( - m_sourceProvider, - BackendSourceFileEntry.ConcatPaths(path, x.Name), - x.Attributes.IsDirectory, - false, - new DateTime(0), - x.Attributes.LastWriteTimeUtc, - x.Attributes.Size - )); + .Select(x => new FileEntry( + x.Name, + x.Attributes.Size, + x.Attributes.LastAccessTimeUtc, + x.Attributes.LastWriteTimeUtc) + { + IsFolder = x.Attributes.IsDirectory + }); } /// - public Task GetEntryAsync(string path, CancellationToken cancellationToken) - => Task.FromResult(null); + public Task GetEntryAsync(string path, CancellationToken cancellationToken) + => Task.FromResult(null); } } \ No newline at end of file diff --git a/Duplicati/Library/Common/IO/Util.cs b/Duplicati/Library/Common/IO/Util.cs index f982b7568..80b247cf6 100644 --- a/Duplicati/Library/Common/IO/Util.cs +++ b/Duplicati/Library/Common/IO/Util.cs @@ -36,11 +36,6 @@ namespace Duplicati.Library.Common.IO /// public static readonly string AltDirectorySeparatorString = Path.AltDirectorySeparatorChar.ToString(); - /// - /// The prefix marker for a remote path - /// - public const string RemotePathPrefix = ">"; - /// /// Filename of a marker file that can be put inside the data folder to prevent Duplicati from fixing lax permissions /// @@ -77,8 +72,6 @@ namespace Duplicati.Library.Common.IO /// The guessed directory separator public static string GuessDirSeparator(string path) { - if (path != null && path.StartsWith(RemotePathPrefix, StringComparison.Ordinal)) - throw new ArgumentException("Path cannot start with @, most likely this is a remote path", nameof(path)); return string.IsNullOrWhiteSpace(path) || path.StartsWith("/", StringComparison.Ordinal) ? "/" : "\\"; } diff --git a/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs b/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs index f73022611..b5e7819a6 100644 --- a/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs +++ b/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs @@ -65,7 +65,7 @@ namespace Duplicati.Library.DynamicLoader /// The url to create the instance for /// The options to pass to the instance constructor /// The instanciated SourceProvider or null if the url is not supported - public ISourceProviderModule GetSourceProvider(string url, Dictionary options) + public ISourceProviderModule GetSourceProvider(string url, string mountPoint, Dictionary options) { var uri = new Utility.Uri(url); @@ -80,7 +80,7 @@ namespace Duplicati.Library.DynamicLoader try { if (m_interfaces.ContainsKey(uri.Scheme)) - return (ISourceProviderModule)Activator.CreateInstance(m_interfaces[uri.Scheme].GetType(), url, newOpts); + return (ISourceProviderModule)Activator.CreateInstance(m_interfaces[uri.Scheme].GetType(), url, mountPoint, newOpts); } catch (System.Reflection.TargetInvocationException tex) { @@ -161,19 +161,20 @@ namespace Duplicati.Library.DynamicLoader /// Instanciates a specific SourceProvider, given the url and options /// /// The url to create the instance for + /// The mount point to use /// The options to pass to the instance constructor /// The instanciated SourceProvider or null if the url is not supported - public static ISourceProviderModule GetSourceProvider(string url, Dictionary options) + public static ISourceProviderModule GetSourceProvider(string url, string mountPoint, Dictionary options) { // Source providers are preferred over backends - var provider = _SourceProviderLoader.GetSourceProvider(url, options); + var provider = _SourceProviderLoader.GetSourceProvider(url, mountPoint, options); if (provider != null) return provider; // See if there is a backend that can also be a source var backend = BackendLoader.GetBackend(url, options); if (backend is IFolderEnabledBackend folderBackend) - return new BackendSourceProvider(folderBackend); + return new BackendSourceProvider(folderBackend, mountPoint); backend?.Dispose(); return null; diff --git a/Duplicati/Library/Interface/IDestinationProvider.cs b/Duplicati/Library/Interface/IDestinationProvider.cs new file mode 100644 index 000000000..20467a0b5 --- /dev/null +++ b/Duplicati/Library/Interface/IDestinationProvider.cs @@ -0,0 +1,43 @@ +#nullable enable + +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace Duplicati.Library.Interface +{ + /// + /// Implements a destination where data can be restored to + /// + public interface IDestinationProvider : IDynamicModule + { + /// + /// Gets the module key + /// + string Key { get; } + /// + /// Gets the display name of the module + /// + string DisplayName { get; } + + /// + /// Checks if a restore is needed + /// + /// The path to restore to + /// The hash of the data to restore + /// The hash of the metadata to restore + /// The cancellation token + /// True if a restore is needed, false otherwise + Task IsRestoreNeededAsync(string path, string dataHash, string? metadataHash, CancellationToken cancel); + + /// + /// Restores a file to the given path + /// + /// The path to restore to + /// The data stream to restore + /// The metadata stream to restore + /// The cancellation token + /// A task that completes when the restore is done + Task WriteAsync(string path, Stream data, Stream? metadata, CancellationToken cancel); + } +} \ No newline at end of file diff --git a/Duplicati/Library/Interface/IFolderEnabledBackend.cs b/Duplicati/Library/Interface/IFolderEnabledBackend.cs index 4b9c6047a..6f9374aed 100644 --- a/Duplicati/Library/Interface/IFolderEnabledBackend.cs +++ b/Duplicati/Library/Interface/IFolderEnabledBackend.cs @@ -34,11 +34,6 @@ namespace Duplicati.Library.Interface; /// public interface IFolderEnabledBackend : IBackend { - /// - /// Returns the key for this backend instance's path, should be prefixed with a "@" - /// - string PathKey { get; } - /// /// Enumerates a list of files and folders found on the remote location /// @@ -46,11 +41,10 @@ public interface IFolderEnabledBackend : IBackend /// The cancellation token /// The list of files, where paths are scoped to the backend. /// - /// The paths returned should be relative to the backend root, and should not include the root path. - /// For example, if the backend is defined as proto://host/rootpath, then listing files should - /// return paths like "/subfolder/file.txt" and not "/rootpath/subfolder/file.txt". + /// The paths returned should be only the filenames or folder names, not the full path. + /// Folders should end with the operating system's directory separator. /// - IAsyncEnumerable ListAsync(string? path, CancellationToken cancellationToken); + IAsyncEnumerable ListAsync(string? path, CancellationToken cancellationToken); /// /// Gets a single file or folder entry @@ -58,6 +52,10 @@ public interface IFolderEnabledBackend : IBackend /// The path to get /// The cancellation token /// The file or folder entry - Task GetEntryAsync(string path, CancellationToken cancellationToken); + /// + /// The paths returned should be only the filenames or folder names, not the full path. + /// Folders should end with the operating system's directory separator. + /// + Task GetEntryAsync(string path, CancellationToken cancellationToken); } diff --git a/Duplicati/Library/Interface/ISourceProvider.cs b/Duplicati/Library/Interface/ISourceProvider.cs index 11c61d4a4..386680f00 100644 --- a/Duplicati/Library/Interface/ISourceProvider.cs +++ b/Duplicati/Library/Interface/ISourceProvider.cs @@ -34,16 +34,16 @@ namespace Duplicati.Library.Interface; public interface ISourceProvider : IDisposable { /// - /// Returns the key for this provider instance's path + /// Returns the path where this provider is logically mounted /// - string PathKey { get; } + string MountedPath { get; } /// /// Gets the root entries /// /// The cancellation token /// An enumerable of file entries - IAsyncEnumerable Enumerate(CancellationToken cancellationToken); + IAsyncEnumerable Enumerate(CancellationToken cancellationToken); /// /// Gets a specific entry @@ -52,5 +52,5 @@ public interface ISourceProvider : IDisposable /// True if the path is a folder /// The cancellation token /// The file entry - Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken); + Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken); } diff --git a/Duplicati/Library/Interface/ISourceFileEntry.cs b/Duplicati/Library/Interface/ISourceProviderEntry.cs similarity index 96% rename from Duplicati/Library/Interface/ISourceFileEntry.cs rename to Duplicati/Library/Interface/ISourceProviderEntry.cs index 0f41fb690..c93d945e2 100644 --- a/Duplicati/Library/Interface/ISourceFileEntry.cs +++ b/Duplicati/Library/Interface/ISourceProviderEntry.cs @@ -30,9 +30,9 @@ using System.Threading.Tasks; namespace Duplicati.Library.Interface; /// -/// Interface for an instance of a file +/// Interface for an instance of a file or folder /// -public interface ISourceFileEntry +public interface ISourceProviderEntry { /// /// True if the entry represents a folder, false otherwise @@ -122,5 +122,5 @@ public interface ISourceFileEntry /// /// A token that can be used to cancel the operation /// The contents of the folder - IAsyncEnumerable Enumerate(CancellationToken cancellationToken); + IAsyncEnumerable Enumerate(CancellationToken cancellationToken); } diff --git a/Duplicati/Library/Main/Controller.cs b/Duplicati/Library/Main/Controller.cs index 312df9f8f..abd3b311e 100644 --- a/Duplicati/Library/Main/Controller.cs +++ b/Duplicati/Library/Main/Controller.cs @@ -948,15 +948,14 @@ namespace Duplicati.Library.Main string source; try { - // Check if this is not a file source - var scheme = Library.Utility.Utility.GuessScheme(expandedSource) ?? "file"; - if (scheme != "file") + // Check if this is a mounted path + if (expandedSource.StartsWith("@")) { // TODO: If the remote source fails to load, // this will be an enumeration warning, but will result // in the backup being recorded without files from the source - // Eventually, this could lead to retention causing the last backup - // with the data from the source to be deleted + // Eventually, this could lead to retention deletion, + // causing the last backup with the data from the source to be deleted foundAnyPaths = true; sources.Add(expandedSource); continue; diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index f558c6e8a..6417344d0 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -878,11 +878,11 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist } } - public string GetFirstLocalPath() + public string GetFirstPath() { using (var cmd = m_connection.CreateCommand()) { - cmd.CommandText = @$"SELECT ""Path"" FROM ""File"" WHERE ""Path"" NOT LIKE '{Util.RemotePathPrefix}%' ORDER BY LENGTH(""Path"") DESC LIMIT 1"; + cmd.CommandText = @$"SELECT ""Path"" FROM ""File"" ORDER BY LENGTH(""Path"") DESC LIMIT 1"; var v0 = cmd.ExecuteScalar(); if (v0 == null || v0 == DBNull.Value) return null; diff --git a/Duplicati/Library/Main/Operation/Backup/Channels.cs b/Duplicati/Library/Main/Operation/Backup/Channels.cs index b5948e13e..f65c545d7 100644 --- a/Duplicati/Library/Main/Operation/Backup/Channels.cs +++ b/Duplicati/Library/Main/Operation/Backup/Channels.cs @@ -52,7 +52,7 @@ namespace Duplicati.Library.Main.Operation.Backup /// /// When enumerating the source folders, all discovered paths are sent by the to the /// - public readonly IChannel SourcePaths = ChannelManager.CreateChannel(); + public readonly IChannel SourcePaths = ChannelManager.CreateChannel(); /// /// All progress events are communicated over this channel, to ensure that parallel progress is reported as a if it was sequential /// diff --git a/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs b/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs index a181907ff..848610186 100644 --- a/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs @@ -80,7 +80,7 @@ namespace Duplicati.Library.Main.Operation.Backup // The mixin queue is used to store symlinks that should be processed // The symlinks are emitted during the enumeration process when they are found - var mixinqueue = new Queue(); + var mixinqueue = new Queue(); // The enumeration filter is used to determine what paths to // recurse into. If the emit filter only has includes, @@ -98,16 +98,16 @@ namespace Duplicati.Library.Main.Operation.Backup // Shared filter function with bound variables - ValueTask FilterEntry(ISourceFileEntry entry) + ValueTask FilterEntry(ISourceProviderEntry entry) => SourceFileEntryFilter(entry, blacklistPaths, hardlinkPolicy, symlinkPolicy, hardlinkmap, fileAttributeFilter, enumeratefilter, ignorenames, mixinqueue, token); // Prepare the work list - IAsyncEnumerable worklist; + IAsyncEnumerable worklist; // If we have a specific list, use that instead of enumerating the filesystem if (changedfilelist != null && changedfilelist.Any()) { - async IAsyncEnumerable ExpandSources(IEnumerable list) + async IAsyncEnumerable ExpandSources(IEnumerable list) { foreach (var s in list) { @@ -203,7 +203,7 @@ namespace Duplicati.Library.Main.Operation.Backup /// /// The item being tracked /// - public required ISourceFileEntry Item; + public required ISourceProviderEntry Item; /// /// A flag indicating if any items are found in this folder /// @@ -216,7 +216,7 @@ namespace Duplicati.Library.Main.Operation.Backup /// /// The list without empty folders. /// The list with potential empty folders. - private static async IAsyncEnumerable ExcludeEmptyFolders(IAsyncEnumerable source, [EnumeratorCancellation] CancellationToken cancellationToken) + private static async IAsyncEnumerable ExcludeEmptyFolders(IAsyncEnumerable source, [EnumeratorCancellation] CancellationToken cancellationToken) { var pathstack = new Stack(); @@ -276,9 +276,9 @@ namespace Duplicati.Library.Main.Operation.Backup /// The entries to recurse /// The filter to apply /// - private static async IAsyncEnumerable RecurseEntries(IAsyncEnumerable entries, Func> filter, [EnumeratorCancellation] CancellationToken cancellationToken) + private static async IAsyncEnumerable RecurseEntries(IAsyncEnumerable entries, Func> filter, [EnumeratorCancellation] CancellationToken cancellationToken) { - var work = new Stack(); + var work = new Stack(); await foreach (var e in entries.WithCancellation(cancellationToken).ConfigureAwait(false)) if (await filter(e).ConfigureAwait(false)) @@ -317,7 +317,7 @@ namespace Duplicati.Library.Main.Operation.Backup /// The mix in queue. /// The emitfilter. /// The enumeratefilter. - private static async IAsyncEnumerable ExpandWorkList(IAsyncEnumerable worklist, Queue mixinqueue, Library.Utility.IFilter emitfilter, Library.Utility.IFilter enumeratefilter, [EnumeratorCancellation] CancellationToken cancellationToken) + private static async IAsyncEnumerable ExpandWorkList(IAsyncEnumerable worklist, Queue mixinqueue, Library.Utility.IFilter emitfilter, Library.Utility.IFilter enumeratefilter, [EnumeratorCancellation] CancellationToken cancellationToken) { // Process each path, and dequeue the mixins with symlinks as we go await foreach (var s in worklist.WithCancellation(cancellationToken).ConfigureAwait(false)) @@ -345,7 +345,7 @@ namespace Duplicati.Library.Main.Operation.Backup /// The entry to evaluate. /// The blacklist paths. /// True if the path should be returned, false otherwise. - private static bool PreFilterSourceEntry(ISourceFileEntry entry, HashSet blacklistPaths) + private static bool PreFilterSourceEntry(ISourceProviderEntry entry, HashSet blacklistPaths) { // Don't filter meta stuff if (entry.IsMetaEntry) @@ -405,7 +405,7 @@ namespace Duplicati.Library.Main.Operation.Backup /// The ignore names. /// The mixin queue. /// True if the path should be returned, false otherwise. - private static async ValueTask SourceFileEntryFilter(ISourceFileEntry entry, HashSet blacklistPaths, Options.HardlinkStrategy hardlinkPolicy, Options.SymlinkStrategy symlinkPolicy, Dictionary hardlinkmap, FileAttributes fileAttributeFilter, Duplicati.Library.Utility.IFilter enumeratefilter, string[]? ignorenames, Queue mixinqueue, CancellationToken cancellationToken) + private static async ValueTask SourceFileEntryFilter(ISourceProviderEntry entry, HashSet blacklistPaths, Options.HardlinkStrategy hardlinkPolicy, Options.SymlinkStrategy symlinkPolicy, Dictionary hardlinkmap, FileAttributes fileAttributeFilter, Duplicati.Library.Utility.IFilter enumeratefilter, string[]? ignorenames, Queue mixinqueue, CancellationToken cancellationToken) { // Do the course pre-filtering first if (!PreFilterSourceEntry(entry, blacklistPaths)) diff --git a/Duplicati/Library/Main/Operation/Backup/MetadataGenerator.cs b/Duplicati/Library/Main/Operation/Backup/MetadataGenerator.cs index 02c6f75c2..6be52d47c 100644 --- a/Duplicati/Library/Main/Operation/Backup/MetadataGenerator.cs +++ b/Duplicati/Library/Main/Operation/Backup/MetadataGenerator.cs @@ -32,7 +32,7 @@ namespace Duplicati.Library.Main.Operation.Backup { private static readonly string METALOGTAG = Logging.Log.LogTagFromType(typeof(MetadataGenerator)) + ".Metadata"; - public static Dictionary GenerateMetadata(ISourceFileEntry entry, System.IO.FileAttributes attributes, Options options) + public static Dictionary GenerateMetadata(ISourceProviderEntry entry, System.IO.FileAttributes attributes, Options options) { try { diff --git a/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs b/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs index f8761ed56..2ae10faa1 100644 --- a/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs @@ -40,7 +40,7 @@ namespace Duplicati.Library.Main.Operation.Backup public class FileEntry { // From input - public ISourceFileEntry Entry; + public ISourceProviderEntry Entry; // Split public long PathPrefixID; @@ -180,7 +180,7 @@ namespace Duplicati.Library.Main.Operation.Backup /// Processes the metadata for the given path. /// /// True if the path should be submitted to more analysis, false if there is nothing else to do - private static async Task ProcessMetadata(ISourceFileEntry entry, FileAttributes attributes, DateTime lastwrite, Options options, IMetahash emptymetadata, BackupDatabase database, IWriteChannel streamblockchannel) + private static async Task ProcessMetadata(ISourceProviderEntry entry, FileAttributes attributes, DateTime lastwrite, Options options, IMetahash emptymetadata, BackupDatabase database, IWriteChannel streamblockchannel) { if (entry.IsSymlink) { diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index de97b7449..a9786b2d8 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -37,6 +37,8 @@ using System.Data; using Duplicati.Library.DynamicLoader; using Duplicati.Library.SourceProvider; using Duplicati.Library.Snapshots.USN; +using System.Text.RegularExpressions; +using System.IO; namespace Duplicati.Library.Main.Operation { @@ -123,7 +125,7 @@ namespace Duplicati.Library.Main.Operation private static List GetSourceProviders(IEnumerable sources, Options options) { // Group the sources by their type, so we can combine all snapshot paths into a single snapshot - var sourceTypes = sources.GroupBy(x => Library.Utility.Utility.GuessScheme(x) ?? "file", StringComparer.OrdinalIgnoreCase); + var sourceTypes = sources.GroupBy(x => x.StartsWith("@") ? "@" : Library.Utility.Utility.GuessScheme(x) ?? "file", StringComparer.OrdinalIgnoreCase); // To avoid leaking snapshot instances, we create all instances first and then dispose them if an exception occurs // The number of instances is expected to be low, so the memory overhead is acceptable @@ -136,13 +138,30 @@ namespace Duplicati.Library.Main.Operation results.Add(new LocalFileSource(GetFileSnapshotService(entry, options))); else if ("vss".Equals(entry.Key, StringComparison.OrdinalIgnoreCase) || "lvm".Equals(entry.Key, StringComparison.OrdinalIgnoreCase)) results.Add(new LocalFileSource(SnapshotUtility.CreateSnapshot(entry, options.RawOptions, options.SymlinkPolicy == Options.SymlinkStrategy.Follow))); - else + else if ("@".Equals(entry.Key, StringComparison.OrdinalIgnoreCase)) { - // Assume it is a remote source foreach (var url in entry) - results.Add(SourceProviderLoader.GetSourceProvider(url, options.RawOptions) - ?? throw new UserInformationException(string.Format("The source \"{0}\" is not supported", url), "SourceNotSupported")); + { + var m = Regex.Match(url, @"^@(?[^|]+)\|(?.+)$", RegexOptions.IgnoreCase); + if (m.Success) + { + var mountpoint = m.Groups["mountpoint"].Value; + + if (mountpoint.Any(x => Path.GetInvalidPathChars().Contains(x))) + throw new UserInformationException(string.Format("The mountpoint \"{0}\" contains invalid characters", mountpoint), "InvalidMountpoint"); + if (!Path.IsPathRooted(mountpoint)) + throw new UserInformationException(string.Format("The mountpoint \"{0}\" is not a valid rooted mountpoint", mountpoint), "InvalidMountpoint"); + + var backendurl = m.Groups["url"].Value; + results.Add(SourceProviderLoader.GetSourceProvider(backendurl, Path.GetFullPath(mountpoint), options.RawOptions) + ?? throw new UserInformationException(string.Format("The source \"{0}\" is not supported", backendurl), "SourceNotSupported")); + } + else + throw new UserInformationException(string.Format("The source \"{0}\" is not a supported format", url), "SourceFormatNotSupported"); + } } + else + throw new UserInformationException(string.Format("The source type \"{0}\" is not supported", entry.Key), "SourceTypeNotSupported"); } } catch @@ -269,7 +288,7 @@ namespace Duplicati.Library.Main.Operation Utility.UpdateOptionsFromDb(database, options); Utility.VerifyOptionsAndUpdateDatabase(database, options); - var probe_path = database.GetFirstLocalPath(); + var probe_path = database.GetFirstPath(); if (probe_path != null && Util.GuessDirSeparator(probe_path) != Util.DirectorySeparatorString) throw new UserInformationException(string.Format("The backup contains files that belong to another operating system. Proceeding with a backup would cause the database to contain paths from two different operation systems, which is not supported. To proceed without losing remote data, delete all filesets and make sure the --{0} option is set, then run the backup again to re-use the existing data on the remote store.", "no-auto-compact"), "CrossOsDatabaseReuseNotSupported"); diff --git a/Duplicati/Library/Snapshots/ISnapshotService.cs b/Duplicati/Library/Snapshots/ISnapshotService.cs index 72542e465..bebf5c92a 100644 --- a/Duplicati/Library/Snapshots/ISnapshotService.cs +++ b/Duplicati/Library/Snapshots/ISnapshotService.cs @@ -44,20 +44,20 @@ namespace Duplicati.Library.Snapshots /// Enumerates the root source files and folders /// /// The source files and folders - IEnumerable EnumerateFilesystemEntries(); + IEnumerable EnumerateFilesystemEntries(); /// /// Enumerates the files and folders in a given folder /// /// The parent folder to enumerate /// The files and folders in the parent folder - IEnumerable EnumerateFilesystemEntries(ISourceFileEntry parent); + IEnumerable EnumerateFilesystemEntries(ISourceProviderEntry parent); /// /// Gets a filesystem entry for a given path /// /// The path to get /// True if the path is a folder /// The filesystem entry - ISourceFileEntry? GetFilesystemEntry(string path, bool isFolder); + ISourceProviderEntry? GetFilesystemEntry(string path, bool isFolder); /// /// Gets the last write time of a given file in UTC diff --git a/Duplicati/Library/Snapshots/LinuxSnapshot.cs b/Duplicati/Library/Snapshots/LinuxSnapshot.cs index 05cf884d7..83b786ddc 100644 --- a/Duplicati/Library/Snapshots/LinuxSnapshot.cs +++ b/Duplicati/Library/Snapshots/LinuxSnapshot.cs @@ -116,7 +116,7 @@ namespace Duplicati.Library.Snapshots /// Enumerates the root source files and folders /// /// The source files and folders - public override IEnumerable EnumerateFilesystemEntries() + public override IEnumerable EnumerateFilesystemEntries() { foreach (var sourceEntry in SourceEntries) { diff --git a/Duplicati/Library/Snapshots/NoSnapshotLinux.cs b/Duplicati/Library/Snapshots/NoSnapshotLinux.cs index 6aaa1ecd6..de21cec98 100644 --- a/Duplicati/Library/Snapshots/NoSnapshotLinux.cs +++ b/Duplicati/Library/Snapshots/NoSnapshotLinux.cs @@ -93,7 +93,7 @@ namespace Duplicati.Library.Snapshots /// Enumerates the root source files and folders /// /// The source files and folders - public override IEnumerable EnumerateFilesystemEntries() + public override IEnumerable EnumerateFilesystemEntries() { foreach (var folder in m_sources) { diff --git a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs index 029ae3791..aa8c9a5ac 100644 --- a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs +++ b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs @@ -84,7 +84,7 @@ namespace Duplicati.Library.Snapshots /// Enumerates the root source files and folders /// /// The source files and folders - public override IEnumerable EnumerateFilesystemEntries() + public override IEnumerable EnumerateFilesystemEntries() { foreach (var folder in m_sources.Select(SystemIOWindows.RemoveExtendedDevicePathPrefix)) { diff --git a/Duplicati/Library/Snapshots/SnapshotBase.cs b/Duplicati/Library/Snapshots/SnapshotBase.cs index 1d8f5a253..0bacef4af 100644 --- a/Duplicati/Library/Snapshots/SnapshotBase.cs +++ b/Duplicati/Library/Snapshots/SnapshotBase.cs @@ -64,7 +64,7 @@ namespace Duplicati.Library.Snapshots /// Enuemrates the root source files and folders /// /// The source files and folders - public abstract IEnumerable EnumerateFilesystemEntries(); + public abstract IEnumerable EnumerateFilesystemEntries(); /// /// Gets a filesystem entry for a given path @@ -72,7 +72,7 @@ namespace Duplicati.Library.Snapshots /// The path to get the entry for /// A flag indicating if the path is a folder /// The filesystem entry - public virtual ISourceFileEntry GetFilesystemEntry(string path, bool isFolder) + public virtual ISourceProviderEntry GetFilesystemEntry(string path, bool isFolder) { if (isFolder && !DirectoryExists(path)) return null; @@ -87,7 +87,7 @@ namespace Duplicati.Library.Snapshots /// /// Source to enumerate /// The files and folders in the given folder - public virtual IEnumerable EnumerateFilesystemEntries(ISourceFileEntry source) + public virtual IEnumerable EnumerateFilesystemEntries(ISourceProviderEntry source) { if (source.IsFolder) { diff --git a/Duplicati/Library/Snapshots/SnapshotSourceFileEntry.cs b/Duplicati/Library/Snapshots/SnapshotSourceFileEntry.cs index 932963611..7fc9cae22 100644 --- a/Duplicati/Library/Snapshots/SnapshotSourceFileEntry.cs +++ b/Duplicati/Library/Snapshots/SnapshotSourceFileEntry.cs @@ -39,7 +39,7 @@ namespace Duplicati.Library.Snapshots; /// The path of the file /// True if the entry is a folder, false otherwise /// True if the entry is the root entry, false otherwise -public class SnapshotSourceFileEntry(ISnapshotService service, string path, bool isFolder, bool isRoot) : ISourceFileEntry +public class SnapshotSourceFileEntry(ISnapshotService service, string path, bool isFolder, bool isRoot) : ISourceProviderEntry { /// /// The symlink target @@ -137,7 +137,7 @@ public class SnapshotSourceFileEntry(ISnapshotService service, string path, bool /// public Task OpenMetadataRead(CancellationToken cancellationToken) => Task.FromResult(new MemoryStream(System.Text.Json.JsonSerializer.SerializeToUtf8Bytes(MinorMetadata))); /// - public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) => service.EnumerateFilesystemEntries(this).ToAsyncEnumerable(); + public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) => service.EnumerateFilesystemEntries(this).ToAsyncEnumerable(); /// public Task FileExists(string filename, CancellationToken cancellationToken) => Task.FromResult(service.FileExists(System.IO.Path.Combine(path, filename))); } \ No newline at end of file diff --git a/Duplicati/Library/Snapshots/USN/UsnJournalService.cs b/Duplicati/Library/Snapshots/USN/UsnJournalService.cs index d44bad907..cc5963199 100644 --- a/Duplicati/Library/Snapshots/USN/UsnJournalService.cs +++ b/Duplicati/Library/Snapshots/USN/UsnJournalService.cs @@ -229,7 +229,7 @@ namespace Duplicati.Library.Snapshots.USN /// /// Cancellation token /// Filtered sources - public async IAsyncEnumerable GetFullScanSources([EnumeratorCancellation] CancellationToken token) + public async IAsyncEnumerable GetFullScanSources([EnumeratorCancellation] CancellationToken token) { // Make the method async enumerable await Task.CompletedTask; @@ -276,7 +276,7 @@ namespace Duplicati.Library.Snapshots.USN /// Filter callback to exclude filtered items /// Cancellation token /// Filtered sources - public async IAsyncEnumerable GetModifiedSources(Func> filter, [EnumeratorCancellation] CancellationToken token) + public async IAsyncEnumerable GetModifiedSources(Func> filter, [EnumeratorCancellation] CancellationToken token) { foreach (var volumeData in m_volumeDataDict.Values.Where(x => !x.IsFullScan)) { @@ -338,10 +338,10 @@ namespace Duplicati.Library.Snapshots.USN /// Cache of included and excluded files / folders /// Cancellation token /// Filtered files - private static async IAsyncEnumerable FilterExcludedFiles( + private static async IAsyncEnumerable FilterExcludedFiles( IEnumerable? files, ISnapshotService snapshot, - Func> filter, + Func> filter, IDictionary cache, [EnumeratorCancellation] CancellationToken token) { @@ -350,7 +350,7 @@ namespace Duplicati.Library.Snapshots.USN foreach (var filePath in files) { - ISourceFileEntry? file; + ISourceProviderEntry? file; try { file = snapshot.GetFilesystemEntry(filePath, false); @@ -392,10 +392,10 @@ namespace Duplicati.Library.Snapshots.USN /// Cache of excluded folders /// Cancellation token /// Filtered folders - private static async IAsyncEnumerable FilterExcludedFolders( + private static async IAsyncEnumerable FilterExcludedFolders( IEnumerable? folders, ISnapshotService snapshot, - Func> filter, + Func> filter, IDictionary cache, [EnumeratorCancellation] CancellationToken token) { @@ -404,7 +404,7 @@ namespace Duplicati.Library.Snapshots.USN foreach (var folderPath in folders) { - ISourceFileEntry? folder; + ISourceProviderEntry? folder; try { folder = snapshot.GetFilesystemEntry(folderPath, true); @@ -431,14 +431,14 @@ namespace Duplicati.Library.Snapshots.USN /// Cache of excluded folders (optional) /// True if excluded, false otherwise private static async ValueTask IsFolderOrAncestorsExcluded( - ISourceFileEntry inputFolder, + ISourceProviderEntry inputFolder, ISnapshotService snapshot, - Func> filter, + Func> filter, IDictionary cache, CancellationToken token) { List? parents = null; - ISourceFileEntry? folder = inputFolder; + ISourceProviderEntry? folder = inputFolder; while (folder != null) { diff --git a/Duplicati/Library/Snapshots/WindowsSnapshot.cs b/Duplicati/Library/Snapshots/WindowsSnapshot.cs index 1c1e70416..bbe4154c3 100644 --- a/Duplicati/Library/Snapshots/WindowsSnapshot.cs +++ b/Duplicati/Library/Snapshots/WindowsSnapshot.cs @@ -126,7 +126,7 @@ namespace Duplicati.Library.Snapshots /// Enumerates the root source files and folders /// /// The source files and folders - public override IEnumerable EnumerateFilesystemEntries() + public override IEnumerable EnumerateFilesystemEntries() { foreach (var folder in _sourceEntries) { diff --git a/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs b/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs index 535ec7d5d..95fe59f91 100644 --- a/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs +++ b/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs @@ -20,6 +20,7 @@ // DEALINGS IN THE SOFTWARE. using System.IO.Pipelines; +using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using Duplicati.Library.Utility; @@ -36,7 +37,7 @@ namespace Duplicati.Library.SourceProvider; /// The last modification time of the entry /// The size of the entry public class BackendSourceFileEntry(BackendSourceProvider parent, string path, bool isFolder, bool isMetaEntry, DateTime createdUtc, DateTime lastModificationUtc, long size) - : ISourceFileEntry + : ISourceProviderEntry { /// /// The log tag for this instance @@ -59,7 +60,7 @@ public class BackendSourceFileEntry(BackendSourceProvider parent, string path, b public DateTime LastModificationUtc => lastModificationUtc; /// - public string Path => ConcatPaths(parent.PathKey, NormalizePath(path)); + public string Path => SystemIO.IO_OS.PathCombine(parent?.MountedPath, NormalizePathToLocalSystem(path)); /// public long Size => size; @@ -90,18 +91,32 @@ public class BackendSourceFileEntry(BackendSourceProvider parent, string path, b public string? HardlinkTargetId => null; /// - public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) + public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) { if (!isFolder) throw new InvalidOperationException("Enumerate can only be called on folders"); return parent.WrappedBackend.ListAsync(path, cancellationToken) + // Remove the current and parent folder entries + .Where(x => !string.IsNullOrWhiteSpace(x.Name) && x.Name != "." && x.Name != "..") + // Remove sub-folder entries + .Where(x => !x.Name[0..^1].Contains('\\') && !x.Name[0..^1].Contains('/')) + // Convert to source file entries .Select(x => { - if (x is BackendSourceFileEntry) - return x; + var localPath = SystemIO.IO_OS.PathCombine(path, NormalizePathToLocalSystem(x.Name)); + if (x.IsFolder) + localPath = Util.AppendDirSeparator(localPath); - return new BackendSourceFileEntry(parent, x.Path, x.IsFolder, false, x.CreatedUtc, x.LastModificationUtc, x.Size); + return new BackendSourceFileEntry( + parent, + localPath, + x.IsFolder, + false, + x.Created, + x.LastModification, + x.Size + ); }); } @@ -109,7 +124,7 @@ public class BackendSourceFileEntry(BackendSourceProvider parent, string path, b public async Task FileExists(string path, CancellationToken cancellationToken) { if (!isFolder) - throw new InvalidOperationException("FileExists can only be called on folders"); + throw new InvalidOperationException("FileExists cannot be called on folders"); try { @@ -235,39 +250,28 @@ public class BackendSourceFileEntry(BackendSourceProvider parent, string path, b } /// - /// Normalizes the path, turning backslashes into forward slashes + /// Normalizes the path, turning backslashes into forward slashes, + /// or vice versa, depending on the platform /// /// The path to normalize /// The normalized path - public static string NormalizePath(string path) - { - if (System.IO.Path.DirectorySeparatorChar == '/') - return path; + public static string NormalizePathToLocalSystem(string path) + => NormalizePathTo(path, System.IO.Path.DirectorySeparatorChar); + /// + /// Normalizes the path, turning backslashes into forward slashes, + /// or vice versa, depending on the platform + /// + /// The path to normalize + /// The normalized path + public static string NormalizePathTo(string path, char separator) + { if (string.IsNullOrEmpty(path)) return path; - return path.Replace('\\', '/'); - } - - /// - /// Concatenates two paths - /// - /// The first path - /// The second path - /// The concatenated path - public static string ConcatPaths(string path1, string path2) - { - if (string.IsNullOrEmpty(path1)) - return path2; - - if (string.IsNullOrEmpty(path2)) - return path1; - - if (path1.EndsWith('/') || path1.EndsWith('\\')) - return path1 + path2; - - return path1 + "/" + path2; + return path + .Replace('/', separator) + .Replace('\\', separator); } /// @@ -278,6 +282,6 @@ public class BackendSourceFileEntry(BackendSourceProvider parent, string path, b /// The prefix to add to the path /// The new backend source entry public static BackendSourceFileEntry FromFileEntry(BackendSourceProvider parent, string prefix, IFileEntry entry) - => new BackendSourceFileEntry(parent, ConcatPaths(prefix, entry.Name), entry.IsFolder, false, entry.Created, entry.LastModification, entry.Size); + => new BackendSourceFileEntry(parent, SystemIO.IO_OS.PathCombine(prefix, entry.Name), entry.IsFolder, false, entry.Created, entry.LastModification, entry.Size); } diff --git a/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs index 3943fb1b0..2ae3d9a71 100644 --- a/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs +++ b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs @@ -19,8 +19,6 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System.Runtime.CompilerServices; -using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; namespace Duplicati.Library.SourceProvider; @@ -29,7 +27,8 @@ namespace Duplicati.Library.SourceProvider; /// A source provider that wraps a backend /// /// The backend to wrap -public class BackendSourceProvider(IFolderEnabledBackend backend) : ISourceProvider, ISourceProviderModule +/// The path to mount the backend +public class BackendSourceProvider(IFolderEnabledBackend backend, string mountedPath) : ISourceProvider, ISourceProviderModule { /// /// The wrapped backend @@ -37,9 +36,9 @@ public class BackendSourceProvider(IFolderEnabledBackend backend) : ISourceProvi public IFolderEnabledBackend WrappedBackend => backend; /// - /// The path key unique for the destination + /// The path where the provider is logically mounted /// - public string PathKey => backend.PathKey; + public string MountedPath => mountedPath; /// public string Key => backend.ProtocolKey; @@ -51,12 +50,15 @@ public class BackendSourceProvider(IFolderEnabledBackend backend) : ISourceProvi public IList SupportedCommands => backend.SupportedCommands; /// - public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) + public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) => new[] { new BackendSourceFileEntry(this, "", true, true, new DateTime(0), new DateTime(0), 0) }.ToAsyncEnumerable(); /// - public Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) - => backend.GetEntryAsync(path, cancellationToken); + public async Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) + { + var entry = await backend.GetEntryAsync(path, cancellationToken).ConfigureAwait(false); + return entry == null ? null : BackendSourceFileEntry.FromFileEntry(this, path, entry); + } /// public void Dispose() @@ -64,40 +66,4 @@ public class BackendSourceProvider(IFolderEnabledBackend backend) : ISourceProvi WrappedBackend.Dispose(); GC.SuppressFinalize(this); } - - /// - /// Helper function to use a List function that returns IFileEntry and convert it to ISourceFileEntry - /// - /// The prefix to hide from the caller - /// The logical path to list - /// The function that returns the IFileEntry items - /// The cancellation token - /// The list of source file entries - public async IAsyncEnumerable ListFromFileEntryAsync(string prefix, string path, Func> listFunction, [EnumeratorCancellation] CancellationToken cancellationToken) - { - if (!string.IsNullOrWhiteSpace(prefix) && !prefix.EndsWith("/", StringComparison.Ordinal)) - prefix += "/"; - - // Logic here is that the prefix is invisible to the caller, so we add it to the prefix to filter the results - // and remove it from the name to make the prefix invisible. - // The input path is preserved so the output looks like a "full path" to the caller. - - var filterPath = prefix + path; - if (filterPath != "" && !filterPath.EndsWith("/", StringComparison.Ordinal)) - filterPath += "/"; - - await foreach (var f in listFunction(filterPath, cancellationToken).ConfigureAwait(false)) - { - if (!f.Name.StartsWith(filterPath, StringComparison.Ordinal)) - continue; - - ((FileEntry)f).Name = f.Name.Substring(filterPath.Length); - - // Skip self and sub-folder items - if (f.Name.Length == 0 || f.Name[0..^1].Contains("/")) - continue; - - yield return BackendSourceFileEntry.FromFileEntry(this, path, f); - } - } } diff --git a/Duplicati/Library/SourceProvider/Builtin/Combiner.cs b/Duplicati/Library/SourceProvider/Builtin/Combiner.cs index 7f027847f..ed886a028 100644 --- a/Duplicati/Library/SourceProvider/Builtin/Combiner.cs +++ b/Duplicati/Library/SourceProvider/Builtin/Combiner.cs @@ -30,7 +30,7 @@ namespace Duplicati.Library.SourceProvider; public class Combiner(IEnumerable providers) : ISourceProvider { /// - public string PathKey => string.Empty; + public string MountedPath => string.Empty; /// /// The providers to combine @@ -38,7 +38,7 @@ public class Combiner(IEnumerable providers) : ISourceProvider private readonly List providers = providers.SelectMany(x => x is Combiner c ? c.providers.AsEnumerable() : [x]).ToList(); /// - public async IAsyncEnumerable Enumerate([EnumeratorCancellation] CancellationToken cancellationToken) + public async IAsyncEnumerable Enumerate([EnumeratorCancellation] CancellationToken cancellationToken) { foreach (var provider in providers) { @@ -59,16 +59,16 @@ public class Combiner(IEnumerable providers) : ISourceProvider /// True if the path is a folder /// The cancellation token /// The filesystem entry - public async Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) + public async Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) { foreach (var provider in providers) - if (!string.IsNullOrWhiteSpace(PathKey) && provider.PathKey.StartsWith(path)) + if (!string.IsNullOrWhiteSpace(MountedPath) && provider.MountedPath.StartsWith(path)) { - var subpath = provider.PathKey.Substring(path.Length); + var subpath = provider.MountedPath.Substring(path.Length); return await provider.GetEntry(subpath, isFolder, cancellationToken).ConfigureAwait(false); } - foreach (var provider in providers.Where(x => string.IsNullOrWhiteSpace(x.PathKey))) + foreach (var provider in providers.Where(x => string.IsNullOrWhiteSpace(x.MountedPath))) { var res = await provider.GetEntry(path, isFolder, cancellationToken).ConfigureAwait(false); if (res != null) diff --git a/Duplicati/Library/SourceProvider/Builtin/LocalFileSource.cs b/Duplicati/Library/SourceProvider/Builtin/LocalFileSource.cs index 9c75ca3bb..d4ae9a336 100644 --- a/Duplicati/Library/SourceProvider/Builtin/LocalFileSource.cs +++ b/Duplicati/Library/SourceProvider/Builtin/LocalFileSource.cs @@ -31,10 +31,10 @@ namespace Duplicati.Library.SourceProvider; public class LocalFileSource(ISnapshotService snapshotService) : ISourceProvider { /// - public string PathKey => string.Empty; + public string MountedPath => string.Empty; /// - public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) + public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) => snapshotService.EnumerateFilesystemEntries().ToAsyncEnumerable(); /// @@ -43,7 +43,7 @@ public class LocalFileSource(ISnapshotService snapshotService) : ISourceProvider public ISnapshotService SnapshotService => snapshotService; /// - public Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) + public Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) => Task.FromResult(snapshotService.GetFilesystemEntry(path, isFolder)); /// diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index 2bffe665c..d3ce83d50 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -1557,6 +1557,9 @@ namespace Duplicati.Library.Utility /// The guessed scheme, or null if no scheme was found public static string? GuessScheme(string url) { + if (string.IsNullOrWhiteSpace(url)) + return null; + var idx = url.IndexOf("://"); if (idx < 0 && idx < 15 && idx + "://".Length < url.Length) return null; From de622b29d07270819911184262d498569f07fe9a Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 21 Feb 2025 16:12:40 +0100 Subject: [PATCH 23/28] Added source tool to launch config --- .vscode/launch.json | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index ba69c97a3..e114a9d04 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -114,6 +114,13 @@ "stopAtEntry": false, "console": "internalConsole" }, - + { + "name": "Launch SourceTool executable", + "type": "coreclr", + "request": "launch", + "preLaunchTask": "build", + "program": "${workspaceFolder}/Executables/net8/Duplicati.CommandLine.SourceTool/bin/Debug/net8.0/Duplicati.CommandLine.SourceTool", + "args": [] + } ] } \ No newline at end of file From 4e73be18061a652e2b87f02722667fcfc548db8b Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 21 Feb 2025 16:25:18 +0100 Subject: [PATCH 24/28] Fixed path conversion on Windows --- Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs | 8 +++++++- Duplicati/Library/Backend/S3/S3Backend.cs | 8 ++------ .../SourceProvider/Builtin/BackendSourceFileEntry.cs | 2 +- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs index db797a927..66d27e56b 100644 --- a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs +++ b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs @@ -236,7 +236,13 @@ namespace Duplicati.Library.Backend /// public IAsyncEnumerable ListAsync(string path, CancellationToken cancellationToken) - => m_s3Client.ListBucketAsync(m_bucket, path, false, cancellationToken); + { + var filterPath = GetFullKey(path); + if (!string.IsNullOrWhiteSpace(filterPath)) + filterPath = Util.AppendDirSeparator(filterPath, "/"); + + return m_s3Client.ListBucketAsync(m_bucket, filterPath, true, cancellationToken); + } /// diff --git a/Duplicati/Library/Backend/S3/S3Backend.cs b/Duplicati/Library/Backend/S3/S3Backend.cs index 2136eedbc..91877df04 100644 --- a/Duplicati/Library/Backend/S3/S3Backend.cs +++ b/Duplicati/Library/Backend/S3/S3Backend.cs @@ -410,13 +410,9 @@ namespace Duplicati.Library.Backend /// public IAsyncEnumerable ListAsync(string path, CancellationToken cancellationToken) { - var prefix = m_prefix; - if (!string.IsNullOrWhiteSpace(prefix)) - prefix = Util.AppendDirSeparator(prefix); - - var filterPath = prefix + path; + var filterPath = GetFullKey(path); if (!string.IsNullOrWhiteSpace(filterPath)) - filterPath = Util.AppendDirSeparator(filterPath); + filterPath = Util.AppendDirSeparator(filterPath, "/"); return s3Client.ListBucketAsync(m_bucket, filterPath, m_recurseLists, cancellationToken); } diff --git a/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs b/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs index 95fe59f91..93bea7d8c 100644 --- a/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs +++ b/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs @@ -96,7 +96,7 @@ public class BackendSourceFileEntry(BackendSourceProvider parent, string path, b if (!isFolder) throw new InvalidOperationException("Enumerate can only be called on folders"); - return parent.WrappedBackend.ListAsync(path, cancellationToken) + return parent.WrappedBackend.ListAsync(NormalizePathTo(path, '/'), cancellationToken) // Remove the current and parent folder entries .Where(x => !string.IsNullOrWhiteSpace(x.Name) && x.Name != "." && x.Name != "..") // Remove sub-folder entries From 26bbe6a5778561bbc37311316b836a7c8d9fbd8d Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 24 Feb 2025 12:50:58 +0100 Subject: [PATCH 25/28] Added setup step for source providers. This enables a fail-early approach, where the remote source loads the initial results. By storing the `IAsyncEnumerable` it is possible to not perform any additional remote calls, even with the initial probing call. This makes it possible to detect the absence of sources, so the backup can be aborted if sources are missing. --- .../SourceTool/Commands/Download.cs | 2 +- .../CommandLine/SourceTool/Commands/List.cs | 2 +- .../CommandLine/SourceTool/GetProvider.cs | 4 +- .../DynamicLoader/SourceProviderLoader.cs | 47 ++++++-- .../Library/Interface/ISourceProvider.cs | 7 ++ .../Library/Main/Operation/BackupHandler.cs | 37 ++++-- .../Main/Operation/TestFilterHandler.cs | 2 +- Duplicati/Library/Main/Options.cs | 4 +- .../Builtin/BackendSourceFileEntry.cs | 108 ++++++++++++++---- .../Builtin/BackendSourceProvider.cs | 32 +++++- .../SourceProvider/Builtin/Combiner.cs | 4 + .../SourceProvider/Builtin/LocalFileSource.cs | 3 + Duplicati/Library/Utility/Utility.cs | 31 +++++ 13 files changed, 231 insertions(+), 52 deletions(-) diff --git a/Duplicati/CommandLine/SourceTool/Commands/Download.cs b/Duplicati/CommandLine/SourceTool/Commands/Download.cs index 295545ccd..0297a2ae3 100644 --- a/Duplicati/CommandLine/SourceTool/Commands/Download.cs +++ b/Duplicati/CommandLine/SourceTool/Commands/Download.cs @@ -47,7 +47,7 @@ public static class Download { var token = new CancellationTokenSource().Token; - using var source = Common.GetProvider(url); + using var source = await Common.GetProvider(url); string localPath(string path) { diff --git a/Duplicati/CommandLine/SourceTool/Commands/List.cs b/Duplicati/CommandLine/SourceTool/Commands/List.cs index fd8cb6c25..f5ceca3fb 100644 --- a/Duplicati/CommandLine/SourceTool/Commands/List.cs +++ b/Duplicati/CommandLine/SourceTool/Commands/List.cs @@ -46,7 +46,7 @@ public static class List var folders = 0L; var files = 0L; - using var source = Common.GetProvider(url); + using var source = await Common.GetProvider(url); await Common.Visit(source, maxdepth, (entry, level) => { if (entry.IsFolder) diff --git a/Duplicati/CommandLine/SourceTool/GetProvider.cs b/Duplicati/CommandLine/SourceTool/GetProvider.cs index 1f8b30e01..f4637e398 100644 --- a/Duplicati/CommandLine/SourceTool/GetProvider.cs +++ b/Duplicati/CommandLine/SourceTool/GetProvider.cs @@ -33,7 +33,7 @@ public static partial class Common /// /// The URL to get the provider for /// The source provider - public static ISourceProvider GetProvider(string url) => - SourceProviderLoader.GetSourceProvider(url, "", []) + public static async Task GetProvider(string url) => + (await SourceProviderLoader.GetSourceProvider(url, "", [], CancellationToken.None)) ?? throw new UserInformationException(string.Format("The source \"{0}\" is not supported", url), "SourceNotSupported"); } \ No newline at end of file diff --git a/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs b/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs index b5e7819a6..979721398 100644 --- a/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs +++ b/Duplicati/Library/DynamicLoader/SourceProviderLoader.cs @@ -26,6 +26,8 @@ using System.Linq; using System.Runtime.ExceptionServices; using Duplicati.Library.SourceProviders; using Duplicati.Library.SourceProvider; +using System.Threading.Tasks; +using System.Threading; namespace Duplicati.Library.DynamicLoader { @@ -154,7 +156,16 @@ namespace Duplicati.Library.DynamicLoader if (string.IsNullOrEmpty(url)) throw new ArgumentNullException(nameof(url)); - return _SourceProviderLoader.GetSupportedCommands(url); + var commands = _SourceProviderLoader.GetSupportedCommands(url); + if (commands != null) + return commands; + + var backend = BackendLoader.GetBackend(url, []); + if (backend is IFolderEnabledBackend folderBackend) + commands = folderBackend.SupportedCommands.AsReadOnly(); + backend?.Dispose(); + + return commands; } /// @@ -163,21 +174,35 @@ namespace Duplicati.Library.DynamicLoader /// The url to create the instance for /// The mount point to use /// The options to pass to the instance constructor + /// The cancellation token /// The instanciated SourceProvider or null if the url is not supported - public static ISourceProviderModule GetSourceProvider(string url, string mountPoint, Dictionary options) + public static async Task GetSourceProvider(string url, string mountPoint, Dictionary options, CancellationToken cancellationToken) { // Source providers are preferred over backends var provider = _SourceProviderLoader.GetSourceProvider(url, mountPoint, options); - if (provider != null) + if (provider == null) + { + // See if there is a backend that can also be a source + var backend = BackendLoader.GetBackend(url, options); + if (backend is IFolderEnabledBackend folderBackend) + provider = new BackendSourceProvider(folderBackend, mountPoint); + else + backend?.Dispose(); + } + + if (provider == null) + return null; + + try + { + await provider.Initialize(cancellationToken).ConfigureAwait(false); return provider; - - // See if there is a backend that can also be a source - var backend = BackendLoader.GetBackend(url, options); - if (backend is IFolderEnabledBackend folderBackend) - return new BackendSourceProvider(folderBackend, mountPoint); - - backend?.Dispose(); - return null; + } + catch + { + provider.Dispose(); + throw; + } } #endregion diff --git a/Duplicati/Library/Interface/ISourceProvider.cs b/Duplicati/Library/Interface/ISourceProvider.cs index 386680f00..6ac0b9ea0 100644 --- a/Duplicati/Library/Interface/ISourceProvider.cs +++ b/Duplicati/Library/Interface/ISourceProvider.cs @@ -38,6 +38,13 @@ public interface ISourceProvider : IDisposable /// string MountedPath { get; } + /// + /// Initializes the provider, if needed + /// + /// The cancellation token + /// An awaitable task + Task Initialize(CancellationToken cancellationToken); + /// /// Gets the root entries /// diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index a9786b2d8..93a8e3222 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -85,8 +85,8 @@ namespace Duplicati.Library.Main.Operation /// The sources to get providers for /// The options to use /// The source providers - public static ISourceProvider GetSourceProvider(IEnumerable sources, Options options) - => Combiner.Combine(GetSourceProviders(sources, options)); + public static async Task GetSourceProvider(IEnumerable sources, Options options, CancellationToken cancellationToken) + => Combiner.Combine(await GetSourceProviders(sources, options, cancellationToken)); /// /// Gets a snapshot service for the given sources @@ -122,7 +122,7 @@ namespace Duplicati.Library.Main.Operation /// The filter to use /// The last fileset id /// The source providers - private static List GetSourceProviders(IEnumerable sources, Options options) + private static async Task> GetSourceProviders(IEnumerable sources, Options options, CancellationToken cancellationToken) { // Group the sources by their type, so we can combine all snapshot paths into a single snapshot var sourceTypes = sources.GroupBy(x => x.StartsWith("@") ? "@" : Library.Utility.Utility.GuessScheme(x) ?? "file", StringComparer.OrdinalIgnoreCase); @@ -142,6 +142,7 @@ namespace Duplicati.Library.Main.Operation { foreach (var url in entry) { + var sanitizedUrl = Library.Utility.Utility.GetUrlWithoutCredentials(url); var m = Regex.Match(url, @"^@(?[^|]+)\|(?.+)$", RegexOptions.IgnoreCase); if (m.Success) { @@ -153,15 +154,32 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException(string.Format("The mountpoint \"{0}\" is not a valid rooted mountpoint", mountpoint), "InvalidMountpoint"); var backendurl = m.Groups["url"].Value; - results.Add(SourceProviderLoader.GetSourceProvider(backendurl, Path.GetFullPath(mountpoint), options.RawOptions) - ?? throw new UserInformationException(string.Format("The source \"{0}\" is not supported", backendurl), "SourceNotSupported")); + + ISourceProvider provider; + try + { + provider = await SourceProviderLoader.GetSourceProvider(backendurl, Path.GetFullPath(mountpoint), options.RawOptions, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + if (options.AllowMissingSource) + { + Log.WriteWarningMessage(LOGTAG, "SourceProviderFailed", ex, "Failed to load source provider for \"{0}\"", sanitizedUrl); + continue; + } + + throw new UserInformationException($"Failed to load source provider for \"{sanitizedUrl}\": {ex.Message}", "SourceProviderFailed", ex); + } + + // Don't accept missing providers + results.Add(provider ?? throw new UserInformationException($"The source \"{sanitizedUrl}\" is not supported", "SourceNotSupported")); } else - throw new UserInformationException(string.Format("The source \"{0}\" is not a supported format", url), "SourceFormatNotSupported"); + throw new UserInformationException($"The source \"{sanitizedUrl}\" is not a supported format", "SourceFormatNotSupported"); } } else - throw new UserInformationException(string.Format("The source type \"{0}\" is not supported", entry.Key), "SourceTypeNotSupported"); + throw new UserInformationException($"The source type \"{entry.Key}\" is not supported", "SourceTypeNotSupported"); } } catch @@ -172,6 +190,9 @@ namespace Duplicati.Library.Main.Operation throw; } + if (results.Count == 0) + throw new UserInformationException("No sources were available for the backup", "NoSourcesAvailable"); + return results; } @@ -600,7 +621,7 @@ namespace Duplicati.Library.Main.Operation { long filesetid; using var counterToken = CancellationTokenSource.CreateLinkedTokenSource(m_taskReader.ProgressToken); - using (var source = GetSourceProvider(sources, m_options)) + using (var source = await GetSourceProvider(sources, m_options, m_taskReader.ProgressToken).ConfigureAwait(false)) { try { diff --git a/Duplicati/Library/Main/Operation/TestFilterHandler.cs b/Duplicati/Library/Main/Operation/TestFilterHandler.cs index b2468738f..b00d37f7a 100644 --- a/Duplicati/Library/Main/Operation/TestFilterHandler.cs +++ b/Duplicati/Library/Main/Operation/TestFilterHandler.cs @@ -48,7 +48,7 @@ namespace Duplicati.Library.Main.Operation { var stopToken = m_result.TaskControl.ProgressToken; - using (var provider = BackupHandler.GetSourceProvider(sources, m_options)) + using (var provider = await BackupHandler.GetSourceProvider(sources, m_options, stopToken).ConfigureAwait(false)) { Backup.Channels channels = new(); var source = Backup.FileEnumerationProcess.Run(channels, provider, null, diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index 4d32c8ae3..bccc5c9c8 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -1042,7 +1042,7 @@ namespace Duplicati.Library.Main return r; } - } + } /// /// Gets a flag indicating if advisory locking should be ignored @@ -1706,7 +1706,7 @@ namespace Duplicati.Library.Main } /// - /// Gets a flag indicating if compacting should not be done automatically + /// Gets a flag indicating if missing source elements should be ignored /// public bool AllowMissingSource { diff --git a/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs b/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs index 93bea7d8c..481af6065 100644 --- a/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs +++ b/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs @@ -20,6 +20,7 @@ // DEALINGS IN THE SOFTWARE. using System.IO.Pipelines; +using System.Runtime.CompilerServices; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using Duplicati.Library.Utility; @@ -32,11 +33,11 @@ namespace Duplicati.Library.SourceProvider; /// The parent backend /// The path of the entry /// True if the entry is a folder -/// True if the entry is a meta entry +/// True if the entry is a meta entry /// The creation time of the entry /// The last modification time of the entry /// The size of the entry -public class BackendSourceFileEntry(BackendSourceProvider parent, string path, bool isFolder, bool isMetaEntry, DateTime createdUtc, DateTime lastModificationUtc, long size) +public class BackendSourceFileEntry(BackendSourceProvider parent, string path, bool isFolder, bool isRootEntry, DateTime createdUtc, DateTime lastModificationUtc, long size) : ISourceProviderEntry { /// @@ -48,10 +49,10 @@ public class BackendSourceFileEntry(BackendSourceProvider parent, string path, b public bool IsFolder => isFolder; /// - public bool IsMetaEntry => isMetaEntry; + public bool IsMetaEntry => false; /// - public bool IsRootEntry => isMetaEntry; + public bool IsRootEntry => isRootEntry; /// public DateTime CreatedUtc => createdUtc; @@ -90,36 +91,93 @@ public class BackendSourceFileEntry(BackendSourceProvider parent, string path, b /// public string? HardlinkTargetId => null; + /// + /// An async enumerator for the entries + /// + private IAsyncEnumerator? preparedEnumerator = null; + + /// + /// A flag to indicate if the prepared enumerator has any entries + /// + private bool preparedEnumeratorAny = false; + + /// + /// Prepares the enumerator for this entry + /// + /// The cancellation token + /// The prepared enumerator + public async Task PrepareEnumerator(CancellationToken cancellationToken) + { + if (!isFolder || !isRootEntry) + throw new InvalidOperationException("PrepareEnumerator can only be called on root folders"); + + if (preparedEnumerator != null) + throw new InvalidOperationException("PrepareEnumerator can only be called once"); + + var result = EnumerateInternal(cancellationToken).GetAsyncEnumerator(cancellationToken); + var prev = Interlocked.Exchange(ref preparedEnumerator, result); + if (prev != null) + throw new InvalidOperationException("PrepareEnumerator can only be called once"); + + // Advance to the first entry, so we are sure it does not throw exceptions + preparedEnumeratorAny = await result.MoveNextAsync().ConfigureAwait(false); + } + /// - public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) + public async IAsyncEnumerable Enumerate([EnumeratorCancellation] CancellationToken cancellationToken) { if (!isFolder) throw new InvalidOperationException("Enumerate can only be called on folders"); - return parent.WrappedBackend.ListAsync(NormalizePathTo(path, '/'), cancellationToken) - // Remove the current and parent folder entries - .Where(x => !string.IsNullOrWhiteSpace(x.Name) && x.Name != "." && x.Name != "..") - // Remove sub-folder entries - .Where(x => !x.Name[0..^1].Contains('\\') && !x.Name[0..^1].Contains('/')) - // Convert to source file entries - .Select(x => + if (isRootEntry) + { + // If we have a prepared enumerator, consume and use it + var enumerator = Interlocked.Exchange(ref preparedEnumerator, null); + if (enumerator != null) { - var localPath = SystemIO.IO_OS.PathCombine(path, NormalizePathToLocalSystem(x.Name)); - if (x.IsFolder) - localPath = Util.AppendDirSeparator(localPath); + if (preparedEnumeratorAny) + { + // It has already been advanced, so return the current value + yield return enumerator.Current; + + while (await enumerator.MoveNextAsync()) + yield return enumerator.Current; + } + + yield break; + } + } + + // Otherwise, enumerate the entries + await foreach (var entry in EnumerateInternal(cancellationToken).ConfigureAwait(false)) + yield return entry; - return new BackendSourceFileEntry( - parent, - localPath, - x.IsFolder, - false, - x.Created, - x.LastModification, - x.Size - ); - }); } + private IAsyncEnumerable EnumerateInternal(CancellationToken cancellationToken) + => parent.WrappedBackend.ListAsync(NormalizePathTo(path, '/'), cancellationToken) + // Remove the current and parent folder entries + .Where(x => !string.IsNullOrWhiteSpace(x.Name) && x.Name != "." && x.Name != "..") + // Remove sub-folder entries + .Where(x => !x.Name[0..^1].Contains('\\') && !x.Name[0..^1].Contains('/')) + // Convert to source file entries + .Select(x => + { + var localPath = SystemIO.IO_OS.PathCombine(path, NormalizePathToLocalSystem(x.Name)); + if (x.IsFolder) + localPath = Util.AppendDirSeparator(localPath); + + return new BackendSourceFileEntry( + parent, + localPath, + x.IsFolder, + false, + x.Created, + x.LastModification, + x.Size + ); + }); + /// public async Task FileExists(string path, CancellationToken cancellationToken) { diff --git a/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs index 2ae3d9a71..85a8ba1b5 100644 --- a/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs +++ b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs @@ -49,9 +49,39 @@ public class BackendSourceProvider(IFolderEnabledBackend backend, string mounted /// public IList SupportedCommands => backend.SupportedCommands; + /// + /// The prepared root entry, if any + /// + private BackendSourceFileEntry? preparedRoot; + + /// + /// Flags if the provider has been initialized + /// + private int isInitialized = 0; + + /// + /// Creates a root entry + /// + /// The root entry + private BackendSourceFileEntry CreateRoot() + => new BackendSourceFileEntry(this, "", true, true, new DateTime(0), new DateTime(0), 0); + + /// + public async Task Initialize(CancellationToken cancellationToken) + { + // Only allow a single intiiialization call + if (Interlocked.Exchange(ref isInitialized, 1) != 0) + return; + + // Prepare the root entry + var root = CreateRoot(); + await root.PrepareEnumerator(cancellationToken).ConfigureAwait(false); + preparedRoot = root; + } + /// public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) - => new[] { new BackendSourceFileEntry(this, "", true, true, new DateTime(0), new DateTime(0), 0) }.ToAsyncEnumerable(); + => new[] { Interlocked.Exchange(ref preparedRoot, null) ?? CreateRoot() }.ToAsyncEnumerable(); /// public async Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) diff --git a/Duplicati/Library/SourceProvider/Builtin/Combiner.cs b/Duplicati/Library/SourceProvider/Builtin/Combiner.cs index ed886a028..5205c049e 100644 --- a/Duplicati/Library/SourceProvider/Builtin/Combiner.cs +++ b/Duplicati/Library/SourceProvider/Builtin/Combiner.cs @@ -37,6 +37,10 @@ public class Combiner(IEnumerable providers) : ISourceProvider /// private readonly List providers = providers.SelectMany(x => x is Combiner c ? c.providers.AsEnumerable() : [x]).ToList(); + /// + public Task Initialize(CancellationToken cancellationToken) + => Task.WhenAll(providers.Select(x => x.Initialize(cancellationToken))); + /// public async IAsyncEnumerable Enumerate([EnumeratorCancellation] CancellationToken cancellationToken) { diff --git a/Duplicati/Library/SourceProvider/Builtin/LocalFileSource.cs b/Duplicati/Library/SourceProvider/Builtin/LocalFileSource.cs index d4ae9a336..2568b4474 100644 --- a/Duplicati/Library/SourceProvider/Builtin/LocalFileSource.cs +++ b/Duplicati/Library/SourceProvider/Builtin/LocalFileSource.cs @@ -33,6 +33,9 @@ public class LocalFileSource(ISnapshotService snapshotService) : ISourceProvider /// public string MountedPath => string.Empty; + /// + public Task Initialize(CancellationToken cancellationToken) => Task.CompletedTask; + /// public IAsyncEnumerable Enumerate(CancellationToken cancellationToken) => snapshotService.EnumerateFilesystemEntries().ToAsyncEnumerable(); diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index d3ce83d50..0874952e3 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -1565,7 +1565,38 @@ namespace Duplicati.Library.Utility return null; return url.Substring(0, idx); + } + /// + /// Returns a url that is safe to display, by removing any credentials + /// + /// The url to sanitize + /// The sanitized url + public static string GetUrlWithoutCredentials(string url) + { + // Assumed safe part of the url to show + const int maxShown = 25; + if (string.IsNullOrWhiteSpace(url)) + return url; + + // Use a reportable url without credentials + var sepIndex = Math.Max(0, url.IndexOf('|')) + 1; + var length = url.Length - sepIndex; + var shown = Math.Min(length, maxShown); + var hidden = length - maxShown; + var sanitizedUrl = $"{url[sepIndex..(sepIndex + shown)]}{new string('*', hidden)}"; + + // If we can parse it, this result is better + try + { + var uri = new Uri(url[sepIndex..]); + sanitizedUrl = new Uri($"{uri.Scheme}://{uri.Host}").SetPath(uri.Path).ToString(); + } + catch + { + } + + return sanitizedUrl; } } } From 1428d8cc6fc39efe01ca3b53b54c8b0f8659e65b Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 24 Feb 2025 12:53:48 +0100 Subject: [PATCH 26/28] Preserve order of input sources --- Duplicati/Library/Main/Controller.cs | 2 +- Duplicati/Library/Utility/Utility.cs | 11 +++++--- Duplicati/UnitTest/UtilityTests.cs | 41 ++++++++++++++-------------- 3 files changed, 28 insertions(+), 26 deletions(-) diff --git a/Duplicati/Library/Main/Controller.cs b/Duplicati/Library/Main/Controller.cs index abd3b311e..637caee3b 100644 --- a/Duplicati/Library/Main/Controller.cs +++ b/Duplicati/Library/Main/Controller.cs @@ -1012,7 +1012,7 @@ namespace Duplicati.Library.Main //Sanity check for duplicate files/folders ISet pathDuplicates; - sources = Library.Utility.Utility.GetUniqueItems(sources, Library.Utility.Utility.ClientFilenameStringComparer, out pathDuplicates).OrderBy(a => a).ToList(); + sources = Library.Utility.Utility.GetUniqueItems(sources, Library.Utility.Utility.ClientFilenameStringComparer, out pathDuplicates).ToList(); foreach (var pathDuplicate in pathDuplicates) Logging.Log.WriteVerboseMessage(LOGTAG, "RemoveDuplicateSource", "Removing duplicate source: {0}", pathDuplicate); diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index 0874952e3..038d3adad 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -981,7 +981,7 @@ namespace Duplicati.Library.Utility /// The collection to remove duplicate items from. /// The duplicate items in . /// The unique items from . - public static ISet GetUniqueItems(IEnumerable collection, out ISet duplicateItems) + public static IList GetUniqueItems(IEnumerable collection, out ISet duplicateItems) { return GetUniqueItems(collection, EqualityComparer.Default, out duplicateItems); } @@ -994,18 +994,21 @@ namespace Duplicati.Library.Utility /// The implementation to use when comparing values in the collection. /// The duplicate items in . /// The unique items from . - public static ISet GetUniqueItems(IEnumerable collection, IEqualityComparer comparer, out ISet duplicateItems) + public static IList GetUniqueItems(IEnumerable collection, IEqualityComparer comparer, out ISet duplicateItems) { var uniqueItems = new HashSet(comparer); + var results = new List(); duplicateItems = new HashSet(comparer); foreach (var item in collection) { - if (!uniqueItems.Add(item)) + if (uniqueItems.Add(item)) + results.Add(item); + else duplicateItems.Add(item); } - return uniqueItems; + return results; } // diff --git a/Duplicati/UnitTest/UtilityTests.cs b/Duplicati/UnitTest/UtilityTests.cs index c4dd5792c..35ca8b92c 100644 --- a/Duplicati/UnitTest/UtilityTests.cs +++ b/Duplicati/UnitTest/UtilityTests.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Utility; @@ -349,8 +349,7 @@ namespace Duplicati.UnitTest string[] duplicateItems = { "A", "c" }; // Test with default comparer. - ISet actualDuplicateItems; - ISet actualUniqueItems = Utility.GetUniqueItems(collection, out actualDuplicateItems); + var actualUniqueItems = Utility.GetUniqueItems(collection, out var actualDuplicateItems); CollectionAssert.AreEquivalent(uniqueItems, actualUniqueItems); CollectionAssert.AreEquivalent(duplicateItems, actualDuplicateItems); From 30b1cd59b8add78f9c73a5483e46ec4389729be1 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 24 Feb 2025 13:04:35 +0100 Subject: [PATCH 27/28] Fixed cases where Windows paths were sent to the remote destination --- .../Library/SourceProvider/Builtin/BackendSourceFileEntry.cs | 4 ++-- .../Library/SourceProvider/Builtin/BackendSourceProvider.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs b/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs index 481af6065..b9e134014 100644 --- a/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs +++ b/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs @@ -186,7 +186,7 @@ public class BackendSourceFileEntry(BackendSourceProvider parent, string path, b try { - var entry = await parent.WrappedBackend.GetEntryAsync(path, cancellationToken); + var entry = await parent.WrappedBackend.GetEntryAsync(NormalizePathTo(path, '/'), cancellationToken); return entry != null && !entry.IsFolder; } catch (FileNotFoundException) @@ -267,7 +267,7 @@ public class BackendSourceFileEntry(BackendSourceProvider parent, string path, b { try { - await streamingBackend.GetAsync(path, pipe.Writer.AsStream(), cancellationToken).ConfigureAwait(false); + await streamingBackend.GetAsync(NormalizePathTo(path, '/'), pipe.Writer.AsStream(), cancellationToken).ConfigureAwait(false); } catch (Exception ex) { diff --git a/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs index 85a8ba1b5..8e9c39853 100644 --- a/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs +++ b/Duplicati/Library/SourceProvider/Builtin/BackendSourceProvider.cs @@ -86,7 +86,7 @@ public class BackendSourceProvider(IFolderEnabledBackend backend, string mounted /// public async Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) { - var entry = await backend.GetEntryAsync(path, cancellationToken).ConfigureAwait(false); + var entry = await backend.GetEntryAsync(BackendSourceFileEntry.NormalizePathTo(path, '/'), cancellationToken).ConfigureAwait(false); return entry == null ? null : BackendSourceFileEntry.FromFileEntry(this, path, entry); } From 6238067f32352e9a81e7f59ca2f754d268ca71c1 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 24 Feb 2025 13:17:08 +0100 Subject: [PATCH 28/28] Updated logic for S3 backend, so the client implementations return the relative (non-prefixed) path --- Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs | 6 ------ Duplicati/Library/Backend/S3/S3AwsClient.cs | 4 ++-- Duplicati/Library/Backend/S3/S3Backend.cs | 8 -------- 3 files changed, 2 insertions(+), 16 deletions(-) diff --git a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs index 66d27e56b..62b5b8dc1 100644 --- a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs +++ b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs @@ -132,13 +132,7 @@ namespace Duplicati.Library.Backend public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { await foreach (IFileEntry file in Connection.ListBucketAsync(m_bucket, m_prefix, false, cancelToken).ConfigureAwait(false)) - { - ((FileEntry)file).Name = file.Name.Substring(m_prefix.Length); - if (file.Name.StartsWith("/", StringComparison.Ordinal) && !m_prefix.StartsWith("/", StringComparison.Ordinal)) - ((FileEntry)file).Name = file.Name.Substring(1); - yield return file; - } } public async Task PutAsync(string remotename, string localname, CancellationToken cancelToken) diff --git a/Duplicati/Library/Backend/S3/S3AwsClient.cs b/Duplicati/Library/Backend/S3/S3AwsClient.cs index 4527826dd..5fd16627b 100644 --- a/Duplicati/Library/Backend/S3/S3AwsClient.cs +++ b/Duplicati/Library/Backend/S3/S3AwsClient.cs @@ -278,7 +278,7 @@ namespace Duplicati.Library.Backend // Because the prefixes are returned, and not the folder objects // we do not get the folder modification date - yield return new Common.IO.FileEntry( + yield return new FileEntry( obj.Substring(prefix.Length), -1, new DateTime(0), @@ -293,7 +293,7 @@ namespace Duplicati.Library.Backend if (obj.Key == prefix || !obj.Key.StartsWith(prefix)) continue; - yield return new Common.IO.FileEntry( + yield return new FileEntry( obj.Key.Substring(prefix.Length), obj.Size, obj.LastModified, diff --git a/Duplicati/Library/Backend/S3/S3Backend.cs b/Duplicati/Library/Backend/S3/S3Backend.cs index 91877df04..4f081ea1c 100644 --- a/Duplicati/Library/Backend/S3/S3Backend.cs +++ b/Duplicati/Library/Backend/S3/S3Backend.cs @@ -281,15 +281,7 @@ namespace Duplicati.Library.Backend public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { await foreach (IFileEntry file in Connection.ListBucketAsync(m_bucket, m_prefix, m_recurseLists, cancelToken)) - { - ((FileEntry)file).Name = file.Name.Substring(m_prefix.Length); - - //Fix for a bug in Duplicati 1.0 beta 3 and earlier, where filenames are incorrectly prefixed with a slash - if (file.Name.StartsWith("/", StringComparison.Ordinal) && !m_prefix.StartsWith("/", StringComparison.Ordinal)) - ((FileEntry)file).Name = file.Name.Substring(1); - yield return file; - } } public async Task PutAsync(string remotename, string localname, CancellationToken cancelToken)