From a4820ccb621310cc1a417258d4500b74d0d4d974 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 8 Jun 2026 19:33:01 +0200 Subject: [PATCH 01/16] Implement support for ADS This PR implements support for reading and writing alternate data streams on Windows. For now, the option is disabled by default and needs to be enabled with `--enable-ads-backup`. After this option is set, Duplicati will also extract any alternate datastreams from the filesystem and include them in the backup. By default, ADS elements will be restored, but `--disable-ads-restore` can be used to ignore any ADS elements during the restore process. This fixes #189 --- .../Library/Common/IO/DefineDosDevice.cs | 43 +- Duplicati/Library/Common/IO/ISystemIO.cs | 14 + Duplicati/Library/Common/IO/SystemIOLinux.cs | 12 + .../Library/Common/IO/SystemIOWindows.cs | 174 +++++++- .../Main/Database/LocalRestoreDatabase.cs | 40 ++ .../Library/Main/Operation/BackupHandler.cs | 2 +- .../Library/Main/Operation/RepairHandler.cs | 2 +- .../Main/Operation/Restore/FileLister.cs | 19 +- .../Library/Main/Operation/RestoreHandler.cs | 6 + Duplicati/Library/Main/Options.cs | 12 + Duplicati/Library/Main/Strings.cs | 4 + Duplicati/Library/Snapshots/LinuxSnapshot.cs | 4 +- Duplicati/Library/Snapshots/MacOSSnapshot.cs | 8 +- .../Library/Snapshots/NoSnapshotLinux.cs | 4 +- .../Library/Snapshots/NoSnapshotWindows.cs | 59 ++- Duplicati/Library/Snapshots/SnapshotBase.cs | 44 +- .../Snapshots/SnapshotSourceFileEntry.cs | 5 +- .../Library/Snapshots/SnapshotUtility.cs | 5 +- .../Library/Snapshots/WindowsSnapshot.cs | 53 ++- .../UnitTest/AlternateDataStreamTests.cs | 422 ++++++++++++++++++ 20 files changed, 854 insertions(+), 78 deletions(-) create mode 100644 Duplicati/UnitTest/AlternateDataStreamTests.cs diff --git a/Duplicati/Library/Common/IO/DefineDosDevice.cs b/Duplicati/Library/Common/IO/DefineDosDevice.cs index 8c9bf971b..ba69e1bb0 100644 --- a/Duplicati/Library/Common/IO/DefineDosDevice.cs +++ b/Duplicati/Library/Common/IO/DefineDosDevice.cs @@ -1,26 +1,25 @@ -// Copyright (C) 2026, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. +// Copyright (C) 2026, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; -using System.Text; using System.IO; namespace Duplicati.Library.Common.IO @@ -68,7 +67,7 @@ namespace Duplicati.Library.Common.IO /// A pointer to an MS-DOS device name string specifying the device the function is defining, redefining, or deleting. The device name string must not have a colon as the last character, unless a drive letter is being defined, redefined, or deleted. For example, drive C would be the string "C:". In no case is a trailing backslash ("\") allowed. /// A pointer to a path string that will implement this device. The string is an MS-DOS path string unless the DDD_RAW_TARGET_PATH flag is specified, in which case this string is a path string. /// True on success, false otherwise - [System.Runtime.InteropServices.DllImport("kernel32", CharSet = System.Runtime.InteropServices.CharSet.Auto, SetLastError = true)] + [System.Runtime.InteropServices.DllImport("kernel32", CharSet = System.Runtime.InteropServices.CharSet.Auto, SetLastError = true)] public static extern bool DefineDosDevice(DDD_Flags flags, string devicename, string targetpath); } diff --git a/Duplicati/Library/Common/IO/ISystemIO.cs b/Duplicati/Library/Common/IO/ISystemIO.cs index 24973ff14..0c31a4c68 100644 --- a/Duplicati/Library/Common/IO/ISystemIO.cs +++ b/Duplicati/Library/Common/IO/ISystemIO.cs @@ -92,6 +92,20 @@ namespace Duplicati.Library.Common.IO /// /// The directory to set permissions on. void DirectorySetPermissionUserRWOnly(string path); + + /// + /// Enumerates the alternate data streams for a file (Windows-only). + /// + /// The path to the file. + /// An enumerable of stream names (e.g., ":streamname"). + IEnumerable EnumerateAlternateDataStreams(string path); + + /// + /// Determines whether the specified path is an alternate data stream. + /// + /// The path to check. + /// True if the path is an alternate data stream; otherwise, false. + bool IsAlternateDataStream(string path); } } diff --git a/Duplicati/Library/Common/IO/SystemIOLinux.cs b/Duplicati/Library/Common/IO/SystemIOLinux.cs index b26cb9ebd..b53d8713a 100644 --- a/Duplicati/Library/Common/IO/SystemIOLinux.cs +++ b/Duplicati/Library/Common/IO/SystemIOLinux.cs @@ -402,6 +402,18 @@ namespace Duplicati.Library.Common.IO Convert.ToInt32("700", 8) /* FilePermissions.S_IRUSR | FilePermissions.S_IWUSR | FilePermissions.S_IXUSR */ ); } + + /// + /// Enumerates the alternate data streams for a file (not supported on Linux/macOS). + /// + public IEnumerable EnumerateAlternateDataStreams(string path) + => Enumerable.Empty(); + + /// + /// Determines whether the specified path is an alternate data stream (always false on Linux/macOS). + /// + public bool IsAlternateDataStream(string path) + => false; } } diff --git a/Duplicati/Library/Common/IO/SystemIOWindows.cs b/Duplicati/Library/Common/IO/SystemIOWindows.cs index 01e54eac5..f97e9d20a 100644 --- a/Duplicati/Library/Common/IO/SystemIOWindows.cs +++ b/Duplicati/Library/Common/IO/SystemIOWindows.cs @@ -24,11 +24,14 @@ using System.Collections.Generic; using System.Security.AccessControl; using System.IO; using System.Linq; +using System.Runtime.InteropServices; using Duplicati.Library.Interface; using Newtonsoft.Json; using System.Runtime.Versioning; using System.Security.Principal; +using Microsoft.Win32.SafeHandles; +using System.ComponentModel; namespace Duplicati.Library.Common.IO { @@ -44,6 +47,93 @@ namespace Duplicati.Library.Common.IO private static readonly string DIRSEP = Util.DirectorySeparatorString; + /// + /// Encapsulation of Win32 calls + /// + private static class Win32API + { + /// + /// The stream info levels for FindFirstStreamW + /// + public enum STREAM_INFO_LEVELS + { + FindStreamInfoStandard = 0, + FindStreamInfoMaxInfoLevel = 1 + } + + /// + /// The access mode for CreateFile to only read attributes + /// + public const uint FILE_READ_ATTRIBUTES = 0x0080; + /// + /// Open only an existing file + /// + public const uint OPEN_EXISTING = 3; + /// + /// The share mode for CreateFile to allow read access + /// + public const uint FILE_SHARE_READ = 0x00000001; + /// + /// The share mode for CreateFile to allow write access + /// + public const uint FILE_SHARE_WRITE = 0x00000002; + /// + /// The share mode for CreateFile to allow delete access + /// + public const uint FILE_SHARE_DELETE = 0x00000004; + + /// + /// Creates a file handle + /// + /// The filename + /// The access mode + /// The share mode + /// Pointer to security attributes + /// Create mode flags + /// Flags and attributes + /// Handle to a template file + /// A filehandle for the entry + [DllImport("kernel32.dll", SetLastError = true, CharSet = CharSet.Unicode)] + public static extern SafeFileHandle CreateFileW( + string lpFileName, + uint dwDesiredAccess, + uint dwShareMode, + IntPtr lpSecurityAttributes, + uint dwCreationDisposition, + uint dwFlagsAndAttributes, + IntPtr hTemplateFile); + + /// + /// Gets the file size + /// + /// The file handle + /// The size of the file + /// true if the call succeeds, false otherwise + [DllImport("kernel32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool GetFileSizeEx(SafeFileHandle hFile, out long lpFileSize); + + /// + /// The WIN32_FIND_STREAM_DATA structure + /// + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)] + public struct WIN32_FIND_STREAM_DATA + { + public long StreamSize; + [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 296)] + public string cStreamName; + } + + [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)] + public static extern IntPtr FindFirstStreamW(string lpFileName, STREAM_INFO_LEVELS InfoLevel, out WIN32_FIND_STREAM_DATA lpFindStreamData, uint dwFlags); + + [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)] + public static extern bool FindNextStreamW(IntPtr hFindStream, out WIN32_FIND_STREAM_DATA lpFindStreamData); + + [DllImport("kernel32.dll", SetLastError = true)] + public static extern bool FindClose(IntPtr hFindFile); + } + /// /// The current user SID /// @@ -370,9 +460,6 @@ namespace Duplicati.Library.Common.IO File.Move(AddExtendedDevicePathPrefix(source), AddExtendedDevicePathPrefix(target)); } - public long FileLength(string path) - => new FileInfo(AddExtendedDevicePathPrefix(path)).Length; - public string GetPathRoot(string path) => IsPrefixedWithExtendedDevicePathPrefix(path) ? Path.GetPathRoot(path) @@ -540,6 +627,87 @@ namespace Duplicati.Library.Common.IO public string PathCombine(params string[] paths) => Path.Combine(paths); + /// + /// Determines whether the specified path is an alternate data stream. + /// + public bool IsAlternateDataStream(string path) + { + if (string.IsNullOrWhiteSpace(path)) return false; + + // Extracts the root component (e.g., "C:\", "C:", "\\server\share\", or "\\?\C:\") + string root = Path.GetPathRoot(path); + + // Isolate the remaining file/folder portion of the path + string relativePath = path.Substring(root.Length); + + // If a colon exists outside of the root structure, it's an alternate data stream + return relativePath.Contains(':'); + } + + /// + /// Enumerates the alternate data streams for a file. + /// + [SupportedOSPlatform("windows")] + public IEnumerable EnumerateAlternateDataStreams(string path) + { + var prefixed = AddExtendedDevicePathPrefix(path); + var data = new Win32API.WIN32_FIND_STREAM_DATA(); + IntPtr handle = IntPtr.Zero; + try + { + handle = Win32API.FindFirstStreamW(prefixed, Win32API.STREAM_INFO_LEVELS.FindStreamInfoStandard, out data, 0); + if (handle == new IntPtr(-1)) + yield break; + + do + { + // Skip the main stream (::$DATA) + if (data.cStreamName == "::$DATA") + continue; + + // Clean the stream name by removing the :$DATA suffix + var streamName = data.cStreamName; + if (streamName.EndsWith(":$DATA", StringComparison.OrdinalIgnoreCase)) + streamName = streamName.Substring(0, streamName.Length - ":$DATA".Length); + + yield return streamName; + } + while (Win32API.FindNextStreamW(handle, out data)); + } + finally + { + if (handle != IntPtr.Zero && handle != new IntPtr(-1)) + Win32API.FindClose(handle); + } + } + + /// + /// Returns the length of a file or alternate data stream. + /// + public long FileLength(string path) + { + if (!IsAlternateDataStream(path)) + return new FileInfo(AddExtendedDevicePathPrefix(path)).Length; + + using SafeFileHandle handle = Win32API.CreateFileW( + AddExtendedDevicePathPrefix(path), + Win32API.FILE_READ_ATTRIBUTES, + Win32API.FILE_SHARE_READ | Win32API.FILE_SHARE_WRITE | Win32API.FILE_SHARE_DELETE, // Broad sharing flags avoid collisions + IntPtr.Zero, + Win32API.OPEN_EXISTING, + 0, + IntPtr.Zero); + + if (handle.IsInvalid) + throw new Win32Exception(Marshal.GetLastWin32Error(), "Failed to open handle to alternative data stream "); + + // Retrieve the size directly from the stream handle + if (!Win32API.GetFileSizeEx(handle, out long streamSize)) + throw new Win32Exception(Marshal.GetLastWin32Error(), "Failed to retrieve alternate data stream size."); + + return streamSize; + } + public void CreateSymlink(string symlinkfile, string target, bool asDir) { if (FileExists(symlinkfile) || DirectoryExists(symlinkfile)) diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index a9b445e10..da9491b44 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -724,6 +724,46 @@ namespace Duplicati.Library.Main.Database return new Tuple(0, 0); } + /// + /// Removes alternate data stream entries from the temporary file table. + /// + /// A cancellation token to monitor for cancellation requests. + public async Task RemoveAlternateDataStreamsAsync(CancellationToken token) + { + if (string.IsNullOrWhiteSpace(m_tempfiletable)) + return; + + var adsIds = new List(); + await using var cmd = m_connection.CreateCommand($@" + SELECT ""ID"", ""Path"" + FROM ""{m_tempfiletable}"" + ") + .SetTransaction(m_rtr); + + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) + { + var id = rd.ConvertValueToInt64(0); + var path = rd.ConvertValueToString(1); + if (path != null && SystemIO.IO_OS.IsAlternateDataStream(path)) + adsIds.Add(id); + } + + if (adsIds.Count == 0) + return; + + await using var tmptable = await TemporaryDbValueList.CreateAsync(this, adsIds, token).ConfigureAwait(false); + + await using var delCmd = m_connection.CreateCommand($@" + DELETE FROM ""{m_tempfiletable}"" + WHERE ""ID"" IN (@Ids) + ") + .SetTransaction(m_rtr); + + await delCmd.ExpandInClauseParameterMssqliteAsync("@Ids", tmptable, token).ConfigureAwait(false); + await delCmd.ExecuteNonQueryAsync(token).ConfigureAwait(false); + } + /// /// Retrieves the first path from the temporary file table. /// diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 5a3f2287e..e57cb60de 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -124,7 +124,7 @@ namespace Duplicati.Library.Main.Operation } } - return SnapshotUtility.CreateNoSnapshot(sources, options.IgnoreAdvisoryLocking, options.SymlinkPolicy == Options.SymlinkStrategy.Follow, useSeBackup, options.HandleMacOSPhotoLibrary, options.MacOSPhotoLibraryPath); + return SnapshotUtility.CreateNoSnapshot(sources, options.IgnoreAdvisoryLocking, options.SymlinkPolicy == Options.SymlinkStrategy.Follow, useSeBackup, options.EnableAdsBackup, options.HandleMacOSPhotoLibrary, options.MacOSPhotoLibraryPath); } /// diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 9e6c80a59..ec6046cee 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -1156,7 +1156,7 @@ namespace Duplicati.Library.Main.Operation continue; } - using var snapshot = Snapshots.SnapshotUtility.CreateNoSnapshot([block.Path], m_options.IgnoreAdvisoryLocking, m_options.SymlinkPolicy == Options.SymlinkStrategy.Follow, PermissionHelper.HasSeBackupPrivilege(), m_options.HandleMacOSPhotoLibrary, m_options.MacOSPhotoLibraryPath); + using var snapshot = Snapshots.SnapshotUtility.CreateNoSnapshot([block.Path], m_options.IgnoreAdvisoryLocking, m_options.SymlinkPolicy == Options.SymlinkStrategy.Follow, PermissionHelper.HasSeBackupPrivilege(), m_options.EnableAdsBackup, m_options.HandleMacOSPhotoLibrary, m_options.MacOSPhotoLibraryPath); var entry = snapshot.GetFilesystemEntry(block.Path, isDir); if (entry == null) { diff --git a/Duplicati/Library/Main/Operation/Restore/FileLister.cs b/Duplicati/Library/Main/Operation/Restore/FileLister.cs index 2d24ffd7e..d167dbee9 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileLister.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileLister.cs @@ -25,7 +25,7 @@ using System.Diagnostics; using System.Linq; using System.Threading.Tasks; using CoCoL; -using Duplicati.Library.Interface; +using Duplicati.Library.Common.IO; using Duplicati.Library.Main.Database; #nullable enable @@ -72,14 +72,27 @@ namespace Duplicati.Library.Main.Operation.Restore { sw_get_files?.Start(); // The enumerables are cast to arrays to force the query to be executed and release the database lock. - var files = await db + var files = (await db .GetFilesAndSymlinksToRestoreAsync(result.TaskControl.ProgressToken) .ToArrayAsync() - .ConfigureAwait(false); + .ConfigureAwait(false)) + .AsEnumerable(); result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_DownloadingRemoteFiles); sw_get_files?.Stop(); + if (options.DisableAdsRestore) + { + var newlist = new List(files.Count()); + foreach (var f in files) + if (SystemIO.IO_OS.IsAlternateDataStream(f.TargetPath)) + Logging.Log.WriteVerboseMessage(LOGTAG, "SkipAdsRestore", "Skipping ADS restore for {0}", f.TargetPath); + else + newlist.Add(f); + + files = newlist; + } + sw_write_file?.Start(); // First, send priority files (if any) to be restored first diff --git a/Duplicati/Library/Main/Operation/RestoreHandler.cs b/Duplicati/Library/Main/Operation/RestoreHandler.cs index 90a96e38d..047748710 100644 --- a/Duplicati/Library/Main/Operation/RestoreHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreHandler.cs @@ -1066,6 +1066,12 @@ namespace Duplicati.Library.Main.Operation result.OperationProgressUpdater.UpdatefileCount(c.Item1, c.Item2, true); } + if (options.DisableAdsRestore) + { + using (new Logging.Timer(LOGTAG, "FilterAds", "Filtering alternate data streams from restore list")) + await database.RemoveAlternateDataStreamsAsync(result.TaskControl.ProgressToken).ConfigureAwait(false); + } + using (new Logging.Timer(LOGTAG, "SetTargetPaths", "SetTargetPaths")) if (!string.IsNullOrEmpty(restoreDestination.TargetDestination)) { diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index dd7e26b7d..adfd6ba6e 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -376,6 +376,8 @@ namespace Duplicati.Library.Main yield return new CommandLineArgument("vss-use-mapping", CommandLineArgument.ArgumentType.Boolean, Strings.Options.VssusemappingShort, Strings.Options.VssusemappingLong, "false"); yield return new CommandLineArgument("usn-policy", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.UsnpolicyShort, Strings.Options.UsnpolicyLong, "off", null, Enum.GetNames(typeof(OptimizationStrategy))); yield return new CommandLineArgument("backupread-policy", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.BackupreadpolicyShort, Strings.Options.BackupreadpolicyLong, DEFAULT_BACKUPREAD_POLICY.ToString(), null, Enum.GetNames(typeof(OptimizationStrategy))); + yield return new CommandLineArgument("enable-ads-backup", CommandLineArgument.ArgumentType.Boolean, Strings.Options.EnableAdsBackupShort, Strings.Options.EnableAdsBackupLong, "false"); + yield return new CommandLineArgument("disable-ads-restore", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DisableAdsRestoreShort, Strings.Options.DisableAdsRestoreLong, "false"); } if (OperatingSystem.IsLinux() || OperatingSystem.IsMacOS()) @@ -1878,6 +1880,16 @@ namespace Duplicati.Library.Main /// public int RestoreChannelBufferSize => GetInt("restore-channel-buffer-size", DEFAULT_RESTORE_CHANNEL_BUFFER_SIZE); + /// + /// Gets a value indicating whether NTFS alternate data streams should be backed up + /// + public bool EnableAdsBackup => GetBool("enable-ads-backup"); + + /// + /// Gets a value indicating whether NTFS alternate data streams should be skipped during restore + /// + public bool DisableAdsRestore => GetBool("disable-ads-restore"); + /// /// Toggles whether internal profiling is enabled and should be logged. /// diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs index 95e62e441..5936a4f32 100644 --- a/Duplicati/Library/Main/Strings.cs +++ b/Duplicati/Library/Main/Strings.cs @@ -404,6 +404,10 @@ namespace Duplicati.Library.Main.Strings public static string RepairRefreshLockInfoLong { get { return LC.L("Use this option to refresh the object lock expiration information from the backend during a repair operation. This will query the backend for the current lock status of each remote volume and update the local database accordingly. If the backend does not support locking, this option will be ignored."); } } public static string RefreshLockInfoCompleteShort { get { return LC.L("Perform a complete refresh of lock information"); } } public static string RefreshLockInfoCompleteLong { get { return LC.L("Use this option to perform a complete refresh of lock information from the backend. This will query the backend for the current lock status of each remote volume and update the local database accordingly. If this option is not specified, only the lock information for the remote volumes with no lock information will be refreshed."); } } + public static string EnableAdsBackupShort { get { return LC.L("Enable backup of NTFS alternate data streams"); } } + public static string EnableAdsBackupLong { get { return LC.L("Use this option to enable backup of NTFS alternate data streams (ADS). This is a Windows-only feature that allows backing up hidden data attached to files."); } } + public static string DisableAdsRestoreShort { get { return LC.L("Disable restore of NTFS alternate data streams"); } } + public static string DisableAdsRestoreLong { get { return LC.L("Use this option to skip restoring NTFS alternate data streams (ADS) during a restore operation. The main file content will still be restored."); } } } internal static class Common diff --git a/Duplicati/Library/Snapshots/LinuxSnapshot.cs b/Duplicati/Library/Snapshots/LinuxSnapshot.cs index 2c0f69424..fc3bec21d 100644 --- a/Duplicati/Library/Snapshots/LinuxSnapshot.cs +++ b/Duplicati/Library/Snapshots/LinuxSnapshot.cs @@ -121,9 +121,9 @@ namespace Duplicati.Library.Snapshots foreach (var sourceEntry in SourceEntries) { if (DirectoryExists(sourceEntry) || sourceEntry.EndsWith(System.IO.Path.DirectorySeparatorChar)) - yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(sourceEntry), true, true); + yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(sourceEntry), true, true, false); else - yield return new SnapshotSourceFileEntry(this, sourceEntry, false, true); + yield return new SnapshotSourceFileEntry(this, sourceEntry, false, true, false); } } diff --git a/Duplicati/Library/Snapshots/MacOSSnapshot.cs b/Duplicati/Library/Snapshots/MacOSSnapshot.cs index b74cf6b20..9576602e9 100644 --- a/Duplicati/Library/Snapshots/MacOSSnapshot.cs +++ b/Duplicati/Library/Snapshots/MacOSSnapshot.cs @@ -116,9 +116,9 @@ namespace Duplicati.Library.Snapshots foreach (var sourceEntry in SourceEntries) { if (DirectoryExists(sourceEntry) || sourceEntry.EndsWith(System.IO.Path.DirectorySeparatorChar.ToString())) - yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(sourceEntry), true, true); + yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(sourceEntry), true, true, false); else - yield return new SnapshotSourceFileEntry(this, sourceEntry, false, true); + yield return new SnapshotSourceFileEntry(this, sourceEntry, false, true, false); } } @@ -393,12 +393,12 @@ namespace Duplicati.Library.Snapshots if (isFolder) { if (System.IO.Directory.Exists(snapPath)) - return new SnapshotSourceFileEntry(this, isFolder ? Util.AppendDirSeparator(path) : path, isFolder, false); + return new SnapshotSourceFileEntry(this, isFolder ? Util.AppendDirSeparator(path) : path, isFolder, false, false); } else { if (System.IO.File.Exists(snapPath)) - return new SnapshotSourceFileEntry(this, isFolder ? Util.AppendDirSeparator(path) : path, isFolder, false); + return new SnapshotSourceFileEntry(this, isFolder ? Util.AppendDirSeparator(path) : path, isFolder, false, false); } } catch diff --git a/Duplicati/Library/Snapshots/NoSnapshotLinux.cs b/Duplicati/Library/Snapshots/NoSnapshotLinux.cs index 236a4d483..38a22835c 100644 --- a/Duplicati/Library/Snapshots/NoSnapshotLinux.cs +++ b/Duplicati/Library/Snapshots/NoSnapshotLinux.cs @@ -98,9 +98,9 @@ namespace Duplicati.Library.Snapshots foreach (var folder in m_sources) { if (DirectoryExists(folder) || folder.EndsWith(Path.DirectorySeparatorChar)) - yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(folder), true, true); + yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(folder), true, true, false); else - yield return new SnapshotSourceFileEntry(this, folder, false, true); + yield return new SnapshotSourceFileEntry(this, folder, false, true, false); } } diff --git a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs index 2a113e21d..4b5902dee 100644 --- a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs +++ b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs @@ -47,16 +47,23 @@ namespace Duplicati.Library.Snapshots /// private readonly IDisposable m_seBackupPrivilege; + /// + /// A flag indicating if alternate data streams should be backed up + /// + private readonly bool m_enableAdsBackup; + /// /// Initializes a new instance of the class. /// /// The list of entries to create snapshots of /// A flag indicating if symlinks should be followed /// A flag indicating if the SeBackupPrivilege should be used - public NoSnapshotWindows(IEnumerable sources, bool followSymlinks, bool useSeBackup) + /// A flag indicating if alternate data streams should be backed up + public NoSnapshotWindows(IEnumerable sources, bool followSymlinks, bool useSeBackup, bool enableAdsBackup) : base(followSymlinks) { m_sources = sources; + m_enableAdsBackup = enableAdsBackup; if (useSeBackup) m_seBackupPrivilege = WindowsShimLoader.NewSeBackupPrivilegeScope(); } @@ -100,9 +107,9 @@ namespace Duplicati.Library.Snapshots foreach (var folder in m_sources.Select(SystemIOWindows.RemoveExtendedDevicePathPrefix)) { if (DirectoryExists(folder) || folder.EndsWith(System.IO.Path.DirectorySeparatorChar)) - yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(folder), true, true); + yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(folder), true, true, false); else - yield return new SnapshotSourceFileEntry(this, folder, false, true); + yield return new SnapshotSourceFileEntry(this, folder, false, true, SystemIO.IO_OS.IsAlternateDataStream(folder)); } } @@ -128,34 +135,58 @@ namespace Duplicati.Library.Snapshots /// /// The full path to the file in non-snapshot format /// An open filestream that can be read - public override System.IO.Stream OpenRead(string localPath) + public override Stream OpenRead(string localPath) => SystemIO.IO_OS.FileOpenRead(localPath); + /// + public override ISourceProviderEntry GetFilesystemEntry(string path, bool isFolder) + { + if (m_enableAdsBackup && SystemIO.IO_OS.IsAlternateDataStream(path)) + { + var entry = base.GetFilesystemEntry(path, false); + if (entry != null) + entry = new SnapshotSourceFileEntry(this, entry.Path, false, entry.IsRootEntry, true); + return entry; + } + + return base.GetFilesystemEntry(path, isFolder); + } + /// /// Lists all files in the given folder /// /// All folders found in the folder /// The folder to examinate - protected override string[] ListFiles(string localFolderPath) + protected override IEnumerable ListFiles(string localFolderPath) { - string[] tmp = SystemIO.IO_OS.GetFiles(localFolderPath); - string[] res = new string[tmp.Length]; - for (int i = 0; i < tmp.Length; i++) - res[i] = SystemIOWindows.RemoveExtendedDevicePathPrefix(tmp[i]); + var tmp = SystemIO.IO_OS.GetFiles(localFolderPath); + var res = new List(tmp.Length); + foreach (var p in tmp) + res.Add(SystemIOWindows.RemoveExtendedDevicePathPrefix(p)); - return res; + if (!m_enableAdsBackup) + return res; + + // Expand ADS for each file + var expanded = ExpandAlternateDataStreams(res, SystemIO.IO_OS.EnumerateAlternateDataStreams).ToList(); + + // Also include ADS on the parent folder itself + var folderPath = localFolderPath.TrimEnd(Path.DirectorySeparatorChar); + foreach (var stream in SystemIO.IO_OS.EnumerateAlternateDataStreams(folderPath)) + expanded.Add(folderPath + stream); + + return expanded; } - /// /// Lists all folders in the given folder /// /// All folders found in the folder /// The folder to examinate - protected override string[] ListFolders(string localFolderPath) + protected override IEnumerable ListFolders(string localFolderPath) { - string[] tmp = SystemIO.IO_OS.GetDirectories(SystemIOWindows.AddExtendedDevicePathPrefix(localFolderPath)); - string[] res = new string[tmp.Length]; + var tmp = SystemIO.IO_OS.GetDirectories(SystemIOWindows.AddExtendedDevicePathPrefix(localFolderPath)); + var res = new string[tmp.Length]; for (int i = 0; i < tmp.Length; i++) res[i] = SystemIOWindows.RemoveExtendedDevicePathPrefix(tmp[i]); diff --git a/Duplicati/Library/Snapshots/SnapshotBase.cs b/Duplicati/Library/Snapshots/SnapshotBase.cs index 6ee60d51b..58823ac29 100644 --- a/Duplicati/Library/Snapshots/SnapshotBase.cs +++ b/Duplicati/Library/Snapshots/SnapshotBase.cs @@ -19,6 +19,8 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +#nullable enable + using System; using System.Collections.Generic; using System.IO; @@ -72,14 +74,14 @@ namespace Duplicati.Library.Snapshots /// The path to get the entry for /// A flag indicating if the path is a folder /// The filesystem entry - public virtual ISourceProviderEntry GetFilesystemEntry(string path, bool isFolder) + public virtual ISourceProviderEntry? GetFilesystemEntry(string path, bool isFolder) { if (isFolder && !DirectoryExists(path)) return null; else if (!isFolder && !FileExists(path)) return null; - return new SnapshotSourceFileEntry(this, isFolder ? Util.AppendDirSeparator(path) : path, isFolder, false); + return new SnapshotSourceFileEntry(this, isFolder ? Util.AppendDirSeparator(path) : path, isFolder, false, false); } /// @@ -94,10 +96,18 @@ namespace Duplicati.Library.Snapshots // Removing the null characters is a workaround for a bug in .NET core: // https://github.com/dotnet/runtime/issues/49803 foreach (var f in ListFiles(source.Path)) - yield return new SnapshotSourceFileEntry(this, f.Trim('\0'), false, false); + { + var entry = GetFilesystemEntry(f.Trim('\0'), false); + if (entry != null) + yield return entry; + } foreach (var d in ListFolders(source.Path)) - yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(d.Trim('\0')), true, false); + { + var entry = GetFilesystemEntry(Util.AppendDirSeparator(d.Trim('\0')), true); + if (entry != null) + yield return entry; + } } } @@ -163,7 +173,7 @@ namespace Duplicati.Library.Snapshots /// The metadata for the given file or folder /// The file or folder to examine /// A flag indicating if the target is a symlink - public abstract Dictionary GetMetadata(string localPath, bool isSymlink); + public abstract Dictionary GetMetadata(string localPath, bool isSymlink); /// /// Gets a value indicating if the path points to a block device @@ -178,7 +188,7 @@ namespace Duplicati.Library.Snapshots /// /// The hardlink ID /// The file or folder to examine - public virtual string HardlinkTargetID(string localPath) + public virtual string? HardlinkTargetID(string localPath) => null; /// @@ -202,7 +212,7 @@ namespace Duplicati.Library.Snapshots /// /// All folders found in the folder /// The folder to examinate - protected virtual string[] ListFolders(string localFolderPath) + protected virtual IEnumerable ListFolders(string localFolderPath) => Directory.GetDirectories(localFolderPath); /// @@ -210,9 +220,27 @@ namespace Duplicati.Library.Snapshots /// /// All folders found in the folder /// The folder to examinate - protected virtual string[] ListFiles(string localFolderPath) + protected virtual IEnumerable ListFiles(string localFolderPath) => Directory.GetFiles(localFolderPath); + /// + /// Expands a list of files to include alternate data streams + /// + /// The list of files + /// A function that enumerates streams for a given file path + /// The expanded list of files + protected static IEnumerable ExpandAlternateDataStreams(IEnumerable files, Func> enumerateStreams) + { + var result = new List(); + foreach (var file in files) + { + result.Add(file); + foreach (var stream in enumerateStreams(file)) + result.Add(file + stream); + } + return result; + } + #region IDisposable interface /// diff --git a/Duplicati/Library/Snapshots/SnapshotSourceFileEntry.cs b/Duplicati/Library/Snapshots/SnapshotSourceFileEntry.cs index 012c9bff2..5aabd37a1 100644 --- a/Duplicati/Library/Snapshots/SnapshotSourceFileEntry.cs +++ b/Duplicati/Library/Snapshots/SnapshotSourceFileEntry.cs @@ -39,7 +39,8 @@ namespace Duplicati.Library.Snapshots; /// The path of the file /// True if the entry is a folder, false otherwise /// True if the entry is the root entry, false otherwise -public class SnapshotSourceFileEntry(ISnapshotService service, string path, bool isFolder, bool isRoot) : ISourceProviderEntry +/// True if the entry is an alternate data stream, false otherwise +public class SnapshotSourceFileEntry(ISnapshotService service, string path, bool isFolder, bool isRoot, bool isAlternateStream) : ISourceProviderEntry { /// /// The symlink target @@ -99,7 +100,7 @@ public class SnapshotSourceFileEntry(ISnapshotService service, string path, bool /// /// Gets a value indicating if the entry is an alternate stream /// - public bool IsAlternateStream => false; + public bool IsAlternateStream => isAlternateStream; /// /// Gets the hardlink target ID, or null if the entry is not a hardlink diff --git a/Duplicati/Library/Snapshots/SnapshotUtility.cs b/Duplicati/Library/Snapshots/SnapshotUtility.cs index ab4c52c37..6586bd55b 100644 --- a/Duplicati/Library/Snapshots/SnapshotUtility.cs +++ b/Duplicati/Library/Snapshots/SnapshotUtility.cs @@ -72,9 +72,10 @@ namespace Duplicati.Library.Snapshots /// Whether to use SeBackupPrivilege on Windows /// Whether to handle MacOS Photos libraries specially /// The user specified MacOS Photos library path + /// Whether to backup alternate data streams on Windows /// The ISnapshotService implementation [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] - public static ISnapshotService CreateNoSnapshot(IEnumerable paths, bool ignoreAdvisoryLocking, bool followSymlinks, bool useSeBackup, MacOSPhotosHandling macOSPhotosHandling, string photosLibraryPath) + public static ISnapshotService CreateNoSnapshot(IEnumerable paths, bool ignoreAdvisoryLocking, bool followSymlinks, bool useSeBackup, bool enableAdsBackup, MacOSPhotosHandling macOSPhotosHandling, string photosLibraryPath) { // MacOS implementation only handles photo libraries specially if requested // Otherwise, it behaves like the Linux implementation @@ -83,7 +84,7 @@ namespace Duplicati.Library.Snapshots else if (OperatingSystem.IsLinux()) return new NoSnapshotLinux(paths, ignoreAdvisoryLocking, followSymlinks); else if (OperatingSystem.IsWindows()) - return new NoSnapshotWindows(paths, followSymlinks, useSeBackup); + return new NoSnapshotWindows(paths, followSymlinks, useSeBackup, enableAdsBackup); else throw new NotSupportedException("Unsupported Operating System"); diff --git a/Duplicati/Library/Snapshots/WindowsSnapshot.cs b/Duplicati/Library/Snapshots/WindowsSnapshot.cs index 961e53df4..2269958c0 100644 --- a/Duplicati/Library/Snapshots/WindowsSnapshot.cs +++ b/Duplicati/Library/Snapshots/WindowsSnapshot.cs @@ -77,6 +77,11 @@ namespace Duplicati.Library.Snapshots /// private readonly IReadOnlyList _sourceEntries; + /// + /// A flag indicating if alternate data streams should be backed up + /// + private readonly bool _enableAdsBackup; + /// /// Constructs a new backup snapshot, using all the required disks /// @@ -86,6 +91,7 @@ namespace Duplicati.Library.Snapshots public WindowsSnapshot(IEnumerable sources, IDictionary options, bool followSymlinks) : base(followSymlinks) { + _enableAdsBackup = Utility.Utility.ParseBoolOption(options.AsReadOnly(), "enable-ads-backup"); // For Windows, ensure we don't store paths with extended device path prefixes (i.e., @"\\?\" or @"\\?\UNC\") _sourceEntries = sources.Select(SystemIOWindows.RemoveExtendedDevicePathPrefix).ToList(); try @@ -148,9 +154,9 @@ namespace Duplicati.Library.Snapshots foreach (var folder in _sourceEntries) { if (folder.EndsWith(Path.DirectorySeparatorChar) || DirectoryExists(folder)) - yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(folder), true, true); + yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(folder), true, true, false); else - yield return new SnapshotSourceFileEntry(this, folder, false, true); + yield return new SnapshotSourceFileEntry(this, folder, false, true, IO_WIN.IsAlternateDataStream(folder)); } } @@ -162,19 +168,16 @@ namespace Duplicati.Library.Snapshots /// /// The non-shadow path of the folder to list /// A list of non-shadow paths - protected override string[] ListFolders(string localFolderPath) + protected override IEnumerable ListFolders(string localFolderPath) { - string[] tmp = null; var spath = ConvertToSnapshotPath(localFolderPath); - tmp = IO_WIN.GetDirectories(spath); + var tmp = IO_WIN.GetDirectories(spath); var root = Util.AppendDirSeparator(IO_WIN.GetPathRoot(localFolderPath)); var volumePath = Util.AppendDirSeparator(ConvertToSnapshotPath(root)); volumePath = SystemIOWindows.AddExtendedDevicePathPrefix(volumePath); for (var i = 0; i < tmp.Length; i++) - { tmp[i] = root + SystemIOWindows.AddExtendedDevicePathPrefix(tmp[i]).Substring(volumePath.Length); - } return tmp; } @@ -186,12 +189,10 @@ namespace Duplicati.Library.Snapshots /// /// The non-shadow path of the folder to list /// A list of non-shadow paths - protected override string[] ListFiles(string localFolderPath) + protected override IEnumerable ListFiles(string localFolderPath) { - - string[] files = null; var spath = ConvertToSnapshotPath(localFolderPath); - files = IO_WIN.GetFiles(spath); + var files = IO_WIN.GetFiles(spath); // convert back to non-shadow, i.e., non-vss version var root = Util.AppendDirSeparator(IO_WIN.GetPathRoot(localFolderPath)); @@ -199,11 +200,21 @@ namespace Duplicati.Library.Snapshots volumePath = SystemIOWindows.AddExtendedDevicePathPrefix(volumePath); for (var i = 0; i < files.Length; i++) - { files[i] = root + SystemIOWindows.AddExtendedDevicePathPrefix(files[i]).Substring(volumePath.Length); - } - return files; + if (!_enableAdsBackup) + return files; + + // Expand ADS for each file + var expanded = ExpandAlternateDataStreams(files, f => SystemIO.IO_OS.EnumerateAlternateDataStreams(ConvertToSnapshotPath(f))).ToList(); + + // Also include ADS on the parent folder itself + var folderPath = localFolderPath.TrimEnd(Path.DirectorySeparatorChar); + var folderShadowPath = ConvertToSnapshotPath(folderPath); + foreach (var stream in SystemIO.IO_OS.EnumerateAlternateDataStreams(folderShadowPath)) + expanded.Add(folderPath + stream); + + return expanded; } #endregion @@ -233,6 +244,20 @@ namespace Duplicati.Library.Snapshots return IO_WIN.GetCreationTimeUtc(SystemIOWindows.AddExtendedDevicePathPrefix(spath)); } + /// + public override ISourceProviderEntry GetFilesystemEntry(string path, bool isFolder) + { + if (_enableAdsBackup && SystemIO.IO_OS.IsAlternateDataStream(path)) + { + var entry = base.GetFilesystemEntry(path, false); + if (entry != null) + entry = new SnapshotSourceFileEntry(this, entry.Path, false, entry.IsRootEntry, true); + return entry; + } + + return base.GetFilesystemEntry(path, isFolder); + } + /// /// Opens a file for reading /// diff --git a/Duplicati/UnitTest/AlternateDataStreamTests.cs b/Duplicati/UnitTest/AlternateDataStreamTests.cs new file mode 100644 index 000000000..fd92c25da --- /dev/null +++ b/Duplicati/UnitTest/AlternateDataStreamTests.cs @@ -0,0 +1,422 @@ +// Copyright (C) 2026, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Duplicati.Library.Common.IO; +using Duplicati.Library.Main; +using NUnit.Framework; +using Assert = NUnit.Framework.Legacy.ClassicAssert; + +namespace Duplicati.UnitTest +{ + [Category("Windows")] + public class AlternateDataStreamTests : BasicSetupHelper + { + /// + /// Creates an alternate data stream on Windows by opening a file stream with the ADS path syntax. + /// + private static void CreateAlternateDataStream(string filePath, string streamName, string content) + { + var adsPath = filePath + ":" + streamName; + File.WriteAllText(adsPath, content); + } + + /// + /// Reads the content of an alternate data stream. + /// + private static string ReadAlternateDataStream(string filePath, string streamName) + { + var adsPath = filePath + ":" + streamName; + return File.ReadAllText(adsPath); + } + + /// + /// Checks whether an alternate data stream exists. + /// + private static bool AlternateDataStreamExists(string filePath, string streamName) + { + var adsPath = filePath + ":" + streamName; + return File.Exists(adsPath); + } + + [Test] + [Category("Windows")] + public void EnumerateAlternateDataStreams() + { + if (!OperatingSystem.IsWindows()) + { + Assert.Ignore("This test is Windows-only."); + return; + } + + var filePath = Path.Combine(DATAFOLDER, "testfile.txt"); + File.WriteAllText(filePath, "main content"); + CreateAlternateDataStream(filePath, "stream1", "stream1 content"); + CreateAlternateDataStream(filePath, "stream2", "stream2 content"); + + var streams = SystemIO.IO_OS.EnumerateAlternateDataStreams(filePath).ToList(); + Assert.AreEqual(2, streams.Count); + Assert.IsTrue(streams.Contains(":stream1")); + Assert.IsTrue(streams.Contains(":stream2")); + } + + [Test] + [Category("Windows")] + public async Task BackupAndRestoreAlternateDataStreamsAsync() + { + if (!OperatingSystem.IsWindows()) + { + Assert.Ignore("This test is Windows-only."); + return; + } + + var filePath = Path.Combine(DATAFOLDER, "testfile.txt"); + File.WriteAllText(filePath, "main content"); + CreateAlternateDataStream(filePath, "hiddenstream", "hidden content"); + + var backupOptions = new Dictionary(TestOptions) + { + ["enable-ads-backup"] = "true" + }; + + using (var c = new Controller("file://" + TARGETFOLDER, backupOptions, null)) + TestUtils.AssertResults(await c.BackupAsync(new[] { DATAFOLDER })); + + // Restore without disabling ADS restore + var restoreOptions = new Dictionary(TestOptions) + { + ["restore-path"] = RESTOREFOLDER + }; + + using (var c = new Controller("file://" + TARGETFOLDER, restoreOptions, null)) + TestUtils.AssertResults(await c.RestoreAsync(new[] { filePath })); + + var restoredFilePath = Path.Combine(RESTOREFOLDER, "testfile.txt"); + Assert.IsTrue(File.Exists(restoredFilePath)); + Assert.AreEqual("main content", File.ReadAllText(restoredFilePath)); + Assert.IsTrue(AlternateDataStreamExists(restoredFilePath, "hiddenstream")); + Assert.AreEqual("hidden content", ReadAlternateDataStream(restoredFilePath, "hiddenstream")); + } + + [Test] + [Category("Windows")] + public async Task DisableAdsRestoreSkipsAlternateDataStreamsAsync() + { + if (!OperatingSystem.IsWindows()) + { + Assert.Ignore("This test is Windows-only."); + return; + } + + var filePath = Path.Combine(DATAFOLDER, "testfile.txt"); + File.WriteAllText(filePath, "main content"); + CreateAlternateDataStream(filePath, "hiddenstream", "hidden content"); + + var backupOptions = new Dictionary(TestOptions) + { + ["enable-ads-backup"] = "true" + }; + + using (var c = new Controller("file://" + TARGETFOLDER, backupOptions, null)) + TestUtils.AssertResults(await c.BackupAsync(new[] { DATAFOLDER })); + + // Restore with ADS restore disabled + var restoreOptions = new Dictionary(TestOptions) + { + ["restore-path"] = RESTOREFOLDER, + ["disable-ads-restore"] = "true" + }; + + using (var c = new Controller("file://" + TARGETFOLDER, restoreOptions, null)) + TestUtils.AssertResults(await c.RestoreAsync(new[] { filePath })); + + var restoredFilePath = Path.Combine(RESTOREFOLDER, "testfile.txt"); + Assert.IsTrue(File.Exists(restoredFilePath)); + Assert.AreEqual("main content", File.ReadAllText(restoredFilePath)); + Assert.IsFalse(AlternateDataStreamExists(restoredFilePath, "hiddenstream")); + } + + [Test] + [Category("Windows")] + public void IsAlternateDataStream() + { + if (!OperatingSystem.IsWindows()) + { + Assert.Ignore("This test is Windows-only."); + return; + } + + var filePath = Path.Combine(DATAFOLDER, "testfile.txt"); + File.WriteAllText(filePath, "main content"); + + Assert.IsFalse(SystemIO.IO_OS.IsAlternateDataStream(filePath)); + Assert.IsTrue(SystemIO.IO_OS.IsAlternateDataStream(filePath + ":stream")); + Assert.IsTrue(SystemIO.IO_OS.IsAlternateDataStream("C:\\file.txt:ads")); + Assert.IsFalse(SystemIO.IO_OS.IsAlternateDataStream("C:\\file.txt")); + + var dirPath = Path.Combine(DATAFOLDER, "testdir"); + Directory.CreateDirectory(dirPath); + + Assert.IsFalse(SystemIO.IO_OS.IsAlternateDataStream(dirPath)); + Assert.IsTrue(SystemIO.IO_OS.IsAlternateDataStream(dirPath + ":stream")); + Assert.IsTrue(SystemIO.IO_OS.IsAlternateDataStream("C:\\folder:ads")); + Assert.IsFalse(SystemIO.IO_OS.IsAlternateDataStream("C:\\folder")); + } + + [Test] + [Category("Windows")] + public void EnumerateAlternateDataStreamsOnDirectory() + { + if (!OperatingSystem.IsWindows()) + { + Assert.Ignore("This test is Windows-only."); + return; + } + + var dirPath = Path.Combine(DATAFOLDER, "testdir"); + Directory.CreateDirectory(dirPath); + CreateAlternateDataStream(dirPath, "stream1", "stream1 content"); + CreateAlternateDataStream(dirPath, "stream2", "stream2 content"); + + var streams = SystemIO.IO_OS.EnumerateAlternateDataStreams(dirPath).ToList(); + Assert.AreEqual(2, streams.Count); + Assert.IsTrue(streams.Contains(":stream1")); + Assert.IsTrue(streams.Contains(":stream2")); + } + + [Test] + [Category("Windows")] + public async Task BackupWithoutEnableAdsBackupIgnoresStreamsAsync() + { + if (!OperatingSystem.IsWindows()) + { + Assert.Ignore("This test is Windows-only."); + return; + } + + var filePath = Path.Combine(DATAFOLDER, "testfile.txt"); + File.WriteAllText(filePath, "main content"); + CreateAlternateDataStream(filePath, "hiddenstream", "hidden content"); + + var backupOptions = new Dictionary(TestOptions); + // enable-ads-backup is not set (defaults to false) + + using (var c = new Controller("file://" + TARGETFOLDER, backupOptions, null)) + TestUtils.AssertResults(await c.BackupAsync(new[] { DATAFOLDER })); + + var restoreOptions = new Dictionary(TestOptions) + { + ["restore-path"] = RESTOREFOLDER + }; + + using (var c = new Controller("file://" + TARGETFOLDER, restoreOptions, null)) + TestUtils.AssertResults(await c.RestoreAsync(new[] { filePath })); + + var restoredFilePath = Path.Combine(RESTOREFOLDER, "testfile.txt"); + Assert.IsTrue(File.Exists(restoredFilePath)); + Assert.AreEqual("main content", File.ReadAllText(restoredFilePath)); + Assert.IsFalse(AlternateDataStreamExists(restoredFilePath, "hiddenstream")); + } + + [Test] + [Category("Windows")] + public async Task BackupAndRestoreAlternateDataStreamsOnDirectoryAsync() + { + if (!OperatingSystem.IsWindows()) + { + Assert.Ignore("This test is Windows-only."); + return; + } + + var dirPath = Path.Combine(DATAFOLDER, "testdir"); + Directory.CreateDirectory(dirPath); + CreateAlternateDataStream(dirPath, "hiddenstream", "hidden content"); + + var backupOptions = new Dictionary(TestOptions) + { + ["enable-ads-backup"] = "true" + }; + + using (var c = new Controller("file://" + TARGETFOLDER, backupOptions, null)) + TestUtils.AssertResults(await c.BackupAsync(new[] { DATAFOLDER })); + + var restoreOptions = new Dictionary(TestOptions) + { + ["restore-path"] = RESTOREFOLDER + }; + + using (var c = new Controller("file://" + TARGETFOLDER, restoreOptions, null)) + TestUtils.AssertResults(await c.RestoreAsync(new[] { dirPath })); + + var restoredDirPath = Path.Combine(RESTOREFOLDER, "testdir"); + Assert.IsTrue(Directory.Exists(restoredDirPath)); + Assert.IsTrue(AlternateDataStreamExists(restoredDirPath, "hiddenstream")); + Assert.AreEqual("hidden content", ReadAlternateDataStream(restoredDirPath, "hiddenstream")); + } + + [Test] + [Category("Windows")] + public async Task DisableAdsRestoreSkipsAlternateDataStreamsOnDirectoryAsync() + { + if (!OperatingSystem.IsWindows()) + { + Assert.Ignore("This test is Windows-only."); + return; + } + + var dirPath = Path.Combine(DATAFOLDER, "testdir"); + Directory.CreateDirectory(dirPath); + CreateAlternateDataStream(dirPath, "hiddenstream", "hidden content"); + + var backupOptions = new Dictionary(TestOptions) + { + ["enable-ads-backup"] = "true" + }; + + using (var c = new Controller("file://" + TARGETFOLDER, backupOptions, null)) + TestUtils.AssertResults(await c.BackupAsync(new[] { DATAFOLDER })); + + var restoreOptions = new Dictionary(TestOptions) + { + ["restore-path"] = RESTOREFOLDER, + ["disable-ads-restore"] = "true" + }; + + using (var c = new Controller("file://" + TARGETFOLDER, restoreOptions, null)) + TestUtils.AssertResults(await c.RestoreAsync(new[] { dirPath })); + + var restoredDirPath = Path.Combine(RESTOREFOLDER, "testdir"); + Assert.IsTrue(Directory.Exists(restoredDirPath)); + Assert.IsFalse(AlternateDataStreamExists(restoredDirPath, "hiddenstream")); + } + + [Test] + [Category("Windows")] + public async Task BackupAndRestoreAlternateDataStreamsOnDirectoryLegacyAsync() + { + if (!OperatingSystem.IsWindows()) + { + Assert.Ignore("This test is Windows-only."); + return; + } + + var dirPath = Path.Combine(DATAFOLDER, "testdir"); + Directory.CreateDirectory(dirPath); + CreateAlternateDataStream(dirPath, "hiddenstream", "hidden content"); + + var backupOptions = new Dictionary(TestOptions) + { + ["enable-ads-backup"] = "true" + }; + + using (var c = new Controller("file://" + TARGETFOLDER, backupOptions, null)) + TestUtils.AssertResults(await c.BackupAsync(new[] { DATAFOLDER })); + + var restoreOptions = new Dictionary(TestOptions) + { + ["restore-path"] = RESTOREFOLDER, + ["restore-legacy"] = "true" + }; + + using (var c = new Controller("file://" + TARGETFOLDER, restoreOptions, null)) + TestUtils.AssertResults(await c.RestoreAsync(new[] { dirPath })); + + var restoredDirPath = Path.Combine(RESTOREFOLDER, "testdir"); + Assert.IsTrue(Directory.Exists(restoredDirPath)); + Assert.IsTrue(AlternateDataStreamExists(restoredDirPath, "hiddenstream")); + Assert.AreEqual("hidden content", ReadAlternateDataStream(restoredDirPath, "hiddenstream")); + } + + [Test] + [Category("Windows")] + public async Task DisableAdsRestoreSkipsAlternateDataStreamsOnDirectoryLegacyAsync() + { + if (!OperatingSystem.IsWindows()) + { + Assert.Ignore("This test is Windows-only."); + return; + } + + var dirPath = Path.Combine(DATAFOLDER, "testdir"); + Directory.CreateDirectory(dirPath); + CreateAlternateDataStream(dirPath, "hiddenstream", "hidden content"); + + var backupOptions = new Dictionary(TestOptions) + { + ["enable-ads-backup"] = "true" + }; + + using (var c = new Controller("file://" + TARGETFOLDER, backupOptions, null)) + TestUtils.AssertResults(await c.BackupAsync(new[] { DATAFOLDER })); + + var restoreOptions = new Dictionary(TestOptions) + { + ["restore-path"] = RESTOREFOLDER, + ["disable-ads-restore"] = "true", + ["restore-legacy"] = "true" + }; + + using (var c = new Controller("file://" + TARGETFOLDER, restoreOptions, null)) + TestUtils.AssertResults(await c.RestoreAsync(new[] { dirPath })); + + var restoredDirPath = Path.Combine(RESTOREFOLDER, "testdir"); + Assert.IsTrue(Directory.Exists(restoredDirPath)); + Assert.IsFalse(AlternateDataStreamExists(restoredDirPath, "hiddenstream")); + } + + [Test] + [Category("Windows")] + public async Task BackupWithoutEnableAdsBackupIgnoresStreamsOnDirectoryAsync() + { + if (!OperatingSystem.IsWindows()) + { + Assert.Ignore("This test is Windows-only."); + return; + } + + var dirPath = Path.Combine(DATAFOLDER, "testdir"); + Directory.CreateDirectory(dirPath); + CreateAlternateDataStream(dirPath, "hiddenstream", "hidden content"); + + var backupOptions = new Dictionary(TestOptions); + // enable-ads-backup is not set (defaults to false) + + using (var c = new Controller("file://" + TARGETFOLDER, backupOptions, null)) + TestUtils.AssertResults(await c.BackupAsync(new[] { DATAFOLDER })); + + var restoreOptions = new Dictionary(TestOptions) + { + ["restore-path"] = RESTOREFOLDER + }; + + using (var c = new Controller("file://" + TARGETFOLDER, restoreOptions, null)) + TestUtils.AssertResults(await c.RestoreAsync(new[] { dirPath })); + + var restoredDirPath = Path.Combine(RESTOREFOLDER, "testdir"); + Assert.IsTrue(Directory.Exists(restoredDirPath)); + Assert.IsFalse(AlternateDataStreamExists(restoredDirPath, "hiddenstream")); + } + } +} From 9f3ea8f805d7f7f0bad828332b2f56a367086bd3 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 15 Jun 2026 14:43:24 +0200 Subject: [PATCH 02/16] Fixed isseus with restoring ADS correctly --- Duplicati/Library/Common/IO/ISystemIO.cs | 14 +++ Duplicati/Library/Common/IO/SystemIOLinux.cs | 19 ++-- .../Library/Common/IO/SystemIOWindows.cs | 35 ++++-- .../Main/Database/LocalRestoreDatabase.cs | 106 +++++++++++++++++- .../Library/Main/Operation/RestoreHandler.cs | 10 +- .../Library/Snapshots/NoSnapshotWindows.cs | 9 +- Duplicati/Library/Snapshots/SnapshotBase.cs | 14 +-- .../Library/Snapshots/WindowsSnapshot.cs | 8 +- 8 files changed, 172 insertions(+), 43 deletions(-) diff --git a/Duplicati/Library/Common/IO/ISystemIO.cs b/Duplicati/Library/Common/IO/ISystemIO.cs index 0c31a4c68..d7f2230f6 100644 --- a/Duplicati/Library/Common/IO/ISystemIO.cs +++ b/Duplicati/Library/Common/IO/ISystemIO.cs @@ -93,6 +93,11 @@ namespace Duplicati.Library.Common.IO /// The directory to set permissions on. void DirectorySetPermissionUserRWOnly(string path); + /// + /// Gets a value indicating whether the operating system supports alternate data streams. + /// + bool SupportsAlternateDataStreams { get; } + /// /// Enumerates the alternate data streams for a file (Windows-only). /// @@ -106,6 +111,15 @@ namespace Duplicati.Library.Common.IO /// The path to check. /// True if the path is an alternate data stream; otherwise, false. bool IsAlternateDataStream(string path); + + /// + /// Returns the parent file or folder path of an alternate data stream path. + /// For a path such as C:\file.txt:stream this returns C:\file.txt. + /// If the path is not an alternate data stream, the path is returned unchanged. + /// + /// The alternate data stream path. + /// The parent file or folder path. + string GetAlternateDataStreamParent(string path); } } diff --git a/Duplicati/Library/Common/IO/SystemIOLinux.cs b/Duplicati/Library/Common/IO/SystemIOLinux.cs index b53d8713a..48daeb0fa 100644 --- a/Duplicati/Library/Common/IO/SystemIOLinux.cs +++ b/Duplicati/Library/Common/IO/SystemIOLinux.cs @@ -403,17 +403,20 @@ namespace Duplicati.Library.Common.IO ); } - /// - /// Enumerates the alternate data streams for a file (not supported on Linux/macOS). - /// - public IEnumerable EnumerateAlternateDataStreams(string path) - => Enumerable.Empty(); + /// + public bool SupportsAlternateDataStreams => false; - /// - /// Determines whether the specified path is an alternate data stream (always false on Linux/macOS). - /// + /// + public IEnumerable EnumerateAlternateDataStreams(string path) + => []; + + /// public bool IsAlternateDataStream(string path) => false; + + /// + public string GetAlternateDataStreamParent(string path) + => path; } } diff --git a/Duplicati/Library/Common/IO/SystemIOWindows.cs b/Duplicati/Library/Common/IO/SystemIOWindows.cs index f97e9d20a..5ba0ef28f 100644 --- a/Duplicati/Library/Common/IO/SystemIOWindows.cs +++ b/Duplicati/Library/Common/IO/SystemIOWindows.cs @@ -627,26 +627,39 @@ namespace Duplicati.Library.Common.IO public string PathCombine(params string[] paths) => Path.Combine(paths); - /// - /// Determines whether the specified path is an alternate data stream. - /// + /// + public bool SupportsAlternateDataStreams => true; + + /// public bool IsAlternateDataStream(string path) { if (string.IsNullOrWhiteSpace(path)) return false; // Extracts the root component (e.g., "C:\", "C:", "\\server\share\", or "\\?\C:\") - string root = Path.GetPathRoot(path); - - // Isolate the remaining file/folder portion of the path - string relativePath = path.Substring(root.Length); + var root = Path.GetPathRoot(path) ?? string.Empty; // If a colon exists outside of the root structure, it's an alternate data stream - return relativePath.Contains(':'); + return path.AsSpan(root.Length).Contains(':'); } - /// - /// Enumerates the alternate data streams for a file. - /// + /// + public string GetAlternateDataStreamParent(string path) + { + if (string.IsNullOrWhiteSpace(path)) + return path; + + // Extracts the root component (e.g., "C:\", "C:", "\\server\share\", or "\\?\C:\") + var root = Path.GetPathRoot(path) ?? string.Empty; + var relativePath = path.AsSpan(root.Length); + + var idx = relativePath.IndexOf(':'); + if (idx < 0) + return path; + + return root + relativePath.Slice(0, idx).ToString(); + } + + /// [SupportedOSPlatform("windows")] public IEnumerable EnumerateAlternateDataStreams(string path) { diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index da9491b44..d539e4772 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -407,13 +407,16 @@ namespace Duplicati.Library.Main.Database /// The time at which the restore operation is being performed. /// An array of version identifiers to filter the filesets. /// An optional filter to apply to the files being restored. + /// A flag indicating whether to disable the restoration of alternate data streams. /// A cancellation token to monitor for cancellation requests. /// A task that, when awaited, returns a tuple containing the count of files to restore and the total size of those files. - public async Task> PrepareRestoreFilelistAsync(DateTime restoretime, long[] versions, IFilter filter, CancellationToken token) + public async Task> PrepareRestoreFilelistAsync(DateTime restoretime, long[] versions, IFilter filter, bool disableAdsRestore, CancellationToken token) { m_tempfiletable = $"Fileset-{m_temptabsetguid}"; m_tempblocktable = $"Blocks-{m_temptabsetguid}"; + var enableAds = !disableAdsRestore && SystemIO.IO_OS.SupportsAlternateDataStreams; + await using (var cmd = m_connection.CreateCommand()) { cmd.SetTransaction(m_rtr); @@ -657,7 +660,15 @@ namespace Duplicati.Library.Main.Database { rd.GetValues(values); var path = values[1] as string; - if (path != null && FilterExpression.Matches(filter, path.ToString()!)) + + if (path == null) + continue; + + var matches = FilterExpression.Matches(filter, path); + if (!matches && enableAds && SystemIO.IO_OS.IsAlternateDataStream(path)) + matches = FilterExpression.Matches(filter, SystemIO.IO_OS.GetAlternateDataStreamParent(path)); + + if (matches) { await cmd2 .SetParameterValue("@ID", values[0]) @@ -671,6 +682,10 @@ namespace Duplicati.Library.Main.Database } } + // Ensure ADS targets exist before we attempt to write them + if (enableAds) + await EnsureAlternateDataStreamParentsAsync(filesetId, cmd, token).ConfigureAwait(false); + //creating indexes after insertion is much faster await cmd.ExecuteNonQueryAsync($@" CREATE INDEX ""{m_tempfiletable}_ID"" @@ -724,6 +739,93 @@ namespace Duplicati.Library.Main.Database return new Tuple(0, 0); } + /// + /// Ensures that the parent file or folder entry of any selected alternate data stream + /// is also present in the temporary file table for the given fileset. Alternate data + /// streams are stored as <parent>:<stream> entries and a filter that targets + /// the parent file or folder will not match them directly (and vice versa); this makes + /// sure both ends are restored together. + /// + /// The fileset being restored. + /// The command used for reading the current selection. + /// A cancellation token to monitor for cancellation requests. + private async Task EnsureAlternateDataStreamParentsAsync(long filesetId, SqliteCommand cmd, CancellationToken token) + { + // Nothing to do on platforms that do not support alternate data streams. + if (!SystemIO.IO_OS.SupportsAlternateDataStreams) + return; + + // Collect the parent paths of any alternate data streams currently selected. + // Folder entries are stored with a trailing directory separator ("folder\"), while + // file entries are not ("file.txt"), so include both forms as lookup candidates. + var dirsep = Util.DirectorySeparatorString; + var parentPaths = new HashSet( + Library.Utility.Utility.IsFSCaseSensitive + ? StringComparer.Ordinal + : StringComparer.OrdinalIgnoreCase); + + cmd.SetCommandAndParameters($@" + SELECT ""Path"" + FROM ""{m_tempfiletable}"" + "); + await using (var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false)) + while (await rd.ReadAsync(token).ConfigureAwait(false)) + { + var path = rd.ConvertValueToString(0); + if (path != null && SystemIO.IO_OS.IsAlternateDataStream(path)) + { + // At this point, we do not know if the ADS target is a file or folder + // On Windows, if the ADS target is a folder, the path does not end with a trailing backslash, + // but Duplicati stores all directory paths with a trailing backslash. + // To ensure we match folders, we append the trailing backslash to the path, + // even though this gives a false positive for files with the same path as a folder, + // but that *should* not happen, and side-effects are minimal. + var parent = SystemIO.IO_OS.GetAlternateDataStreamParent(path); + parentPaths.Add(parent); + parentPaths.Add(parent + dirsep); + } + } + + if (parentPaths.Count == 0) + return; + + // Push the candidate paths into a temporary table and let the database perform the + // entire insert in a single statement, which is much faster for larger sets. + await using var parentTable = await TemporaryDbValueList.CreateAsync(this, parentPaths, token).ConfigureAwait(false); + + await using var insertCmd = m_connection.CreateCommand($@" + INSERT INTO ""{m_tempfiletable}"" ( + ""ID"", + ""Path"", + ""BlocksetID"", + ""MetadataID"", + ""DataVerified"" + ) + SELECT + ""File"".""ID"", + ""File"".""Path"", + ""File"".""BlocksetID"", + ""File"".""MetadataID"", + 0 + FROM + ""File"", + ""FilesetEntry"" + WHERE + ""File"".""ID"" = ""FilesetEntry"".""FileID"" + AND ""FilesetEntry"".""FilesetID"" = @FilesetId + AND ""File"".""Path"" IN (@ParentPaths) + AND ""File"".""ID"" NOT IN ( + SELECT ""ID"" + FROM ""{m_tempfiletable}"" + ) + ") + .SetTransaction(m_rtr) + .SetParameterValue("@FilesetId", filesetId); + + await insertCmd.ExpandInClauseParameterMssqliteAsync("@ParentPaths", parentTable, token).ConfigureAwait(false); + await insertCmd.ExecuteNonQueryAsync(token).ConfigureAwait(false); + } + /// /// Removes alternate data stream entries from the temporary file table. /// diff --git a/Duplicati/Library/Main/Operation/RestoreHandler.cs b/Duplicati/Library/Main/Operation/RestoreHandler.cs index 047748710..37c91f054 100644 --- a/Duplicati/Library/Main/Operation/RestoreHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreHandler.cs @@ -500,7 +500,9 @@ namespace Duplicati.Library.Main.Operation } else if (m_result.RestoredFiles == 0) { - if (m_result.UnmodifiedFiles == 0) + if (m_result.UnmodifiedFiles == 0 && m_result.RestoredFolders == 0) + Logging.Log.WriteWarningMessage(LOGTAG, "NoFilesOrFoldersRestored", null, "Restore completed without errors but no files or folders were restored"); + else if (m_result.UnmodifiedFiles == 0) Logging.Log.WriteWarningMessage(LOGTAG, "NoFilesRestored", null, "Restore completed without errors but no files were restored"); else Logging.Log.WriteInformationMessage(LOGTAG, "NoFilesNeededRestore", null, "Restore completed but all files were already present"); @@ -606,7 +608,9 @@ namespace Duplicati.Library.Main.Operation if (m_result.RestoredFiles == 0) { - if (m_result.UnmodifiedFiles == 0) + if (m_result.UnmodifiedFiles == 0 && m_result.RestoredFolders == 0) + Logging.Log.WriteWarningMessage(LOGTAG, "NoFilesOrFoldersRestored", null, "Restore completed without errors but no files or folders were restored"); + else if (m_result.UnmodifiedFiles == 0) Logging.Log.WriteWarningMessage(LOGTAG, "NoFilesRestored", null, "Restore completed without errors but no files were restored"); else Logging.Log.WriteInformationMessage(LOGTAG, "NoFilesNeededRestore", null, "Restore completed but all files were already present"); @@ -1061,7 +1065,7 @@ namespace Duplicati.Library.Main.Operation using (new Logging.Timer(LOGTAG, "PrepareRestoreFileList", "PrepareRestoreFileList")) { var c = await database - .PrepareRestoreFilelistAsync(options.Time, options.Version, filter, result.TaskControl.ProgressToken) + .PrepareRestoreFilelistAsync(options.Time, options.Version, filter, options.DisableAdsRestore, result.TaskControl.ProgressToken) .ConfigureAwait(false); result.OperationProgressUpdater.UpdatefileCount(c.Item1, c.Item2, true); } diff --git a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs index 4b5902dee..908445b15 100644 --- a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs +++ b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs @@ -164,18 +164,15 @@ namespace Duplicati.Library.Snapshots foreach (var p in tmp) res.Add(SystemIOWindows.RemoveExtendedDevicePathPrefix(p)); - if (!m_enableAdsBackup) + if (!m_enableAdsBackup || !SystemIO.IO_OS.SupportsAlternateDataStreams) return res; // Expand ADS for each file - var expanded = ExpandAlternateDataStreams(res, SystemIO.IO_OS.EnumerateAlternateDataStreams).ToList(); + var expanded = ExpandAlternateDataStreams(res, SystemIO.IO_OS.EnumerateAlternateDataStreams); // Also include ADS on the parent folder itself var folderPath = localFolderPath.TrimEnd(Path.DirectorySeparatorChar); - foreach (var stream in SystemIO.IO_OS.EnumerateAlternateDataStreams(folderPath)) - expanded.Add(folderPath + stream); - - return expanded; + return expanded.Concat(SystemIO.IO_OS.EnumerateAlternateDataStreams(folderPath).Select(stream => folderPath + stream)); } /// diff --git a/Duplicati/Library/Snapshots/SnapshotBase.cs b/Duplicati/Library/Snapshots/SnapshotBase.cs index 58823ac29..d1d58dcd7 100644 --- a/Duplicati/Library/Snapshots/SnapshotBase.cs +++ b/Duplicati/Library/Snapshots/SnapshotBase.cs @@ -28,6 +28,7 @@ using Duplicati.Library.Interface; using System.Threading.Tasks; using System.Threading; using Duplicati.Library.Common.IO; +using System.Linq; namespace Duplicati.Library.Snapshots { @@ -231,14 +232,11 @@ namespace Duplicati.Library.Snapshots /// The expanded list of files protected static IEnumerable ExpandAlternateDataStreams(IEnumerable files, Func> enumerateStreams) { - var result = new List(); - foreach (var file in files) - { - result.Add(file); - foreach (var stream in enumerateStreams(file)) - result.Add(file + stream); - } - return result; + // With no ADS support, just return the original list + if (!SystemIO.IO_OS.SupportsAlternateDataStreams) + return files; + + return files.SelectMany(file => new[] { file }.Concat(enumerateStreams(file).Select(stream => file + stream))); } #region IDisposable interface diff --git a/Duplicati/Library/Snapshots/WindowsSnapshot.cs b/Duplicati/Library/Snapshots/WindowsSnapshot.cs index 2269958c0..4e08bb397 100644 --- a/Duplicati/Library/Snapshots/WindowsSnapshot.cs +++ b/Duplicati/Library/Snapshots/WindowsSnapshot.cs @@ -202,19 +202,17 @@ namespace Duplicati.Library.Snapshots for (var i = 0; i < files.Length; i++) files[i] = root + SystemIOWindows.AddExtendedDevicePathPrefix(files[i]).Substring(volumePath.Length); - if (!_enableAdsBackup) + if (!_enableAdsBackup || !SystemIO.IO_OS.SupportsAlternateDataStreams) return files; // Expand ADS for each file - var expanded = ExpandAlternateDataStreams(files, f => SystemIO.IO_OS.EnumerateAlternateDataStreams(ConvertToSnapshotPath(f))).ToList(); + var expanded = ExpandAlternateDataStreams(files, f => SystemIO.IO_OS.EnumerateAlternateDataStreams(ConvertToSnapshotPath(f))); // Also include ADS on the parent folder itself var folderPath = localFolderPath.TrimEnd(Path.DirectorySeparatorChar); var folderShadowPath = ConvertToSnapshotPath(folderPath); - foreach (var stream in SystemIO.IO_OS.EnumerateAlternateDataStreams(folderShadowPath)) - expanded.Add(folderPath + stream); - return expanded; + return expanded.Concat(SystemIO.IO_OS.EnumerateAlternateDataStreams(folderShadowPath).Select(stream => folderPath + stream)); } #endregion From da83bf1cd20a121e4cb0efb4a741e3673ed3ae79 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 15 Jun 2026 15:45:51 +0200 Subject: [PATCH 03/16] Fixed a few failing tests --- Duplicati/UnitTest/BasicSetupHelper.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/UnitTest/BasicSetupHelper.cs b/Duplicati/UnitTest/BasicSetupHelper.cs index 4fe16f4f5..1a0e6852e 100644 --- a/Duplicati/UnitTest/BasicSetupHelper.cs +++ b/Duplicati/UnitTest/BasicSetupHelper.cs @@ -37,7 +37,7 @@ namespace Duplicati.UnitTest /// protected static readonly string BASEFOLDER = Path.GetFullPath( string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("UNITTEST_BASEFOLDER")) - ? "duplicati_testdata" + ? "C:\\duplicati_testdata" : Environment.GetEnvironmentVariable("UNITTEST_BASEFOLDER") ); From 1f3f4976f2a00691546a8889cd2d9db1ded7f154 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 15 Jun 2026 15:48:23 +0200 Subject: [PATCH 04/16] Added support for ADS in TargetCalculation --- Duplicati/Library/Common/IO/Util.cs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/Duplicati/Library/Common/IO/Util.cs b/Duplicati/Library/Common/IO/Util.cs index 23653d29f..2cc3b7a8f 100644 --- a/Duplicati/Library/Common/IO/Util.cs +++ b/Duplicati/Library/Common/IO/Util.cs @@ -165,10 +165,14 @@ namespace Duplicati.Library.Common.IO { if (string.IsNullOrEmpty(path)) return path; - var current = Path.GetFullPath(path); + // 1. Separate the Alternate Data Stream suffix if it exists + var basePath = SystemIO.IO_OS.GetAlternateDataStreamParent(path); + var streamSuffix = path.Substring(basePath.Length); + + var current = Path.GetFullPath(basePath); var ghostSegments = new Stack(); - // 1. Walk up the tree until we find a segment that exists on disk + // 2. Walk up the tree until we find a segment that exists on disk while (!Path.Exists(current)) { var parent = Path.GetDirectoryName(current); @@ -183,7 +187,7 @@ namespace Duplicati.Library.Common.IO current = parent; } - // 2. Resolve symlinks for the part of the path that actually exists + // 3. Resolve symlinks for the part of the path that actually exists // Skip symlink resolution for Windows drive roots (e.g., "C:\") as // ResolveLinkTarget can throw DirectoryNotFoundException on some systems var resolvedPath = current; @@ -203,11 +207,12 @@ namespace Duplicati.Library.Common.IO } } - // 3. Re-attach the non-existent segments to the resolved base path + // 4. Re-attach the non-existent segments to the resolved base path while (ghostSegments.Count > 0) resolvedPath = Path.Combine(resolvedPath, ghostSegments.Pop()); - return resolvedPath; + // 5. Re-attach the alternate data stream suffix at the very end + return resolvedPath + streamSuffix; } } } From 2c64191c399ed2fc093ad5afb68d0503f2708f0d Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 15 Jun 2026 15:58:50 +0200 Subject: [PATCH 05/16] Fixes to tests --- .../UnitTest/AlternateDataStreamTests.cs | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/Duplicati/UnitTest/AlternateDataStreamTests.cs b/Duplicati/UnitTest/AlternateDataStreamTests.cs index fd92c25da..68aba9afc 100644 --- a/Duplicati/UnitTest/AlternateDataStreamTests.cs +++ b/Duplicati/UnitTest/AlternateDataStreamTests.cs @@ -84,7 +84,7 @@ namespace Duplicati.UnitTest [Test] [Category("Windows")] - public async Task BackupAndRestoreAlternateDataStreamsAsync() + public async Task BackupAndRestoreAlternateDataStreamsAsync([Values(true, false)] bool useLegacyRestore) { if (!OperatingSystem.IsWindows()) { @@ -98,7 +98,8 @@ namespace Duplicati.UnitTest var backupOptions = new Dictionary(TestOptions) { - ["enable-ads-backup"] = "true" + ["enable-ads-backup"] = "true", + ["restore-legacy"] = useLegacyRestore.ToString() }; using (var c = new Controller("file://" + TARGETFOLDER, backupOptions, null)) @@ -110,8 +111,11 @@ namespace Duplicati.UnitTest ["restore-path"] = RESTOREFOLDER }; + if (Directory.Exists(RESTOREFOLDER)) + Directory.Delete(RESTOREFOLDER); + using (var c = new Controller("file://" + TARGETFOLDER, restoreOptions, null)) - TestUtils.AssertResults(await c.RestoreAsync(new[] { filePath })); + TestUtils.AssertResults(await c.RestoreAsync(new[] { "*" })); var restoredFilePath = Path.Combine(RESTOREFOLDER, "testfile.txt"); Assert.IsTrue(File.Exists(restoredFilePath)); @@ -290,6 +294,8 @@ namespace Duplicati.UnitTest Directory.CreateDirectory(dirPath); CreateAlternateDataStream(dirPath, "hiddenstream", "hidden content"); + await File.WriteAllTextAsync(Path.Combine(DATAFOLDER, "onefile.txt"), "content"); + var backupOptions = new Dictionary(TestOptions) { ["enable-ads-backup"] = "true" @@ -305,7 +311,7 @@ namespace Duplicati.UnitTest }; using (var c = new Controller("file://" + TARGETFOLDER, restoreOptions, null)) - TestUtils.AssertResults(await c.RestoreAsync(new[] { dirPath })); + TestUtils.AssertResults(await c.RestoreAsync(new[] { "*" })); var restoredDirPath = Path.Combine(RESTOREFOLDER, "testdir"); Assert.IsTrue(Directory.Exists(restoredDirPath)); @@ -363,6 +369,8 @@ namespace Duplicati.UnitTest Directory.CreateDirectory(dirPath); CreateAlternateDataStream(dirPath, "hiddenstream", "hidden content"); + await File.WriteAllTextAsync(Path.Combine(DATAFOLDER, "onefile.txt"), "content"); + var backupOptions = new Dictionary(TestOptions) { ["enable-ads-backup"] = "true" @@ -379,7 +387,7 @@ namespace Duplicati.UnitTest }; using (var c = new Controller("file://" + TARGETFOLDER, restoreOptions, null)) - TestUtils.AssertResults(await c.RestoreAsync(new[] { dirPath })); + TestUtils.AssertResults(await c.RestoreAsync(new[] { "*" })); var restoredDirPath = Path.Combine(RESTOREFOLDER, "testdir"); Assert.IsTrue(Directory.Exists(restoredDirPath)); @@ -400,6 +408,8 @@ namespace Duplicati.UnitTest Directory.CreateDirectory(dirPath); CreateAlternateDataStream(dirPath, "hiddenstream", "hidden content"); + await File.WriteAllTextAsync(Path.Combine(DATAFOLDER, "onefile.txt"), "content"); + var backupOptions = new Dictionary(TestOptions); // enable-ads-backup is not set (defaults to false) @@ -412,7 +422,7 @@ namespace Duplicati.UnitTest }; using (var c = new Controller("file://" + TARGETFOLDER, restoreOptions, null)) - TestUtils.AssertResults(await c.RestoreAsync(new[] { dirPath })); + TestUtils.AssertResults(await c.RestoreAsync(new[] { "*" })); var restoredDirPath = Path.Combine(RESTOREFOLDER, "testdir"); Assert.IsTrue(Directory.Exists(restoredDirPath)); From 980f80084ceeeaed36d60b494f8bd37c08bd4420 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 15 Jun 2026 16:10:26 +0200 Subject: [PATCH 06/16] Handle case where ADS was restored before the host entry. --- .../Main/Operation/Restore/FileLister.cs | 23 +++++++++++++++---- .../Main/Operation/Restore/FileProcessor.cs | 17 ++++++++++---- .../Main/Operation/Restore/Interfaces.cs | 4 +++- 3 files changed, 34 insertions(+), 10 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/FileLister.cs b/Duplicati/Library/Main/Operation/Restore/FileLister.cs index d167dbee9..344378673 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileLister.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileLister.cs @@ -81,16 +81,23 @@ namespace Duplicati.Library.Main.Operation.Restore result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_DownloadingRemoteFiles); sw_get_files?.Stop(); - if (options.DisableAdsRestore) + // Separate out alternate data streams so they are restored last + var adsStreams = new List(); + if (SystemIO.IO_OS.SupportsAlternateDataStreams) { - var newlist = new List(files.Count()); + var hostFiles = new List(); foreach (var f in files) if (SystemIO.IO_OS.IsAlternateDataStream(f.TargetPath)) - Logging.Log.WriteVerboseMessage(LOGTAG, "SkipAdsRestore", "Skipping ADS restore for {0}", f.TargetPath); + { + if (options.DisableAdsRestore) + Logging.Log.WriteVerboseMessage(LOGTAG, "SkipAdsRestore", "Skipping ADS restore for {0}", f.TargetPath); + else + adsStreams.Add(f); + } else - newlist.Add(f); + hostFiles.Add(f); - files = newlist; + files = hostFiles; } sw_write_file?.Start(); @@ -137,6 +144,12 @@ namespace Duplicati.Library.Main.Operation.Restore await self.Output.WriteAsync(folder).ConfigureAwait(false); sw_write_folder?.Stop(); } + + // Send the alternate data streams last, so their hosts are restored + sw_write_file?.Start(); + foreach (var file in adsStreams) + await self.Output.WriteAsync(new FileRequest(file.ID, file.OriginalPath, file.TargetPath, file.Hash, file.Length, file.BlocksetID, IsAlternateDataStream: true)).ConfigureAwait(false); + sw_write_file?.Stop(); } catch (Exception ex) { diff --git a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs index e66fc2cef..cb68e77a6 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs @@ -129,7 +129,10 @@ namespace Duplicati.Library.Main.Operation.Restore Logging.Log.WriteExplicitMessage(LOGTAG, "FileRestored", null, "{0} Restoring file {1}", my_id, file.TargetPath); - if (file.BlocksetID == LocalDatabase.FOLDER_BLOCKSET_ID && !options.SkipMetadata) + // Alternate data streams and folder metadata are restored only after all + // main file content has been restored. + // We rendezvous with all other processors to guarantee the main content is complete. + if ((file.BlocksetID == LocalDatabase.FOLDER_BLOCKSET_ID && !options.SkipMetadata) || file.IsAlternateDataStream) { // Check if there are other FileProcessor's still restoring files if (!decremented) @@ -142,10 +145,16 @@ namespace Duplicati.Library.Main.Operation.Restore decremented = true; } - await RestoreMetadataAsync(db, file, restoreDestination, block_request, block_response, options, sw_meta, sw_work_meta, sw_req, sw_resp, results.TaskControl.ProgressToken) - .ConfigureAwait(false); + if (file.BlocksetID == LocalDatabase.FOLDER_BLOCKSET_ID) + { + if (!options.SkipMetadata) + await RestoreMetadataAsync(db, file, restoreDestination, block_request, block_response, options, sw_meta, sw_work_meta, sw_req, sw_resp, results.TaskControl.ProgressToken) + .ConfigureAwait(false); - continue; + continue; + } + + // ADS stream: fall through to normal file restoration now that the host is restored. } // Get information about the blocks for the file diff --git a/Duplicati/Library/Main/Operation/Restore/Interfaces.cs b/Duplicati/Library/Main/Operation/Restore/Interfaces.cs index 33ccc1191..5927be71d 100644 --- a/Duplicati/Library/Main/Operation/Restore/Interfaces.cs +++ b/Duplicati/Library/Main/Operation/Restore/Interfaces.cs @@ -176,7 +176,8 @@ namespace Duplicati.Library.Main.Operation.Restore /// The length of the file. /// The BlocksetID of the file. /// Whether this is a priority file that should be processed before other files. - public class FileRequest(long ID, string OriginalPath, string TargetPath, string Hash, long Length, long BlocksetID, bool IsPriorityFile = false) + /// Whether this request is for an alternate data stream that must be restored after its host file/folder. + public class FileRequest(long ID, string OriginalPath, string TargetPath, string Hash, long Length, long BlocksetID, bool IsPriorityFile = false, bool IsAlternateDataStream = false) { public long ID { get; } = ID; public string OriginalPath { get; } = OriginalPath; @@ -185,6 +186,7 @@ namespace Duplicati.Library.Main.Operation.Restore public long Length { get; } = Length; public long BlocksetID { get; } = BlocksetID; public bool IsPriorityFile { get; } = IsPriorityFile; + public bool IsAlternateDataStream { get; } = IsAlternateDataStream; } } \ No newline at end of file From a406c418ab67486544d43b2f4fdd4d3670e0b2ba Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 15 Jun 2026 16:12:28 +0200 Subject: [PATCH 07/16] Expand test to cover directory ADS --- Duplicati/UnitTest/AlternateDataStreamTests.cs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Duplicati/UnitTest/AlternateDataStreamTests.cs b/Duplicati/UnitTest/AlternateDataStreamTests.cs index 68aba9afc..8651c2dbc 100644 --- a/Duplicati/UnitTest/AlternateDataStreamTests.cs +++ b/Duplicati/UnitTest/AlternateDataStreamTests.cs @@ -96,6 +96,10 @@ namespace Duplicati.UnitTest File.WriteAllText(filePath, "main content"); CreateAlternateDataStream(filePath, "hiddenstream", "hidden content"); + var dirPath = Path.Combine(DATAFOLDER, "testdir"); + Directory.CreateDirectory(dirPath); + CreateAlternateDataStream(dirPath, "dirstream", "dir hidden content"); + var backupOptions = new Dictionary(TestOptions) { ["enable-ads-backup"] = "true", @@ -122,6 +126,11 @@ namespace Duplicati.UnitTest Assert.AreEqual("main content", File.ReadAllText(restoredFilePath)); Assert.IsTrue(AlternateDataStreamExists(restoredFilePath, "hiddenstream")); Assert.AreEqual("hidden content", ReadAlternateDataStream(restoredFilePath, "hiddenstream")); + + var restoredDirPath = Path.Combine(RESTOREFOLDER, "testdir"); + Assert.IsTrue(Directory.Exists(restoredDirPath)); + Assert.IsTrue(AlternateDataStreamExists(restoredDirPath, "dirstream")); + Assert.AreEqual("dir hidden content", ReadAlternateDataStream(restoredDirPath, "dirstream")); } [Test] From a3741d559c7271dfd264ca2756b6050a2e752302 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 15 Jun 2026 16:24:50 +0200 Subject: [PATCH 08/16] Revert testing change --- Duplicati/UnitTest/BasicSetupHelper.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/UnitTest/BasicSetupHelper.cs b/Duplicati/UnitTest/BasicSetupHelper.cs index 1a0e6852e..4fe16f4f5 100644 --- a/Duplicati/UnitTest/BasicSetupHelper.cs +++ b/Duplicati/UnitTest/BasicSetupHelper.cs @@ -37,7 +37,7 @@ namespace Duplicati.UnitTest /// protected static readonly string BASEFOLDER = Path.GetFullPath( string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("UNITTEST_BASEFOLDER")) - ? "C:\\duplicati_testdata" + ? "duplicati_testdata" : Environment.GetEnvironmentVariable("UNITTEST_BASEFOLDER") ); From aea61969e4601a7d50a5346b820da1f773de9c63 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 15 Jun 2026 17:55:08 +0200 Subject: [PATCH 09/16] Cleaned up problematic path tests to return warning messages if seen --- Duplicati/UnitTest/ProblematicPathTests.cs | 77 +++++----------------- 1 file changed, 18 insertions(+), 59 deletions(-) diff --git a/Duplicati/UnitTest/ProblematicPathTests.cs b/Duplicati/UnitTest/ProblematicPathTests.cs index 13a0c6e02..106465eed 100644 --- a/Duplicati/UnitTest/ProblematicPathTests.cs +++ b/Duplicati/UnitTest/ProblematicPathTests.cs @@ -89,31 +89,23 @@ namespace Duplicati.UnitTest // Backup all files. var options = new Dictionary(this.TestOptions); using (var c = new Controller("file://" + this.TARGETFOLDER, options, null)) - { - var backupResults = await c.BackupAsync(new[] { this.DATAFOLDER }); - Assert.AreEqual(0, backupResults.Errors.Count()); - Assert.AreEqual(0, backupResults.Warnings.Count()); - } + TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER })); // Restore all files. var restoreOptions = new Dictionary(options) { ["restore-path"] = this.RESTOREFOLDER }; using (var c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null)) { - var restoreResults = await c.RestoreAsync(null); - Assert.AreEqual(0, restoreResults.Errors.Count()); - Assert.AreEqual(0, restoreResults.Warnings.Count()); + TestUtils.AssertResults(await c.RestoreAsync(null)); TestUtils.AssertDirectoryTreesAreEquivalent(this.DATAFOLDER, this.RESTOREFOLDER, true, "Restore"); // List results using * should return a match for each directory. var listResults = await c.ListAsync(SystemIO.IO_OS.PathCombine(dirWithAsterisk, file)); - Assert.AreEqual(0, listResults.Errors.Count()); - Assert.AreEqual(0, listResults.Warnings.Count()); + TestUtils.AssertResults(listResults); Assert.AreEqual(directories.Count, listResults.Files.Count()); listResults = await c.ListAsync(SystemIO.IO_OS.PathCombine(dirWithQuestionMark, file)); - Assert.AreEqual(0, listResults.Errors.Count()); - Assert.AreEqual(0, listResults.Warnings.Count()); + TestUtils.AssertResults(listResults); // List results using ? should return 3 matches in Linux, // one for the directory with '*' and one for the directory // with '?', plus one for directory 'X'; but should return @@ -134,14 +126,11 @@ namespace Duplicati.UnitTest // Verify that list result using verbatim identifier contains only one file. var listResults = await c.ListAsync(verbatimFilePath); - Assert.AreEqual(0, listResults.Errors.Count()); - Assert.AreEqual(0, listResults.Warnings.Count()); + TestUtils.AssertResults(listResults); Assert.AreEqual(1, listResults.Files.Count()); Assert.AreEqual(expectedFilePath, listResults.Files.Single().Path); - var restoreResults = await c.RestoreAsync(new[] { verbatimFilePath }); - Assert.AreEqual(0, restoreResults.Errors.Count()); - Assert.AreEqual(0, restoreResults.Warnings.Count()); + TestUtils.AssertResults(await c.RestoreAsync(new[] { verbatimFilePath })); var fileName = SystemIO.IO_OS.PathGetFileName(expectedFilePath); var restoredFilePath = SystemIO.IO_OS.PathCombine(this.RESTOREFOLDER, fileName); @@ -157,8 +146,7 @@ namespace Duplicati.UnitTest using (var c = new Controller("file://" + this.TARGETFOLDER, options, null)) { var backupResults = await c.BackupAsync(new[] { this.DATAFOLDER }, filter); - Assert.AreEqual(0, backupResults.Errors.Count()); - Assert.AreEqual(0, backupResults.Warnings.Count()); + TestUtils.AssertResults(backupResults); Assert.AreEqual(directories.Count, backupResults.ExaminedFiles); } @@ -171,8 +159,7 @@ namespace Duplicati.UnitTest using (var c = new Controller("file://" + this.TARGETFOLDER, options, null)) { var backupResults = await c.BackupAsync(new[] { this.DATAFOLDER }, filter); - Assert.AreEqual(0, backupResults.Errors.Count()); - Assert.AreEqual(0, backupResults.Warnings.Count()); + TestUtils.AssertResults(backupResults); Assert.AreEqual(verbatimAsteriskShouldMatchCount, backupResults.ExaminedFiles); } } @@ -213,17 +200,12 @@ namespace Duplicati.UnitTest var options = new Dictionary(this.TestOptions); using (var c = new Controller("file://" + this.TARGETFOLDER, options, null)) - { - var backupResults = await c.BackupAsync(new[] { this.DATAFOLDER }, filter); - Assert.AreEqual(0, backupResults.Errors.Count()); - Assert.AreEqual(0, backupResults.Warnings.Count()); - } + TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER }, filter)); using (var c = new Controller("file://" + this.TARGETFOLDER, options, null)) { var listResults = await c.ListAsync("*"); - Assert.AreEqual(0, listResults.Errors.Count()); - Assert.AreEqual(0, listResults.Warnings.Count()); + TestUtils.AssertResults(listResults); var backedUpPaths = listResults.Files.Select(x => x.Path).ToArray(); Assert.AreEqual(2, backedUpPaths.Length); @@ -246,28 +228,18 @@ namespace Duplicati.UnitTest var options = new Dictionary(this.TestOptions); using (var c = new Controller("file://" + this.TARGETFOLDER, options, null)) - { - var backupResults = await c.BackupAsync(new[] { this.DATAFOLDER }); - Assert.AreEqual(0, backupResults.Errors.Count()); - Assert.AreEqual(0, backupResults.Warnings.Count()); - } + TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER })); var restoreOptions = new Dictionary(this.TestOptions) { ["restore-path"] = this.RESTOREFOLDER }; using (var c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null)) - { - var restoreResults = await c.RestoreAsync(new[] { filePath }); - Assert.AreEqual(0, restoreResults.Errors.Count()); - Assert.AreEqual(0, restoreResults.Warnings.Count()); - } + TestUtils.AssertResults(await c.RestoreAsync(new[] { filePath })); var restoreFilePath = SystemIO.IO_OS.PathCombine(this.RESTOREFOLDER, fileName); Assert.IsTrue(SystemIO.IO_OS.FileExists(restoreFilePath)); var restoredStream = new MemoryStream(); - using (FileStream fileStream = SystemIO.IO_OS.FileOpenRead(restoreFilePath)) - { + using (var fileStream = SystemIO.IO_OS.FileOpenRead(restoreFilePath)) Utility.CopyStream(fileStream, restoredStream); - } Assert.AreEqual(fileBytes, restoredStream.ToArray()); } @@ -282,9 +254,8 @@ namespace Duplicati.UnitTest public async Task ProblematicSuffixesAsync(string pathComponent, bool skipOnWindows) { if (OperatingSystem.IsWindows() && skipOnWindows) - { - return; - } + Assert.Ignore("This test is not for Windows."); + var folderPath = SystemIO.IO_OS.PathCombine(this.DATAFOLDER, pathComponent); SystemIO.IO_OS.DirectoryCreate(folderPath); @@ -295,21 +266,13 @@ namespace Duplicati.UnitTest var options = new Dictionary(this.TestOptions); using (var c = new Controller("file://" + this.TARGETFOLDER, options, null)) - { - var backupResults = await c.BackupAsync(new[] { this.DATAFOLDER }); - Assert.AreEqual(0, backupResults.Errors.Count()); - Assert.AreEqual(0, backupResults.Warnings.Count()); - } + TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER })); var restoreOptions = new Dictionary(this.TestOptions) { ["restore-path"] = this.RESTOREFOLDER }; // Restore just the file. using (var c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null)) - { - var restoreResults = await c.RestoreAsync(new[] { filePath }); - Assert.AreEqual(0, restoreResults.Errors.Count()); - Assert.AreEqual(0, restoreResults.Warnings.Count()); - } + TestUtils.AssertResults(await c.RestoreAsync(new[] { filePath })); var restoreFilePath = SystemIO.IO_OS.PathCombine(this.RESTOREFOLDER, pathComponent); TestUtils.AssertFilesAreEqual(filePath, restoreFilePath, true, pathComponent); @@ -318,11 +281,7 @@ namespace Duplicati.UnitTest // Restore the entire directory. var pathSpec = $"[{Regex.Escape(Util.AppendDirSeparator(this.DATAFOLDER))}.*]"; using (var c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null)) - { - var restoreResults = await c.RestoreAsync(new[] { pathSpec }); - Assert.AreEqual(0, restoreResults.Errors.Count()); - Assert.AreEqual(0, restoreResults.Warnings.Count()); - } + TestUtils.AssertResults(await c.RestoreAsync(new[] { pathSpec })); TestUtils.AssertDirectoryTreesAreEquivalent(this.DATAFOLDER, this.RESTOREFOLDER, true, pathComponent); } From 4aa61b235251fc6f201bb1e1fd0bb71c0e342e9d Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 15 Jun 2026 22:51:13 +0200 Subject: [PATCH 10/16] Avoid using .NET File operations during restore as they normalize paths. --- .../Main/Operation/Restore/FileProcessor.cs | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs index cb68e77a6..c7e634088 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs @@ -811,7 +811,7 @@ namespace Duplicati.Library.Main.Operation.Restore List verified_blocks = []; // Check if the file exists - if (File.Exists(file.TargetPath)) + if (await restoreDestination.FileExists(file.TargetPath, cancellationToken).ConfigureAwait(false)) { filehasher.Initialize(); try @@ -867,17 +867,19 @@ namespace Duplicati.Library.Main.Operation.Restore if (filehasher.Hash != null && Convert.ToBase64String(filehasher.Hash) == file.Hash) { // Truncate the file if it is larger than the expected size. - FileInfo fi = new(file.TargetPath); - if (file.Length < fi.Length) + var currentLength = await restoreDestination.GetFileLength(file.TargetPath, cancellationToken).ConfigureAwait(false); + if (file.Length < currentLength) { if (options.Dryrun) { - Logging.Log.WriteDryrunMessage(LOGTAG, "DryrunRestore", @$"Would have truncated ""{file.TargetPath}"" from {fi.Length} to {file.Length}"); + Logging.Log.WriteDryrunMessage(LOGTAG, "DryrunRestore", @$"Would have truncated ""{file.TargetPath}"" from {currentLength} to {file.Length}"); } else { // Reopen file with write permission - fi.IsReadOnly = false; // The metadata handler will revert this back later. + // The metadata handler will revert this back later. + if (await restoreDestination.HasReadOnlyAttribute(file.TargetPath, cancellationToken).ConfigureAwait(false)) + await restoreDestination.ClearReadOnlyAttribute(file.TargetPath, cancellationToken).ConfigureAwait(false); try { using var f = await restoreDestination.OpenWrite(file.TargetPath, cancellationToken).ConfigureAwait(false); @@ -931,15 +933,14 @@ namespace Duplicati.Library.Main.Operation.Restore List missing_blocks = []; List verified_blocks = []; - // Check if the file exists - if (File.Exists(file.OriginalPath)) + // Check if the file exists (ignore if this was a remote source) + if (SystemIO.IO_OS.FileExists(file.OriginalPath)) { filehasher.Initialize(); // Open both files, as the target file is still being read to produce the overall file hash, if all the blocks are present across both the target and original files. // Note: The original file is the source file on the local system (not within the restore destination), - // so we use File.OpenRead directly instead of restoreDestination.OpenRead - using var f_original = File.OpenRead(file.OriginalPath); + using var f_original = SystemIO.IO_OS.FileOpenRead(file.OriginalPath); using var f_target = options.Dryrun ? (await restoreDestination.FileExists(file.TargetPath, cancellationToken).ConfigureAwait(false) ? await restoreDestination.OpenRead(file.TargetPath, cancellationToken).ConfigureAwait(false) : From c58087db7efd8a81ce9be842be7ebb16d6a337e4 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 16 Jun 2026 10:20:27 +0200 Subject: [PATCH 11/16] Fixed tests to check for errors, instead of crashing the restore process on path traversal checks --- .../Library/Main/Operation/RestoreHandler.cs | 16 ++++++++++++-- .../UnitTest/RestorePathTraversalTests.cs | 21 +++++++++++++++---- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/Duplicati/Library/Main/Operation/RestoreHandler.cs b/Duplicati/Library/Main/Operation/RestoreHandler.cs index 37c91f054..2ff6f5c75 100644 --- a/Duplicati/Library/Main/Operation/RestoreHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreHandler.cs @@ -1173,7 +1173,19 @@ namespace Duplicati.Library.Main.Operation var targetfileid = restorelist.TargetFileID; var targetfilehash = restorelist.TargetHash; var targetfilelength = restorelist.Length; - if (await restoreDestination.FileExists(targetpath, result.TaskControl.ProgressToken).ConfigureAwait(false)) + var fileExists = false; + try + { + fileExists = await restoreDestination.FileExists(targetpath, result.TaskControl.ProgressToken).ConfigureAwait(false); + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "FileExistsFailed", ex, "Failed to check if file exists: \"{0}\", message: {1}", targetpath, ex.Message); + if (options.UnittestMode) + throw; + } + + if (fileExists) { try { @@ -1208,7 +1220,7 @@ namespace Duplicati.Library.Main.Operation { // a file hash for verification will only be necessary if the file has exactly // the wanted size so we have a chance to already mark the file as data-verified. - bool calcFileHash = (currentfilelength == targetfilelength); + var calcFileHash = currentfilelength == targetfilelength; if (calcFileHash) filehasher.Initialize(); using (var file = await restoreDestination.OpenRead(targetpath, result.TaskControl.ProgressToken).ConfigureAwait(false)) diff --git a/Duplicati/UnitTest/RestorePathTraversalTests.cs b/Duplicati/UnitTest/RestorePathTraversalTests.cs index e63c9e49c..0ae32bcc0 100644 --- a/Duplicati/UnitTest/RestorePathTraversalTests.cs +++ b/Duplicati/UnitTest/RestorePathTraversalTests.cs @@ -16,7 +16,7 @@ namespace Duplicati.UnitTest { [Test] [Category("RestoreHandler")] - public async Task RestoreToSymlinkTargetAsync() + public async Task RestoreToSymlinkTargetAsync([Values(true, false)] bool legacyRestore) { if (OperatingSystem.IsWindows()) Assert.Ignore("Symlink tests are not supported on Windows"); @@ -53,12 +53,25 @@ namespace Duplicati.UnitTest var restoreOptions = new Dictionary(this.TestOptions); restoreOptions["restore-path"] = restoreTarget; restoreOptions["overwrite"] = "true"; + restoreOptions["restore-legacy"] = legacyRestore.ToString(); + restoreOptions["unittest-mode"] = "false"; using (var c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null)) { - // We expect a UserInformationException due to path traversal detection - var ex = NUnit.Framework.Assert.ThrowsAsync(() => c.RestoreAsync(null)); - NUnit.Framework.Assert.That(ex.Message.Contains("Path traversal detected"), "Expected path traversal error message"); + // Verify that path traversal is detected + var restoreResults = await c.RestoreAsync(null); + var foundError = restoreResults.Errors.Any(e => e.Contains("Path traversal detected")); + if (!foundError) + { + Console.WriteLine("Errors found:"); + foreach (var e in restoreResults.Errors) + Console.WriteLine(e); + + Console.WriteLine("Warnings found:"); + foreach (var e in restoreResults.Warnings) + Console.WriteLine(e); + } + Assert.IsTrue(foundError, "Expected path traversal error message"); } // 5. Verify vulnerability From bfc4b00d96f349f7c81697291adb0bbfbd7ad4a2 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 16 Jun 2026 12:03:55 +0200 Subject: [PATCH 12/16] Added quick exit if no files match when restoring --- Duplicati/Library/Main/Operation/RestoreHandler.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Duplicati/Library/Main/Operation/RestoreHandler.cs b/Duplicati/Library/Main/Operation/RestoreHandler.cs index 2ff6f5c75..b7ae4688e 100644 --- a/Duplicati/Library/Main/Operation/RestoreHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreHandler.cs @@ -1068,6 +1068,10 @@ namespace Duplicati.Library.Main.Operation .PrepareRestoreFilelistAsync(options.Time, options.Version, filter, options.DisableAdsRestore, result.TaskControl.ProgressToken) .ConfigureAwait(false); result.OperationProgressUpdater.UpdatefileCount(c.Item1, c.Item2, true); + + // If there are no files to restore, just stop now, most likely not what is desired + if (c.Item1 == 0) + throw new UserInformationException("Restore selection matched zero files, nothing to restore", "EmptyRestoreOperation"); } if (options.DisableAdsRestore) From a6071a66d87925ea022b172b8042e67135c4259e Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 16 Jun 2026 12:04:29 +0200 Subject: [PATCH 13/16] Updated test to check that backup actually sees problematic files --- Duplicati/UnitTest/ProblematicPathTests.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Duplicati/UnitTest/ProblematicPathTests.cs b/Duplicati/UnitTest/ProblematicPathTests.cs index 106465eed..46a7faac8 100644 --- a/Duplicati/UnitTest/ProblematicPathTests.cs +++ b/Duplicati/UnitTest/ProblematicPathTests.cs @@ -266,7 +266,11 @@ namespace Duplicati.UnitTest var options = new Dictionary(this.TestOptions); using (var c = new Controller("file://" + this.TARGETFOLDER, options, null)) - TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER })); + { + var r = await c.BackupAsync(new[] { this.DATAFOLDER }); + Assert.AreNotEqual(r.AddedFiles, 0, "No files added"); + TestUtils.AssertResults(r); + } var restoreOptions = new Dictionary(this.TestOptions) { ["restore-path"] = this.RESTOREFOLDER }; From bea9d3a84defc4c5313a2796a9efe6ec8815b12a Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 16 Jun 2026 12:05:00 +0200 Subject: [PATCH 14/16] Fix issue with problematic paths dissapearing on Windows --- Duplicati/Library/Snapshots/NoSnapshotWindows.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs index 908445b15..9c5e4cdb0 100644 --- a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs +++ b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs @@ -138,6 +138,14 @@ namespace Duplicati.Library.Snapshots public override Stream OpenRead(string localPath) => SystemIO.IO_OS.FileOpenRead(localPath); + /// + public override bool DirectoryExists(string localFolderPath) + => SystemIO.IO_OS.DirectoryExists(localFolderPath); + + /// + public override bool FileExists(string localFilePath) + => SystemIO.IO_OS.FileExists(localFilePath); + /// public override ISourceProviderEntry GetFilesystemEntry(string path, bool isFolder) { From 5e2b13cc7b101cf21435c8315cfa3db11ed51d7b Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 16 Jun 2026 12:08:27 +0200 Subject: [PATCH 15/16] Made test logic similar to other places with ignore --- Duplicati/UnitTest/ProblematicPathTests.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Duplicati/UnitTest/ProblematicPathTests.cs b/Duplicati/UnitTest/ProblematicPathTests.cs index 46a7faac8..177c8fd07 100644 --- a/Duplicati/UnitTest/ProblematicPathTests.cs +++ b/Duplicati/UnitTest/ProblematicPathTests.cs @@ -254,8 +254,10 @@ namespace Duplicati.UnitTest public async Task ProblematicSuffixesAsync(string pathComponent, bool skipOnWindows) { if (OperatingSystem.IsWindows() && skipOnWindows) + { Assert.Ignore("This test is not for Windows."); - + return; + } var folderPath = SystemIO.IO_OS.PathCombine(this.DATAFOLDER, pathComponent); SystemIO.IO_OS.DirectoryCreate(folderPath); From d06b217975563216c8f4fb3b4732d62a88adf4ff Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 16 Jun 2026 15:05:44 +0200 Subject: [PATCH 16/16] Improved check for empty restore --- Duplicati/Library/Main/Operation/RestoreHandler.cs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Main/Operation/RestoreHandler.cs b/Duplicati/Library/Main/Operation/RestoreHandler.cs index b7ae4688e..34d4c2de9 100644 --- a/Duplicati/Library/Main/Operation/RestoreHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreHandler.cs @@ -1068,9 +1068,13 @@ namespace Duplicati.Library.Main.Operation .PrepareRestoreFilelistAsync(options.Time, options.Version, filter, options.DisableAdsRestore, result.TaskControl.ProgressToken) .ConfigureAwait(false); result.OperationProgressUpdater.UpdatefileCount(c.Item1, c.Item2, true); - - // If there are no files to restore, just stop now, most likely not what is desired - if (c.Item1 == 0) + + // If the selection is completely empty (no files, folders, or symlinks), + // stop now as this is most likely not what is desired. + var firstPath = await database + .GetFirstPathAsync(result.TaskControl.ProgressToken) + .ConfigureAwait(false); + if (string.IsNullOrWhiteSpace(firstPath)) throw new UserInformationException("Restore selection matched zero files, nothing to restore", "EmptyRestoreOperation"); }