haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #6954 from duplicati/feature/support-ads

Browse Source 
Implement support for ADS
This commit is contained in:
Kenneth Skovhede
2026-06-17 08:58:29 +02:00
committed by GitHub
parent 03f30e3f7d d06b217975
commit 59be19660c
25 changed files with 1122 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/Library/Common/IO/DefineDosDevice.cs
+21 -22
View File
@@ -1,26 +1,25 @@
// Copyright (C) 2026, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
// Copyright (C) 2026, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
namespace Duplicati.Library.Common.IO
@@ -68,7 +67,7 @@ namespace Duplicati.Library.Common.IO
/// <param name="devicename">A pointer to an MS-DOS device name string specifying the device the function is defining, redefining, or deleting. The device name string must not have a colon as the last character, unless a drive letter is being defined, redefined, or deleted. For example, drive C would be the string &quot;C:&quot;. In no case is a trailing backslash (&quot;\&quot;) allowed.</param>
/// <param name="targetpath">A pointer to a path string that will implement this device. The string is an MS-DOS path string unless the DDD_RAW_TARGET_PATH flag is specified, in which case this string is a path string.</param>
/// <returns>True on success, false otherwise</returns>
[System.Runtime.InteropServices.DllImport("kernel32", CharSet = System.Runtime.InteropServices.CharSet.Auto, SetLastError = true)]
[System.Runtime.InteropServices.DllImport("kernel32", CharSet = System.Runtime.InteropServices.CharSet.Auto, SetLastError = true)]
public static extern bool DefineDosDevice(DDD_Flags flags, string devicename, string targetpath);
}
Duplicati/Library/Common/IO/ISystemIO.cs
+28
View File
@@ -92,6 +92,34 @@ namespace Duplicati.Library.Common.IO
/// </summary>
/// <param name="path">The directory to set permissions on.</param>
void DirectorySetPermissionUserRWOnly(string path);
/// <summary>
/// Gets a value indicating whether the operating system supports alternate data streams.
/// </summary>
bool SupportsAlternateDataStreams { get; }
/// <summary>
/// Enumerates the alternate data streams for a file (Windows-only).
/// </summary>
/// <param name="path">The path to the file.</param>
/// <returns>An enumerable of stream names (e.g., ":streamname").</returns>
IEnumerable<string> EnumerateAlternateDataStreams(string path);
/// <summary>
/// Determines whether the specified path is an alternate data stream.
/// </summary>
/// <param name="path">The path to check.</param>
/// <returns>True if the path is an alternate data stream; otherwise, false.</returns>
bool IsAlternateDataStream(string path);
/// <summary>
/// Returns the parent file or folder path of an alternate data stream path.
/// For a path such as <c>C:\file.txt:stream</c> this returns <c>C:\file.txt</c>.
/// If the path is not an alternate data stream, the path is returned unchanged.
/// </summary>
/// <param name="path">The alternate data stream path.</param>
/// <returns>The parent file or folder path.</returns>
string GetAlternateDataStreamParent(string path);
}
}
Duplicati/Library/Common/IO/SystemIOLinux.cs
+15
View File
@@ -402,6 +402,21 @@ namespace Duplicati.Library.Common.IO
Convert.ToInt32("700", 8) /* FilePermissions.S_IRUSR | FilePermissions.S_IWUSR | FilePermissions.S_IXUSR */
);
}
/// <inheritdoc />
public bool SupportsAlternateDataStreams => false;
/// <inheritdoc />
public IEnumerable<string> EnumerateAlternateDataStreams(string path)
=> [];
/// <inheritdoc />
public bool IsAlternateDataStream(string path)
=> false;
/// <inheritdoc />
public string GetAlternateDataStreamParent(string path)
=> path;
}
}
Duplicati/Library/Common/IO/SystemIOWindows.cs
+184 -3
View File
@@ -24,11 +24,14 @@ using System.Collections.Generic;
using System.Security.AccessControl;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
using Duplicati.Library.Interface;
using Newtonsoft.Json;
using System.Runtime.Versioning;
using System.Security.Principal;
using Microsoft.Win32.SafeHandles;
using System.ComponentModel;
namespace Duplicati.Library.Common.IO
{
@@ -44,6 +47,93 @@ namespace Duplicati.Library.Common.IO
private static readonly string DIRSEP = Util.DirectorySeparatorString;
/// <summary>
/// Encapsulation of Win32 calls
/// </summary>
private static class Win32API
{
/// <summary>
/// The stream info levels for FindFirstStreamW
/// </summary>
public enum STREAM_INFO_LEVELS
{
FindStreamInfoStandard = 0,
FindStreamInfoMaxInfoLevel = 1
}
/// <summary>
/// The access mode for CreateFile to only read attributes
/// </summary>
public const uint FILE_READ_ATTRIBUTES = 0x0080;
/// <summary>
/// Open only an existing file
/// </summary>
public const uint OPEN_EXISTING = 3;
/// <summary>
/// The share mode for CreateFile to allow read access
/// </summary>
public const uint FILE_SHARE_READ = 0x00000001;
/// <summary>
/// The share mode for CreateFile to allow write access
/// </summary>
public const uint FILE_SHARE_WRITE = 0x00000002;
/// <summary>
/// The share mode for CreateFile to allow delete access
/// </summary>
public const uint FILE_SHARE_DELETE = 0x00000004;
/// <summary>
/// Creates a file handle
/// </summary>
/// <param name="lpFileName">The filename</param>
/// <param name="dwDesiredAccess">The access mode</param>
/// <param name="dwShareMode">The share mode</param>
/// <param name="lpSecurityAttributes">Pointer to security attributes</param>
/// <param name="dwCreationDisposition">Create mode flags</param>
/// <param name="dwFlagsAndAttributes">Flags and attributes</param>
/// <param name="hTemplateFile">Handle to a template file</param>
/// <returns>A filehandle for the entry</returns>
[DllImport("kernel32.dll", SetLastError = true, CharSet = CharSet.Unicode)]
public static extern SafeFileHandle CreateFileW(
string lpFileName,
uint dwDesiredAccess,
uint dwShareMode,
IntPtr lpSecurityAttributes,
uint dwCreationDisposition,
uint dwFlagsAndAttributes,
IntPtr hTemplateFile);
/// <summary>
/// Gets the file size
/// </summary>
/// <param name="hFile">The file handle</param>
/// <param name="lpFileSize">The size of the file</param>
/// <returns><c>true</c> if the call succeeds, <c>false</c> otherwise
[DllImport("kernel32.dll", SetLastError = true)]
[return: MarshalAs(UnmanagedType.Bool)]
public static extern bool GetFileSizeEx(SafeFileHandle hFile, out long lpFileSize);
/// <summary>
/// The WIN32_FIND_STREAM_DATA structure
/// </summary>
[StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)]
public struct WIN32_FIND_STREAM_DATA
{
public long StreamSize;
[MarshalAs(UnmanagedType.ByValTStr, SizeConst = 296)]
public string cStreamName;
}
[DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)]
public static extern IntPtr FindFirstStreamW(string lpFileName, STREAM_INFO_LEVELS InfoLevel, out WIN32_FIND_STREAM_DATA lpFindStreamData, uint dwFlags);
[DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)]
public static extern bool FindNextStreamW(IntPtr hFindStream, out WIN32_FIND_STREAM_DATA lpFindStreamData);
[DllImport("kernel32.dll", SetLastError = true)]
public static extern bool FindClose(IntPtr hFindFile);
}
/// <summary>
/// The current user SID
/// </summary>
@@ -370,9 +460,6 @@ namespace Duplicati.Library.Common.IO
File.Move(AddExtendedDevicePathPrefix(source), AddExtendedDevicePathPrefix(target));
}
public long FileLength(string path)
=> new FileInfo(AddExtendedDevicePathPrefix(path)).Length;
public string GetPathRoot(string path)
=> IsPrefixedWithExtendedDevicePathPrefix(path)
? Path.GetPathRoot(path)
@@ -540,6 +627,100 @@ namespace Duplicati.Library.Common.IO
public string PathCombine(params string[] paths)
=> Path.Combine(paths);
/// <inheritdoc />
public bool SupportsAlternateDataStreams => true;
/// <inheritdoc />
public bool IsAlternateDataStream(string path)
{
if (string.IsNullOrWhiteSpace(path)) return false;
// Extracts the root component (e.g., "C:\", "C:", "\\server\share\", or "\\?\C:\")
var root = Path.GetPathRoot(path) ?? string.Empty;
// If a colon exists outside of the root structure, it's an alternate data stream
return path.AsSpan(root.Length).Contains(':');
}
/// <inheritdoc />
public string GetAlternateDataStreamParent(string path)
{
if (string.IsNullOrWhiteSpace(path))
return path;
// Extracts the root component (e.g., "C:\", "C:", "\\server\share\", or "\\?\C:\")
var root = Path.GetPathRoot(path) ?? string.Empty;
var relativePath = path.AsSpan(root.Length);
var idx = relativePath.IndexOf(':');
if (idx < 0)
return path;
return root + relativePath.Slice(0, idx).ToString();
}
/// <inheritdoc />
[SupportedOSPlatform("windows")]
public IEnumerable<string> EnumerateAlternateDataStreams(string path)
{
var prefixed = AddExtendedDevicePathPrefix(path);
var data = new Win32API.WIN32_FIND_STREAM_DATA();
IntPtr handle = IntPtr.Zero;
try
{
handle = Win32API.FindFirstStreamW(prefixed, Win32API.STREAM_INFO_LEVELS.FindStreamInfoStandard, out data, 0);
if (handle == new IntPtr(-1))
yield break;
do
{
// Skip the main stream (::$DATA)
if (data.cStreamName == "::$DATA")
continue;
// Clean the stream name by removing the :$DATA suffix
var streamName = data.cStreamName;
if (streamName.EndsWith(":$DATA", StringComparison.OrdinalIgnoreCase))
streamName = streamName.Substring(0, streamName.Length - ":$DATA".Length);
yield return streamName;
}
while (Win32API.FindNextStreamW(handle, out data));
}
finally
{
if (handle != IntPtr.Zero && handle != new IntPtr(-1))
Win32API.FindClose(handle);
}
}
/// <summary>
/// Returns the length of a file or alternate data stream.
/// </summary>
public long FileLength(string path)
{
if (!IsAlternateDataStream(path))
return new FileInfo(AddExtendedDevicePathPrefix(path)).Length;
using SafeFileHandle handle = Win32API.CreateFileW(
AddExtendedDevicePathPrefix(path),
Win32API.FILE_READ_ATTRIBUTES,
Win32API.FILE_SHARE_READ | Win32API.FILE_SHARE_WRITE | Win32API.FILE_SHARE_DELETE, // Broad sharing flags avoid collisions
IntPtr.Zero,
Win32API.OPEN_EXISTING,
0,
IntPtr.Zero);
if (handle.IsInvalid)
throw new Win32Exception(Marshal.GetLastWin32Error(), "Failed to open handle to alternative data stream ");
// Retrieve the size directly from the stream handle
if (!Win32API.GetFileSizeEx(handle, out long streamSize))
throw new Win32Exception(Marshal.GetLastWin32Error(), "Failed to retrieve alternate data stream size.");
return streamSize;
}
public void CreateSymlink(string symlinkfile, string target, bool asDir)
{
if (FileExists(symlinkfile) || DirectoryExists(symlinkfile))
Duplicati/Library/Common/IO/Util.cs
+10 -5
View File
@@ -165,10 +165,14 @@ namespace Duplicati.Library.Common.IO
{
if (string.IsNullOrEmpty(path)) return path;
var current = Path.GetFullPath(path);
// 1. Separate the Alternate Data Stream suffix if it exists
var basePath = SystemIO.IO_OS.GetAlternateDataStreamParent(path);
var streamSuffix = path.Substring(basePath.Length);
var current = Path.GetFullPath(basePath);
var ghostSegments = new Stack<string>();
// 1. Walk up the tree until we find a segment that exists on disk
// 2. Walk up the tree until we find a segment that exists on disk
while (!Path.Exists(current))
{
var parent = Path.GetDirectoryName(current);
@@ -183,7 +187,7 @@ namespace Duplicati.Library.Common.IO
current = parent;
}
// 2. Resolve symlinks for the part of the path that actually exists
// 3. Resolve symlinks for the part of the path that actually exists
// Skip symlink resolution for Windows drive roots (e.g., "C:\") as
// ResolveLinkTarget can throw DirectoryNotFoundException on some systems
var resolvedPath = current;
@@ -203,11 +207,12 @@ namespace Duplicati.Library.Common.IO
}
}
// 3. Re-attach the non-existent segments to the resolved base path
// 4. Re-attach the non-existent segments to the resolved base path
while (ghostSegments.Count > 0)
resolvedPath = Path.Combine(resolvedPath, ghostSegments.Pop());
return resolvedPath;
// 5. Re-attach the alternate data stream suffix at the very end
return resolvedPath + streamSuffix;
}
}
}
Duplicati/Library/Main/Database/LocalRestoreDatabase.cs
+144 -2
View File
@@ -407,13 +407,16 @@ namespace Duplicati.Library.Main.Database
/// <param name="restoretime">The time at which the restore operation is being performed.</param>
/// <param name="versions">An array of version identifiers to filter the filesets.</param>
/// <param name="filter">An optional filter to apply to the files being restored.</param>
/// <param name="disableAdsRestore">A flag indicating whether to disable the restoration of alternate data streams.</param>
/// <param name="token">A cancellation token to monitor for cancellation requests.</param>
/// <returns>A task that, when awaited, returns a tuple containing the count of files to restore and the total size of those files.</returns>
public async Task<Tuple<long, long>> PrepareRestoreFilelistAsync(DateTime restoretime, long[] versions, IFilter filter, CancellationToken token)
public async Task<Tuple<long, long>> PrepareRestoreFilelistAsync(DateTime restoretime, long[] versions, IFilter filter, bool disableAdsRestore, CancellationToken token)
{
m_tempfiletable = $"Fileset-{m_temptabsetguid}";
m_tempblocktable = $"Blocks-{m_temptabsetguid}";
var enableAds = !disableAdsRestore && SystemIO.IO_OS.SupportsAlternateDataStreams;
await using (var cmd = m_connection.CreateCommand())
{
cmd.SetTransaction(m_rtr);
@@ -657,7 +660,15 @@ namespace Duplicati.Library.Main.Database
{
rd.GetValues(values);
var path = values[1] as string;
if (path != null && FilterExpression.Matches(filter, path.ToString()!))
if (path == null)
continue;
var matches = FilterExpression.Matches(filter, path);
if (!matches && enableAds && SystemIO.IO_OS.IsAlternateDataStream(path))
matches = FilterExpression.Matches(filter, SystemIO.IO_OS.GetAlternateDataStreamParent(path));
if (matches)
{
await cmd2
.SetParameterValue("@ID", values[0])
@@ -671,6 +682,10 @@ namespace Duplicati.Library.Main.Database
}
}
// Ensure ADS targets exist before we attempt to write them
if (enableAds)
await EnsureAlternateDataStreamParentsAsync(filesetId, cmd, token).ConfigureAwait(false);
//creating indexes after insertion is much faster
await cmd.ExecuteNonQueryAsync($@"
CREATE INDEX ""{m_tempfiletable}_ID""
@@ -724,6 +739,133 @@ namespace Duplicati.Library.Main.Database
return new Tuple<long, long>(0, 0);
}
/// <summary>
/// Ensures that the parent file or folder entry of any selected alternate data stream
/// is also present in the temporary file table for the given fileset. Alternate data
/// streams are stored as <c>&lt;parent&gt;:&lt;stream&gt;</c> entries and a filter that targets
/// the parent file or folder will not match them directly (and vice versa); this makes
/// sure both ends are restored together.
/// </summary>
/// <param name="filesetId">The fileset being restored.</param>
/// <param name="cmd">The command used for reading the current selection.</param>
/// <param name="token">A cancellation token to monitor for cancellation requests.</param>
private async Task EnsureAlternateDataStreamParentsAsync(long filesetId, SqliteCommand cmd, CancellationToken token)
{
// Nothing to do on platforms that do not support alternate data streams.
if (!SystemIO.IO_OS.SupportsAlternateDataStreams)
return;
// Collect the parent paths of any alternate data streams currently selected.
// Folder entries are stored with a trailing directory separator ("folder\"), while
// file entries are not ("file.txt"), so include both forms as lookup candidates.
var dirsep = Util.DirectorySeparatorString;
var parentPaths = new HashSet<string>(
Library.Utility.Utility.IsFSCaseSensitive
? StringComparer.Ordinal
: StringComparer.OrdinalIgnoreCase);
cmd.SetCommandAndParameters($@"
SELECT ""Path""
FROM ""{m_tempfiletable}""
");
await using (var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false))
while (await rd.ReadAsync(token).ConfigureAwait(false))
{
var path = rd.ConvertValueToString(0);
if (path != null && SystemIO.IO_OS.IsAlternateDataStream(path))
{
// At this point, we do not know if the ADS target is a file or folder
// On Windows, if the ADS target is a folder, the path does not end with a trailing backslash,
// but Duplicati stores all directory paths with a trailing backslash.
// To ensure we match folders, we append the trailing backslash to the path,
// even though this gives a false positive for files with the same path as a folder,
// but that *should* not happen, and side-effects are minimal.
var parent = SystemIO.IO_OS.GetAlternateDataStreamParent(path);
parentPaths.Add(parent);
parentPaths.Add(parent + dirsep);
}
}
if (parentPaths.Count == 0)
return;
// Push the candidate paths into a temporary table and let the database perform the
// entire insert in a single statement, which is much faster for larger sets.
await using var parentTable = await TemporaryDbValueList.CreateAsync(this, parentPaths, token).ConfigureAwait(false);
await using var insertCmd = m_connection.CreateCommand($@"
INSERT INTO ""{m_tempfiletable}"" (
""ID"",
""Path"",
""BlocksetID"",
""MetadataID"",
""DataVerified""
)
SELECT
""File"".""ID"",
""File"".""Path"",
""File"".""BlocksetID"",
""File"".""MetadataID"",
0
FROM
""File"",
""FilesetEntry""
WHERE
""File"".""ID"" = ""FilesetEntry"".""FileID""
AND ""FilesetEntry"".""FilesetID"" = @FilesetId
AND ""File"".""Path"" IN (@ParentPaths)
AND ""File"".""ID"" NOT IN (
SELECT ""ID""
FROM ""{m_tempfiletable}""
)
")
.SetTransaction(m_rtr)
.SetParameterValue("@FilesetId", filesetId);
await insertCmd.ExpandInClauseParameterMssqliteAsync("@ParentPaths", parentTable, token).ConfigureAwait(false);
await insertCmd.ExecuteNonQueryAsync(token).ConfigureAwait(false);
}
/// <summary>
/// Removes alternate data stream entries from the temporary file table.
/// </summary>
/// <param name="token">A cancellation token to monitor for cancellation requests.</param>
public async Task RemoveAlternateDataStreamsAsync(CancellationToken token)
{
if (string.IsNullOrWhiteSpace(m_tempfiletable))
return;
var adsIds = new List<long>();
await using var cmd = m_connection.CreateCommand($@"
SELECT ""ID"", ""Path""
FROM ""{m_tempfiletable}""
")
.SetTransaction(m_rtr);
await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false);
while (await rd.ReadAsync(token).ConfigureAwait(false))
{
var id = rd.ConvertValueToInt64(0);
var path = rd.ConvertValueToString(1);
if (path != null && SystemIO.IO_OS.IsAlternateDataStream(path))
adsIds.Add(id);
}
if (adsIds.Count == 0)
return;
await using var tmptable = await TemporaryDbValueList.CreateAsync(this, adsIds, token).ConfigureAwait(false);
await using var delCmd = m_connection.CreateCommand($@"
DELETE FROM ""{m_tempfiletable}""
WHERE ""ID"" IN (@Ids)
")
.SetTransaction(m_rtr);
await delCmd.ExpandInClauseParameterMssqliteAsync("@Ids", tmptable, token).ConfigureAwait(false);
await delCmd.ExecuteNonQueryAsync(token).ConfigureAwait(false);
}
/// <summary>
/// Retrieves the first path from the temporary file table.
/// </summary>
Duplicati/Library/Main/Operation/BackupHandler.cs
+1 -1
View File
@@ -124,7 +124,7 @@ namespace Duplicati.Library.Main.Operation
}
}
return SnapshotUtility.CreateNoSnapshot(sources, options.IgnoreAdvisoryLocking, options.SymlinkPolicy == Options.SymlinkStrategy.Follow, useSeBackup, options.HandleMacOSPhotoLibrary, options.MacOSPhotoLibraryPath);
return SnapshotUtility.CreateNoSnapshot(sources, options.IgnoreAdvisoryLocking, options.SymlinkPolicy == Options.SymlinkStrategy.Follow, useSeBackup, options.EnableAdsBackup, options.HandleMacOSPhotoLibrary, options.MacOSPhotoLibraryPath);
}
/// <summary>
Duplicati/Library/Main/Operation/RepairHandler.cs
+1 -1
View File
@@ -1156,7 +1156,7 @@ namespace Duplicati.Library.Main.Operation
continue;
}
using var snapshot = Snapshots.SnapshotUtility.CreateNoSnapshot([block.Path], m_options.IgnoreAdvisoryLocking, m_options.SymlinkPolicy == Options.SymlinkStrategy.Follow, PermissionHelper.HasSeBackupPrivilege(), m_options.HandleMacOSPhotoLibrary, m_options.MacOSPhotoLibraryPath);
using var snapshot = Snapshots.SnapshotUtility.CreateNoSnapshot([block.Path], m_options.IgnoreAdvisoryLocking, m_options.SymlinkPolicy == Options.SymlinkStrategy.Follow, PermissionHelper.HasSeBackupPrivilege(), m_options.EnableAdsBackup, m_options.HandleMacOSPhotoLibrary, m_options.MacOSPhotoLibraryPath);
var entry = snapshot.GetFilesystemEntry(block.Path, isDir);
if (entry == null)
{
Duplicati/Library/Main/Operation/Restore/FileLister.cs
+29 -3
View File
@@ -25,7 +25,7 @@ using System.Diagnostics;
using System.Linq;
using System.Threading.Tasks;
using CoCoL;
using Duplicati.Library.Interface;
using Duplicati.Library.Common.IO;
using Duplicati.Library.Main.Database;
#nullable enable
@@ -72,14 +72,34 @@ namespace Duplicati.Library.Main.Operation.Restore
{
sw_get_files?.Start();
// The enumerables are cast to arrays to force the query to be executed and release the database lock.
var files = await db
var files = (await db
.GetFilesAndSymlinksToRestoreAsync(result.TaskControl.ProgressToken)
.ToArrayAsync()
.ConfigureAwait(false);
.ConfigureAwait(false))
.AsEnumerable();
result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_DownloadingRemoteFiles);
sw_get_files?.Stop();
// Separate out alternate data streams so they are restored last
var adsStreams = new List<FileRequest>();
if (SystemIO.IO_OS.SupportsAlternateDataStreams)
{
var hostFiles = new List<FileRequest>();
foreach (var f in files)
if (SystemIO.IO_OS.IsAlternateDataStream(f.TargetPath))
{
if (options.DisableAdsRestore)
Logging.Log.WriteVerboseMessage(LOGTAG, "SkipAdsRestore", "Skipping ADS restore for {0}", f.TargetPath);
else
adsStreams.Add(f);
}
else
hostFiles.Add(f);
files = hostFiles;
}
sw_write_file?.Start();
// First, send priority files (if any) to be restored first
@@ -124,6 +144,12 @@ namespace Duplicati.Library.Main.Operation.Restore
await self.Output.WriteAsync(folder).ConfigureAwait(false);
sw_write_folder?.Stop();
}
// Send the alternate data streams last, so their hosts are restored
sw_write_file?.Start();
foreach (var file in adsStreams)
await self.Output.WriteAsync(new FileRequest(file.ID, file.OriginalPath, file.TargetPath, file.Hash, file.Length, file.BlocksetID, IsAlternateDataStream: true)).ConfigureAwait(false);
sw_write_file?.Stop();
}
catch (Exception ex)
{
Duplicati/Library/Main/Operation/Restore/FileProcessor.cs
+23 -13
View File
@@ -129,7 +129,10 @@ namespace Duplicati.Library.Main.Operation.Restore
Logging.Log.WriteExplicitMessage(LOGTAG, "FileRestored", null, "{0} Restoring file {1}", my_id, file.TargetPath);
if (file.BlocksetID == LocalDatabase.FOLDER_BLOCKSET_ID && !options.SkipMetadata)
// Alternate data streams and folder metadata are restored only after all
// main file content has been restored.
// We rendezvous with all other processors to guarantee the main content is complete.
if ((file.BlocksetID == LocalDatabase.FOLDER_BLOCKSET_ID && !options.SkipMetadata) || file.IsAlternateDataStream)
{
// Check if there are other FileProcessor's still restoring files
if (!decremented)
@@ -142,10 +145,16 @@ namespace Duplicati.Library.Main.Operation.Restore
decremented = true;
}
await RestoreMetadataAsync(db, file, restoreDestination, block_request, block_response, options, sw_meta, sw_work_meta, sw_req, sw_resp, results.TaskControl.ProgressToken)
.ConfigureAwait(false);
if (file.BlocksetID == LocalDatabase.FOLDER_BLOCKSET_ID)
{
if (!options.SkipMetadata)
await RestoreMetadataAsync(db, file, restoreDestination, block_request, block_response, options, sw_meta, sw_work_meta, sw_req, sw_resp, results.TaskControl.ProgressToken)
.ConfigureAwait(false);
continue;
continue;
}
// ADS stream: fall through to normal file restoration now that the host is restored.
}
// Get information about the blocks for the file
@@ -802,7 +811,7 @@ namespace Duplicati.Library.Main.Operation.Restore
List<BlockRequest> verified_blocks = [];
// Check if the file exists
if (File.Exists(file.TargetPath))
if (await restoreDestination.FileExists(file.TargetPath, cancellationToken).ConfigureAwait(false))
{
filehasher.Initialize();
try
@@ -858,17 +867,19 @@ namespace Duplicati.Library.Main.Operation.Restore
if (filehasher.Hash != null && Convert.ToBase64String(filehasher.Hash) == file.Hash)
{
// Truncate the file if it is larger than the expected size.
FileInfo fi = new(file.TargetPath);
if (file.Length < fi.Length)
var currentLength = await restoreDestination.GetFileLength(file.TargetPath, cancellationToken).ConfigureAwait(false);
if (file.Length < currentLength)
{
if (options.Dryrun)
{
Logging.Log.WriteDryrunMessage(LOGTAG, "DryrunRestore", @$"Would have truncated ""{file.TargetPath}"" from {fi.Length} to {file.Length}");
Logging.Log.WriteDryrunMessage(LOGTAG, "DryrunRestore", @$"Would have truncated ""{file.TargetPath}"" from {currentLength} to {file.Length}");
}
else
{
// Reopen file with write permission
fi.IsReadOnly = false; // The metadata handler will revert this back later.
// The metadata handler will revert this back later.
if (await restoreDestination.HasReadOnlyAttribute(file.TargetPath, cancellationToken).ConfigureAwait(false))
await restoreDestination.ClearReadOnlyAttribute(file.TargetPath, cancellationToken).ConfigureAwait(false);
try
{
using var f = await restoreDestination.OpenWrite(file.TargetPath, cancellationToken).ConfigureAwait(false);
@@ -922,15 +933,14 @@ namespace Duplicati.Library.Main.Operation.Restore
List<BlockRequest> missing_blocks = [];
List<BlockRequest> verified_blocks = [];
// Check if the file exists
if (File.Exists(file.OriginalPath))
// Check if the file exists (ignore if this was a remote source)
if (SystemIO.IO_OS.FileExists(file.OriginalPath))
{
filehasher.Initialize();
// Open both files, as the target file is still being read to produce the overall file hash, if all the blocks are present across both the target and original files.
// Note: The original file is the source file on the local system (not within the restore destination),
// so we use File.OpenRead directly instead of restoreDestination.OpenRead
using var f_original = File.OpenRead(file.OriginalPath);
using var f_original = SystemIO.IO_OS.FileOpenRead(file.OriginalPath);
using var f_target = options.Dryrun ?
(await restoreDestination.FileExists(file.TargetPath, cancellationToken).ConfigureAwait(false) ?
await restoreDestination.OpenRead(file.TargetPath, cancellationToken).ConfigureAwait(false) :
Duplicati/Library/Main/Operation/Restore/Interfaces.cs
+3 -1
View File
@@ -176,7 +176,8 @@ namespace Duplicati.Library.Main.Operation.Restore
/// <param name="Length">The length of the file.</param>
/// <param name="BlocksetID">The BlocksetID of the file.</param>
/// <param name="IsPriorityFile">Whether this is a priority file that should be processed before other files.</param>
public class FileRequest(long ID, string OriginalPath, string TargetPath, string Hash, long Length, long BlocksetID, bool IsPriorityFile = false)
/// <param name="IsAlternateDataStream">Whether this request is for an alternate data stream that must be restored after its host file/folder.</param>
public class FileRequest(long ID, string OriginalPath, string TargetPath, string Hash, long Length, long BlocksetID, bool IsPriorityFile = false, bool IsAlternateDataStream = false)
{
public long ID { get; } = ID;
public string OriginalPath { get; } = OriginalPath;
@@ -185,6 +186,7 @@ namespace Duplicati.Library.Main.Operation.Restore
public long Length { get; } = Length;
public long BlocksetID { get; } = BlocksetID;
public bool IsPriorityFile { get; } = IsPriorityFile;
public bool IsAlternateDataStream { get; } = IsAlternateDataStream;
}
}
Duplicati/Library/Main/Operation/RestoreHandler.cs
+35 -5
View File
@@ -500,7 +500,9 @@ namespace Duplicati.Library.Main.Operation
}
else if (m_result.RestoredFiles == 0)
{
if (m_result.UnmodifiedFiles == 0)
if (m_result.UnmodifiedFiles == 0 && m_result.RestoredFolders == 0)
Logging.Log.WriteWarningMessage(LOGTAG, "NoFilesOrFoldersRestored", null, "Restore completed without errors but no files or folders were restored");
else if (m_result.UnmodifiedFiles == 0)
Logging.Log.WriteWarningMessage(LOGTAG, "NoFilesRestored", null, "Restore completed without errors but no files were restored");
else
Logging.Log.WriteInformationMessage(LOGTAG, "NoFilesNeededRestore", null, "Restore completed but all files were already present");
@@ -606,7 +608,9 @@ namespace Duplicati.Library.Main.Operation
if (m_result.RestoredFiles == 0)
{
if (m_result.UnmodifiedFiles == 0)
if (m_result.UnmodifiedFiles == 0 && m_result.RestoredFolders == 0)
Logging.Log.WriteWarningMessage(LOGTAG, "NoFilesOrFoldersRestored", null, "Restore completed without errors but no files or folders were restored");
else if (m_result.UnmodifiedFiles == 0)
Logging.Log.WriteWarningMessage(LOGTAG, "NoFilesRestored", null, "Restore completed without errors but no files were restored");
else
Logging.Log.WriteInformationMessage(LOGTAG, "NoFilesNeededRestore", null, "Restore completed but all files were already present");
@@ -1061,9 +1065,23 @@ namespace Duplicati.Library.Main.Operation
using (new Logging.Timer(LOGTAG, "PrepareRestoreFileList", "PrepareRestoreFileList"))
{
var c = await database
.PrepareRestoreFilelistAsync(options.Time, options.Version, filter, result.TaskControl.ProgressToken)
.PrepareRestoreFilelistAsync(options.Time, options.Version, filter, options.DisableAdsRestore, result.TaskControl.ProgressToken)
.ConfigureAwait(false);
result.OperationProgressUpdater.UpdatefileCount(c.Item1, c.Item2, true);
// If the selection is completely empty (no files, folders, or symlinks),
// stop now as this is most likely not what is desired.
var firstPath = await database
.GetFirstPathAsync(result.TaskControl.ProgressToken)
.ConfigureAwait(false);
if (string.IsNullOrWhiteSpace(firstPath))
throw new UserInformationException("Restore selection matched zero files, nothing to restore", "EmptyRestoreOperation");
}
if (options.DisableAdsRestore)
{
using (new Logging.Timer(LOGTAG, "FilterAds", "Filtering alternate data streams from restore list"))
await database.RemoveAlternateDataStreamsAsync(result.TaskControl.ProgressToken).ConfigureAwait(false);
}
using (new Logging.Timer(LOGTAG, "SetTargetPaths", "SetTargetPaths"))
@@ -1163,7 +1181,19 @@ namespace Duplicati.Library.Main.Operation
var targetfileid = restorelist.TargetFileID;
var targetfilehash = restorelist.TargetHash;
var targetfilelength = restorelist.Length;
if (await restoreDestination.FileExists(targetpath, result.TaskControl.ProgressToken).ConfigureAwait(false))
var fileExists = false;
try
{
fileExists = await restoreDestination.FileExists(targetpath, result.TaskControl.ProgressToken).ConfigureAwait(false);
}
catch (Exception ex)
{
Logging.Log.WriteWarningMessage(LOGTAG, "FileExistsFailed", ex, "Failed to check if file exists: \"{0}\", message: {1}", targetpath, ex.Message);
if (options.UnittestMode)
throw;
}
if (fileExists)
{
try
{
@@ -1198,7 +1228,7 @@ namespace Duplicati.Library.Main.Operation
{
// a file hash for verification will only be necessary if the file has exactly
// the wanted size so we have a chance to already mark the file as data-verified.
bool calcFileHash = (currentfilelength == targetfilelength);
var calcFileHash = currentfilelength == targetfilelength;
if (calcFileHash) filehasher.Initialize();
using (var file = await restoreDestination.OpenRead(targetpath, result.TaskControl.ProgressToken).ConfigureAwait(false))
Duplicati/Library/Main/Options.cs
+12
View File
@@ -377,6 +377,8 @@ namespace Duplicati.Library.Main
yield return new CommandLineArgument("usn-policy", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.UsnpolicyShort, Strings.Options.UsnpolicyLong, "off", null, Enum.GetNames(typeof(OptimizationStrategy)));
yield return new CommandLineArgument("backupread-policy", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.BackupreadpolicyShort, Strings.Options.BackupreadpolicyLong, DEFAULT_BACKUPREAD_POLICY.ToString(), null, Enum.GetNames(typeof(OptimizationStrategy)));
yield return new CommandLineArgument("exclude-non-local-files", CommandLineArgument.ArgumentType.Boolean, Strings.Options.ExcludenonlocalfilesShort, Strings.Options.ExcludenonlocalfilesLong, "false");
yield return new CommandLineArgument("enable-ads-backup", CommandLineArgument.ArgumentType.Boolean, Strings.Options.EnableAdsBackupShort, Strings.Options.EnableAdsBackupLong, "false");
yield return new CommandLineArgument("disable-ads-restore", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DisableAdsRestoreShort, Strings.Options.DisableAdsRestoreLong, "false");
}
if (OperatingSystem.IsLinux() || OperatingSystem.IsMacOS())
@@ -1888,6 +1890,16 @@ namespace Duplicati.Library.Main
/// </summary>
public int RestoreChannelBufferSize => GetInt("restore-channel-buffer-size", DEFAULT_RESTORE_CHANNEL_BUFFER_SIZE);
/// <summary>
/// Gets a value indicating whether NTFS alternate data streams should be backed up
/// </summary>
public bool EnableAdsBackup => GetBool("enable-ads-backup");
/// <summary>
/// Gets a value indicating whether NTFS alternate data streams should be skipped during restore
/// </summary>
public bool DisableAdsRestore => GetBool("disable-ads-restore");
/// <summary>
/// Toggles whether internal profiling is enabled and should be logged.
/// </summary>
Duplicati/Library/Main/Strings.cs
+4
View File
@@ -406,6 +406,10 @@ namespace Duplicati.Library.Main.Strings
public static string RepairRefreshLockInfoLong { get { return LC.L("Use this option to refresh the object lock expiration information from the backend during a repair operation. This will query the backend for the current lock status of each remote volume and update the local database accordingly. If the backend does not support locking, this option will be ignored."); } }
public static string RefreshLockInfoCompleteShort { get { return LC.L("Perform a complete refresh of lock information"); } }
public static string RefreshLockInfoCompleteLong { get { return LC.L("Use this option to perform a complete refresh of lock information from the backend. This will query the backend for the current lock status of each remote volume and update the local database accordingly. If this option is not specified, only the lock information for the remote volumes with no lock information will be refreshed."); } }
public static string EnableAdsBackupShort { get { return LC.L("Enable backup of NTFS alternate data streams"); } }
public static string EnableAdsBackupLong { get { return LC.L("Use this option to enable backup of NTFS alternate data streams (ADS). This is a Windows-only feature that allows backing up hidden data attached to files."); } }
public static string DisableAdsRestoreShort { get { return LC.L("Disable restore of NTFS alternate data streams"); } }
public static string DisableAdsRestoreLong { get { return LC.L("Use this option to skip restoring NTFS alternate data streams (ADS) during a restore operation. The main file content will still be restored."); } }
}
internal static class Common
Duplicati/Library/Snapshots/LinuxSnapshot.cs
+2 -2
View File
@@ -121,9 +121,9 @@ namespace Duplicati.Library.Snapshots
foreach (var sourceEntry in SourceEntries)
{
if (DirectoryExists(sourceEntry) || sourceEntry.EndsWith(System.IO.Path.DirectorySeparatorChar))
yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(sourceEntry), true, true);
yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(sourceEntry), true, true, false);
else
yield return new SnapshotSourceFileEntry(this, sourceEntry, false, true);
yield return new SnapshotSourceFileEntry(this, sourceEntry, false, true, false);
}
}
Duplicati/Library/Snapshots/MacOSSnapshot.cs
+4 -4
View File
@@ -116,9 +116,9 @@ namespace Duplicati.Library.Snapshots
foreach (var sourceEntry in SourceEntries)
{
if (DirectoryExists(sourceEntry) || sourceEntry.EndsWith(System.IO.Path.DirectorySeparatorChar.ToString()))
yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(sourceEntry), true, true);
yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(sourceEntry), true, true, false);
else
yield return new SnapshotSourceFileEntry(this, sourceEntry, false, true);
yield return new SnapshotSourceFileEntry(this, sourceEntry, false, true, false);
}
}
@@ -393,12 +393,12 @@ namespace Duplicati.Library.Snapshots
if (isFolder)
{
if (System.IO.Directory.Exists(snapPath))
return new SnapshotSourceFileEntry(this, isFolder ? Util.AppendDirSeparator(path) : path, isFolder, false);
return new SnapshotSourceFileEntry(this, isFolder ? Util.AppendDirSeparator(path) : path, isFolder, false, false);
}
else
{
if (System.IO.File.Exists(snapPath))
return new SnapshotSourceFileEntry(this, isFolder ? Util.AppendDirSeparator(path) : path, isFolder, false);
return new SnapshotSourceFileEntry(this, isFolder ? Util.AppendDirSeparator(path) : path, isFolder, false, false);
}
}
catch
Duplicati/Library/Snapshots/NoSnapshotLinux.cs
+2 -2
View File
@@ -98,9 +98,9 @@ namespace Duplicati.Library.Snapshots
foreach (var folder in m_sources)
{
if (DirectoryExists(folder) || folder.EndsWith(Path.DirectorySeparatorChar))
yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(folder), true, true);
yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(folder), true, true, false);
else
yield return new SnapshotSourceFileEntry(this, folder, false, true);
yield return new SnapshotSourceFileEntry(this, folder, false, true, false);
}
}
Duplicati/Library/Snapshots/NoSnapshotWindows.cs
+50 -14
View File
@@ -47,16 +47,23 @@ namespace Duplicati.Library.Snapshots
/// </summary>
private readonly IDisposable m_seBackupPrivilege;
/// <summary>
/// A flag indicating if alternate data streams should be backed up
/// </summary>
private readonly bool m_enableAdsBackup;
/// <summary>
/// Initializes a new instance of the <see cref="NoSnapshotWindows"/> class.
/// </summary>
/// <param name="sources">The list of entries to create snapshots of</param>
/// <param name="followSymlinks">A flag indicating if symlinks should be followed</param>
/// <param name="useSeBackup">A flag indicating if the SeBackupPrivilege should be used</param>
public NoSnapshotWindows(IEnumerable<string> sources, bool followSymlinks, bool useSeBackup)
/// <param name="enableAdsBackup">A flag indicating if alternate data streams should be backed up</param>
public NoSnapshotWindows(IEnumerable<string> sources, bool followSymlinks, bool useSeBackup, bool enableAdsBackup)
: base(followSymlinks)
{
m_sources = sources;
m_enableAdsBackup = enableAdsBackup;
if (useSeBackup)
m_seBackupPrivilege = WindowsShimLoader.NewSeBackupPrivilegeScope();
}
@@ -100,9 +107,9 @@ namespace Duplicati.Library.Snapshots
foreach (var folder in m_sources.Select(SystemIOWindows.RemoveExtendedDevicePathPrefix))
{
if (DirectoryExists(folder) || folder.EndsWith(System.IO.Path.DirectorySeparatorChar))
yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(folder), true, true);
yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(folder), true, true, false);
else
yield return new SnapshotSourceFileEntry(this, folder, false, true);
yield return new SnapshotSourceFileEntry(this, folder, false, true, SystemIO.IO_OS.IsAlternateDataStream(folder));
}
}
@@ -128,34 +135,63 @@ namespace Duplicati.Library.Snapshots
/// </summary>
/// <param name="localPath">The full path to the file in non-snapshot format</param>
/// <returns>An open filestream that can be read</returns>
public override System.IO.Stream OpenRead(string localPath)
public override Stream OpenRead(string localPath)
=> SystemIO.IO_OS.FileOpenRead(localPath);
/// <inheritdoc/>
public override bool DirectoryExists(string localFolderPath)
=> SystemIO.IO_OS.DirectoryExists(localFolderPath);
/// <inheritdoc/>
public override bool FileExists(string localFilePath)
=> SystemIO.IO_OS.FileExists(localFilePath);
/// <inheritdoc />
public override ISourceProviderEntry GetFilesystemEntry(string path, bool isFolder)
{
if (m_enableAdsBackup && SystemIO.IO_OS.IsAlternateDataStream(path))
{
var entry = base.GetFilesystemEntry(path, false);
if (entry != null)
entry = new SnapshotSourceFileEntry(this, entry.Path, false, entry.IsRootEntry, true);
return entry;
}
return base.GetFilesystemEntry(path, isFolder);
}
/// <summary>
/// Lists all files in the given folder
/// </summary>
/// <returns>All folders found in the folder</returns>
/// <param name='localFolderPath'>The folder to examinate</param>
protected override string[] ListFiles(string localFolderPath)
protected override IEnumerable<string> ListFiles(string localFolderPath)
{
string[] tmp = SystemIO.IO_OS.GetFiles(localFolderPath);
string[] res = new string[tmp.Length];
for (int i = 0; i < tmp.Length; i++)
res[i] = SystemIOWindows.RemoveExtendedDevicePathPrefix(tmp[i]);
var tmp = SystemIO.IO_OS.GetFiles(localFolderPath);
var res = new List<string>(tmp.Length);
foreach (var p in tmp)
res.Add(SystemIOWindows.RemoveExtendedDevicePathPrefix(p));
return res;
if (!m_enableAdsBackup || !SystemIO.IO_OS.SupportsAlternateDataStreams)
return res;
// Expand ADS for each file
var expanded = ExpandAlternateDataStreams(res, SystemIO.IO_OS.EnumerateAlternateDataStreams);
// Also include ADS on the parent folder itself
var folderPath = localFolderPath.TrimEnd(Path.DirectorySeparatorChar);
return expanded.Concat(SystemIO.IO_OS.EnumerateAlternateDataStreams(folderPath).Select(stream => folderPath + stream));
}
/// <summary>
/// Lists all folders in the given folder
/// </summary>
/// <returns>All folders found in the folder</returns>
/// <param name='localFolderPath'>The folder to examinate</param>
protected override string[] ListFolders(string localFolderPath)
protected override IEnumerable<string> ListFolders(string localFolderPath)
{
string[] tmp = SystemIO.IO_OS.GetDirectories(SystemIOWindows.AddExtendedDevicePathPrefix(localFolderPath));
string[] res = new string[tmp.Length];
var tmp = SystemIO.IO_OS.GetDirectories(SystemIOWindows.AddExtendedDevicePathPrefix(localFolderPath));
var res = new string[tmp.Length];
for (int i = 0; i < tmp.Length; i++)
res[i] = SystemIOWindows.RemoveExtendedDevicePathPrefix(tmp[i]);
Duplicati/Library/Snapshots/SnapshotBase.cs
+34 -8
View File
@@ -19,6 +19,8 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
using System;
using System.Collections.Generic;
using System.IO;
@@ -26,6 +28,7 @@ using Duplicati.Library.Interface;
using System.Threading.Tasks;
using System.Threading;
using Duplicati.Library.Common.IO;
using System.Linq;
namespace Duplicati.Library.Snapshots
{
@@ -72,14 +75,14 @@ namespace Duplicati.Library.Snapshots
/// <param name="path">The path to get the entry for</param>
/// <param name="isFolder">A flag indicating if the path is a folder</param>
/// <returns>The filesystem entry</returns>
public virtual ISourceProviderEntry GetFilesystemEntry(string path, bool isFolder)
public virtual ISourceProviderEntry? GetFilesystemEntry(string path, bool isFolder)
{
if (isFolder && !DirectoryExists(path))
return null;
else if (!isFolder && !FileExists(path))
return null;
return new SnapshotSourceFileEntry(this, isFolder ? Util.AppendDirSeparator(path) : path, isFolder, false);
return new SnapshotSourceFileEntry(this, isFolder ? Util.AppendDirSeparator(path) : path, isFolder, false, false);
}
/// <summary>
@@ -94,10 +97,18 @@ namespace Duplicati.Library.Snapshots
// Removing the null characters is a workaround for a bug in .NET core:
// https://github.com/dotnet/runtime/issues/49803
foreach (var f in ListFiles(source.Path))
yield return new SnapshotSourceFileEntry(this, f.Trim('\0'), false, false);
{
var entry = GetFilesystemEntry(f.Trim('\0'), false);
if (entry != null)
yield return entry;
}
foreach (var d in ListFolders(source.Path))
yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(d.Trim('\0')), true, false);
{
var entry = GetFilesystemEntry(Util.AppendDirSeparator(d.Trim('\0')), true);
if (entry != null)
yield return entry;
}
}
}
@@ -163,7 +174,7 @@ namespace Duplicati.Library.Snapshots
/// <returns>The metadata for the given file or folder</returns>
/// <param name="localPath">The file or folder to examine</param>
/// <param name="isSymlink">A flag indicating if the target is a symlink</param>
public abstract Dictionary<string, string> GetMetadata(string localPath, bool isSymlink);
public abstract Dictionary<string, string?> GetMetadata(string localPath, bool isSymlink);
/// <summary>
/// Gets a value indicating if the path points to a block device
@@ -178,7 +189,7 @@ namespace Duplicati.Library.Snapshots
/// </summary>
/// <returns>The hardlink ID</returns>
/// <param name="localPath">The file or folder to examine</param>
public virtual string HardlinkTargetID(string localPath)
public virtual string? HardlinkTargetID(string localPath)
=> null;
/// <inheritdoc />
@@ -202,7 +213,7 @@ namespace Duplicati.Library.Snapshots
/// </summary>
/// <returns>All folders found in the folder</returns>
/// <param name='localFolderPath'>The folder to examinate</param>
protected virtual string[] ListFolders(string localFolderPath)
protected virtual IEnumerable<string> ListFolders(string localFolderPath)
=> Directory.GetDirectories(localFolderPath);
/// <summary>
@@ -210,9 +221,24 @@ namespace Duplicati.Library.Snapshots
/// </summary>
/// <returns>All folders found in the folder</returns>
/// <param name='localFolderPath'>The folder to examinate</param>
protected virtual string[] ListFiles(string localFolderPath)
protected virtual IEnumerable<string> ListFiles(string localFolderPath)
=> Directory.GetFiles(localFolderPath);
/// <summary>
/// Expands a list of files to include alternate data streams
/// </summary>
/// <param name="files">The list of files</param>
/// <param name="enumerateStreams">A function that enumerates streams for a given file path</param>
/// <returns>The expanded list of files</returns>
protected static IEnumerable<string> ExpandAlternateDataStreams(IEnumerable<string> files, Func<string, IEnumerable<string>> enumerateStreams)
{
// With no ADS support, just return the original list
if (!SystemIO.IO_OS.SupportsAlternateDataStreams)
return files;
return files.SelectMany(file => new[] { file }.Concat(enumerateStreams(file).Select(stream => file + stream)));
}
#region IDisposable interface
/// <inheritdoc />
Duplicati/Library/Snapshots/SnapshotSourceFileEntry.cs
+3 -2
View File
@@ -39,7 +39,8 @@ namespace Duplicati.Library.Snapshots;
/// <param name="path">The path of the file</param>
/// <param name="isFolder">True if the entry is a folder, false otherwise</param>
/// <param name="isRoot">True if the entry is the root entry, false otherwise</param>
public class SnapshotSourceFileEntry(ISnapshotService service, string path, bool isFolder, bool isRoot) : ISourceProviderEntry
/// <param name="isAlternateStream">True if the entry is an alternate data stream, false otherwise</param>
public class SnapshotSourceFileEntry(ISnapshotService service, string path, bool isFolder, bool isRoot, bool isAlternateStream) : ISourceProviderEntry
{
/// <summary>
/// The symlink target
@@ -99,7 +100,7 @@ public class SnapshotSourceFileEntry(ISnapshotService service, string path, bool
/// <summary>
/// Gets a value indicating if the entry is an alternate stream
/// </summary>
public bool IsAlternateStream => false;
public bool IsAlternateStream => isAlternateStream;
/// <summary>
/// Gets the hardlink target ID, or null if the entry is not a hardlink
Duplicati/Library/Snapshots/SnapshotUtility.cs
+3 -2
View File
@@ -72,9 +72,10 @@ namespace Duplicati.Library.Snapshots
/// <param name="useSeBackup">Whether to use SeBackupPrivilege on Windows</param>
/// <param name="macOSPhotosHandling">Whether to handle MacOS Photos libraries specially</param>
/// <param name="photosLibraryPath">The user specified MacOS Photos library path</param>
/// <param name="enableAdsBackup">Whether to backup alternate data streams on Windows</param>
/// <returns>The ISnapshotService implementation</returns>
[System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)]
public static ISnapshotService CreateNoSnapshot(IEnumerable<string> paths, bool ignoreAdvisoryLocking, bool followSymlinks, bool useSeBackup, MacOSPhotosHandling macOSPhotosHandling, string photosLibraryPath)
public static ISnapshotService CreateNoSnapshot(IEnumerable<string> paths, bool ignoreAdvisoryLocking, bool followSymlinks, bool useSeBackup, bool enableAdsBackup, MacOSPhotosHandling macOSPhotosHandling, string photosLibraryPath)
{
// MacOS implementation only handles photo libraries specially if requested
// Otherwise, it behaves like the Linux implementation
@@ -83,7 +84,7 @@ namespace Duplicati.Library.Snapshots
else if (OperatingSystem.IsLinux())
return new NoSnapshotLinux(paths, ignoreAdvisoryLocking, followSymlinks);
else if (OperatingSystem.IsWindows())
return new NoSnapshotWindows(paths, followSymlinks, useSeBackup);
return new NoSnapshotWindows(paths, followSymlinks, useSeBackup, enableAdsBackup);
else
throw new NotSupportedException("Unsupported Operating System");
Duplicati/Library/Snapshots/WindowsSnapshot.cs
+37 -14
View File
@@ -77,6 +77,11 @@ namespace Duplicati.Library.Snapshots
/// </summary>
private readonly IReadOnlyList<string> _sourceEntries;
/// <summary>
/// A flag indicating if alternate data streams should be backed up
/// </summary>
private readonly bool _enableAdsBackup;
/// <summary>
/// Constructs a new backup snapshot, using all the required disks
/// </summary>
@@ -86,6 +91,7 @@ namespace Duplicati.Library.Snapshots
public WindowsSnapshot(IEnumerable<string> sources, IDictionary<string, string> options, bool followSymlinks)
: base(followSymlinks)
{
_enableAdsBackup = Utility.Utility.ParseBoolOption(options.AsReadOnly(), "enable-ads-backup");
// For Windows, ensure we don't store paths with extended device path prefixes (i.e., @"\\?\" or @"\\?\UNC\")
_sourceEntries = sources.Select(SystemIOWindows.RemoveExtendedDevicePathPrefix).ToList();
try
@@ -148,9 +154,9 @@ namespace Duplicati.Library.Snapshots
foreach (var folder in _sourceEntries)
{
if (folder.EndsWith(Path.DirectorySeparatorChar) || DirectoryExists(folder))
yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(folder), true, true);
yield return new SnapshotSourceFileEntry(this, Util.AppendDirSeparator(folder), true, true, false);
else
yield return new SnapshotSourceFileEntry(this, folder, false, true);
yield return new SnapshotSourceFileEntry(this, folder, false, true, IO_WIN.IsAlternateDataStream(folder));
}
}
@@ -162,19 +168,16 @@ namespace Duplicati.Library.Snapshots
/// </summary>
/// <param name="localFolderPath">The non-shadow path of the folder to list</param>
/// <returns>A list of non-shadow paths</returns>
protected override string[] ListFolders(string localFolderPath)
protected override IEnumerable<string> ListFolders(string localFolderPath)
{
string[] tmp = null;
var spath = ConvertToSnapshotPath(localFolderPath);
tmp = IO_WIN.GetDirectories(spath);
var tmp = IO_WIN.GetDirectories(spath);
var root = Util.AppendDirSeparator(IO_WIN.GetPathRoot(localFolderPath));
var volumePath = Util.AppendDirSeparator(ConvertToSnapshotPath(root));
volumePath = SystemIOWindows.AddExtendedDevicePathPrefix(volumePath);
for (var i = 0; i < tmp.Length; i++)
{
tmp[i] = root + SystemIOWindows.AddExtendedDevicePathPrefix(tmp[i]).Substring(volumePath.Length);
}
return tmp;
}
@@ -186,12 +189,10 @@ namespace Duplicati.Library.Snapshots
/// </summary>
/// <param name="localFolderPath">The non-shadow path of the folder to list</param>
/// <returns>A list of non-shadow paths</returns>
protected override string[] ListFiles(string localFolderPath)
protected override IEnumerable<string> ListFiles(string localFolderPath)
{
string[] files = null;
var spath = ConvertToSnapshotPath(localFolderPath);
files = IO_WIN.GetFiles(spath);
var files = IO_WIN.GetFiles(spath);
// convert back to non-shadow, i.e., non-vss version
var root = Util.AppendDirSeparator(IO_WIN.GetPathRoot(localFolderPath));
@@ -199,11 +200,19 @@ namespace Duplicati.Library.Snapshots
volumePath = SystemIOWindows.AddExtendedDevicePathPrefix(volumePath);
for (var i = 0; i < files.Length; i++)
{
files[i] = root + SystemIOWindows.AddExtendedDevicePathPrefix(files[i]).Substring(volumePath.Length);
}
return files;
if (!_enableAdsBackup || !SystemIO.IO_OS.SupportsAlternateDataStreams)
return files;
// Expand ADS for each file
var expanded = ExpandAlternateDataStreams(files, f => SystemIO.IO_OS.EnumerateAlternateDataStreams(ConvertToSnapshotPath(f)));
// Also include ADS on the parent folder itself
var folderPath = localFolderPath.TrimEnd(Path.DirectorySeparatorChar);
var folderShadowPath = ConvertToSnapshotPath(folderPath);
return expanded.Concat(SystemIO.IO_OS.EnumerateAlternateDataStreams(folderShadowPath).Select(stream => folderPath + stream));
}
#endregion
@@ -233,6 +242,20 @@ namespace Duplicati.Library.Snapshots
return IO_WIN.GetCreationTimeUtc(SystemIOWindows.AddExtendedDevicePathPrefix(spath));
}
/// <inheritdoc />
public override ISourceProviderEntry GetFilesystemEntry(string path, bool isFolder)
{
if (_enableAdsBackup && SystemIO.IO_OS.IsAlternateDataStream(path))
{
var entry = base.GetFilesystemEntry(path, false);
if (entry != null)
entry = new SnapshotSourceFileEntry(this, entry.Path, false, entry.IsRootEntry, true);
return entry;
}
return base.GetFilesystemEntry(path, isFolder);
}
/// <summary>
/// Opens a file for reading
/// </summary>
Duplicati/UnitTest/AlternateDataStreamTests.cs
+441
View File
@@ -0,0 +1,441 @@
// Copyright (C) 2026, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Duplicati.Library.Common.IO;
using Duplicati.Library.Main;
using NUnit.Framework;
using Assert = NUnit.Framework.Legacy.ClassicAssert;
namespace Duplicati.UnitTest
{
[Category("Windows")]
public class AlternateDataStreamTests : BasicSetupHelper
{
/// <summary>
/// Creates an alternate data stream on Windows by opening a file stream with the ADS path syntax.
/// </summary>
private static void CreateAlternateDataStream(string filePath, string streamName, string content)
{
var adsPath = filePath + ":" + streamName;
File.WriteAllText(adsPath, content);
}
/// <summary>
/// Reads the content of an alternate data stream.
/// </summary>
private static string ReadAlternateDataStream(string filePath, string streamName)
{
var adsPath = filePath + ":" + streamName;
return File.ReadAllText(adsPath);
}
/// <summary>
/// Checks whether an alternate data stream exists.
/// </summary>
private static bool AlternateDataStreamExists(string filePath, string streamName)
{
var adsPath = filePath + ":" + streamName;
return File.Exists(adsPath);
}
[Test]
[Category("Windows")]
public void EnumerateAlternateDataStreams()
{
if (!OperatingSystem.IsWindows())
{
Assert.Ignore("This test is Windows-only.");
return;
}
var filePath = Path.Combine(DATAFOLDER, "testfile.txt");
File.WriteAllText(filePath, "main content");
CreateAlternateDataStream(filePath, "stream1", "stream1 content");
CreateAlternateDataStream(filePath, "stream2", "stream2 content");
var streams = SystemIO.IO_OS.EnumerateAlternateDataStreams(filePath).ToList();
Assert.AreEqual(2, streams.Count);
Assert.IsTrue(streams.Contains(":stream1"));
Assert.IsTrue(streams.Contains(":stream2"));
}
[Test]
[Category("Windows")]
public async Task BackupAndRestoreAlternateDataStreamsAsync([Values(true, false)] bool useLegacyRestore)
{
if (!OperatingSystem.IsWindows())
{
Assert.Ignore("This test is Windows-only.");
return;
}
var filePath = Path.Combine(DATAFOLDER, "testfile.txt");
File.WriteAllText(filePath, "main content");
CreateAlternateDataStream(filePath, "hiddenstream", "hidden content");
var dirPath = Path.Combine(DATAFOLDER, "testdir");
Directory.CreateDirectory(dirPath);
CreateAlternateDataStream(dirPath, "dirstream", "dir hidden content");
var backupOptions = new Dictionary<string, string>(TestOptions)
{
["enable-ads-backup"] = "true",
["restore-legacy"] = useLegacyRestore.ToString()
};
using (var c = new Controller("file://" + TARGETFOLDER, backupOptions, null))
TestUtils.AssertResults(await c.BackupAsync(new[] { DATAFOLDER }));
// Restore without disabling ADS restore
var restoreOptions = new Dictionary<string, string>(TestOptions)
{
["restore-path"] = RESTOREFOLDER
};
if (Directory.Exists(RESTOREFOLDER))
Directory.Delete(RESTOREFOLDER);
using (var c = new Controller("file://" + TARGETFOLDER, restoreOptions, null))
TestUtils.AssertResults(await c.RestoreAsync(new[] { "*" }));
var restoredFilePath = Path.Combine(RESTOREFOLDER, "testfile.txt");
Assert.IsTrue(File.Exists(restoredFilePath));
Assert.AreEqual("main content", File.ReadAllText(restoredFilePath));
Assert.IsTrue(AlternateDataStreamExists(restoredFilePath, "hiddenstream"));
Assert.AreEqual("hidden content", ReadAlternateDataStream(restoredFilePath, "hiddenstream"));
var restoredDirPath = Path.Combine(RESTOREFOLDER, "testdir");
Assert.IsTrue(Directory.Exists(restoredDirPath));
Assert.IsTrue(AlternateDataStreamExists(restoredDirPath, "dirstream"));
Assert.AreEqual("dir hidden content", ReadAlternateDataStream(restoredDirPath, "dirstream"));
}
[Test]
[Category("Windows")]
public async Task DisableAdsRestoreSkipsAlternateDataStreamsAsync()
{
if (!OperatingSystem.IsWindows())
{
Assert.Ignore("This test is Windows-only.");
return;
}
var filePath = Path.Combine(DATAFOLDER, "testfile.txt");
File.WriteAllText(filePath, "main content");
CreateAlternateDataStream(filePath, "hiddenstream", "hidden content");
var backupOptions = new Dictionary<string, string>(TestOptions)
{
["enable-ads-backup"] = "true"
};
using (var c = new Controller("file://" + TARGETFOLDER, backupOptions, null))
TestUtils.AssertResults(await c.BackupAsync(new[] { DATAFOLDER }));
// Restore with ADS restore disabled
var restoreOptions = new Dictionary<string, string>(TestOptions)
{
["restore-path"] = RESTOREFOLDER,
["disable-ads-restore"] = "true"
};
using (var c = new Controller("file://" + TARGETFOLDER, restoreOptions, null))
TestUtils.AssertResults(await c.RestoreAsync(new[] { filePath }));
var restoredFilePath = Path.Combine(RESTOREFOLDER, "testfile.txt");
Assert.IsTrue(File.Exists(restoredFilePath));
Assert.AreEqual("main content", File.ReadAllText(restoredFilePath));
Assert.IsFalse(AlternateDataStreamExists(restoredFilePath, "hiddenstream"));
}
[Test]
[Category("Windows")]
public void IsAlternateDataStream()
{
if (!OperatingSystem.IsWindows())
{
Assert.Ignore("This test is Windows-only.");
return;
}
var filePath = Path.Combine(DATAFOLDER, "testfile.txt");
File.WriteAllText(filePath, "main content");
Assert.IsFalse(SystemIO.IO_OS.IsAlternateDataStream(filePath));
Assert.IsTrue(SystemIO.IO_OS.IsAlternateDataStream(filePath + ":stream"));
Assert.IsTrue(SystemIO.IO_OS.IsAlternateDataStream("C:\\file.txt:ads"));
Assert.IsFalse(SystemIO.IO_OS.IsAlternateDataStream("C:\\file.txt"));
var dirPath = Path.Combine(DATAFOLDER, "testdir");
Directory.CreateDirectory(dirPath);
Assert.IsFalse(SystemIO.IO_OS.IsAlternateDataStream(dirPath));
Assert.IsTrue(SystemIO.IO_OS.IsAlternateDataStream(dirPath + ":stream"));
Assert.IsTrue(SystemIO.IO_OS.IsAlternateDataStream("C:\\folder:ads"));
Assert.IsFalse(SystemIO.IO_OS.IsAlternateDataStream("C:\\folder"));
}
[Test]
[Category("Windows")]
public void EnumerateAlternateDataStreamsOnDirectory()
{
if (!OperatingSystem.IsWindows())
{
Assert.Ignore("This test is Windows-only.");
return;
}
var dirPath = Path.Combine(DATAFOLDER, "testdir");
Directory.CreateDirectory(dirPath);
CreateAlternateDataStream(dirPath, "stream1", "stream1 content");
CreateAlternateDataStream(dirPath, "stream2", "stream2 content");
var streams = SystemIO.IO_OS.EnumerateAlternateDataStreams(dirPath).ToList();
Assert.AreEqual(2, streams.Count);
Assert.IsTrue(streams.Contains(":stream1"));
Assert.IsTrue(streams.Contains(":stream2"));
}
[Test]
[Category("Windows")]
public async Task BackupWithoutEnableAdsBackupIgnoresStreamsAsync()
{
if (!OperatingSystem.IsWindows())
{
Assert.Ignore("This test is Windows-only.");
return;
}
var filePath = Path.Combine(DATAFOLDER, "testfile.txt");
File.WriteAllText(filePath, "main content");
CreateAlternateDataStream(filePath, "hiddenstream", "hidden content");
var backupOptions = new Dictionary<string, string>(TestOptions);
// enable-ads-backup is not set (defaults to false)
using (var c = new Controller("file://" + TARGETFOLDER, backupOptions, null))
TestUtils.AssertResults(await c.BackupAsync(new[] { DATAFOLDER }));
var restoreOptions = new Dictionary<string, string>(TestOptions)
{
["restore-path"] = RESTOREFOLDER
};
using (var c = new Controller("file://" + TARGETFOLDER, restoreOptions, null))
TestUtils.AssertResults(await c.RestoreAsync(new[] { filePath }));
var restoredFilePath = Path.Combine(RESTOREFOLDER, "testfile.txt");
Assert.IsTrue(File.Exists(restoredFilePath));
Assert.AreEqual("main content", File.ReadAllText(restoredFilePath));
Assert.IsFalse(AlternateDataStreamExists(restoredFilePath, "hiddenstream"));
}
[Test]
[Category("Windows")]
public async Task BackupAndRestoreAlternateDataStreamsOnDirectoryAsync()
{
if (!OperatingSystem.IsWindows())
{
Assert.Ignore("This test is Windows-only.");
return;
}
var dirPath = Path.Combine(DATAFOLDER, "testdir");
Directory.CreateDirectory(dirPath);
CreateAlternateDataStream(dirPath, "hiddenstream", "hidden content");
var backupOptions = new Dictionary<string, string>(TestOptions)
{
["enable-ads-backup"] = "true"
};
using (var c = new Controller("file://" + TARGETFOLDER, backupOptions, null))
TestUtils.AssertResults(await c.BackupAsync(new[] { DATAFOLDER }));
var restoreOptions = new Dictionary<string, string>(TestOptions)
{
["restore-path"] = RESTOREFOLDER
};
using (var c = new Controller("file://" + TARGETFOLDER, restoreOptions, null))
TestUtils.AssertResults(await c.RestoreAsync(new[] { dirPath }));
var restoredDirPath = Path.Combine(RESTOREFOLDER, "testdir");
Assert.IsTrue(Directory.Exists(restoredDirPath));
Assert.IsTrue(AlternateDataStreamExists(restoredDirPath, "hiddenstream"));
Assert.AreEqual("hidden content", ReadAlternateDataStream(restoredDirPath, "hiddenstream"));
}
[Test]
[Category("Windows")]
public async Task DisableAdsRestoreSkipsAlternateDataStreamsOnDirectoryAsync()
{
if (!OperatingSystem.IsWindows())
{
Assert.Ignore("This test is Windows-only.");
return;
}
var dirPath = Path.Combine(DATAFOLDER, "testdir");
Directory.CreateDirectory(dirPath);
CreateAlternateDataStream(dirPath, "hiddenstream", "hidden content");
await File.WriteAllTextAsync(Path.Combine(DATAFOLDER, "onefile.txt"), "content");
var backupOptions = new Dictionary<string, string>(TestOptions)
{
["enable-ads-backup"] = "true"
};
using (var c = new Controller("file://" + TARGETFOLDER, backupOptions, null))
TestUtils.AssertResults(await c.BackupAsync(new[] { DATAFOLDER }));
var restoreOptions = new Dictionary<string, string>(TestOptions)
{
["restore-path"] = RESTOREFOLDER,
["disable-ads-restore"] = "true"
};
using (var c = new Controller("file://" + TARGETFOLDER, restoreOptions, null))
TestUtils.AssertResults(await c.RestoreAsync(new[] { "*" }));
var restoredDirPath = Path.Combine(RESTOREFOLDER, "testdir");
Assert.IsTrue(Directory.Exists(restoredDirPath));
Assert.IsFalse(AlternateDataStreamExists(restoredDirPath, "hiddenstream"));
}
[Test]
[Category("Windows")]
public async Task BackupAndRestoreAlternateDataStreamsOnDirectoryLegacyAsync()
{
if (!OperatingSystem.IsWindows())
{
Assert.Ignore("This test is Windows-only.");
return;
}
var dirPath = Path.Combine(DATAFOLDER, "testdir");
Directory.CreateDirectory(dirPath);
CreateAlternateDataStream(dirPath, "hiddenstream", "hidden content");
var backupOptions = new Dictionary<string, string>(TestOptions)
{
["enable-ads-backup"] = "true"
};
using (var c = new Controller("file://" + TARGETFOLDER, backupOptions, null))
TestUtils.AssertResults(await c.BackupAsync(new[] { DATAFOLDER }));
var restoreOptions = new Dictionary<string, string>(TestOptions)
{
["restore-path"] = RESTOREFOLDER,
["restore-legacy"] = "true"
};
using (var c = new Controller("file://" + TARGETFOLDER, restoreOptions, null))
TestUtils.AssertResults(await c.RestoreAsync(new[] { dirPath }));
var restoredDirPath = Path.Combine(RESTOREFOLDER, "testdir");
Assert.IsTrue(Directory.Exists(restoredDirPath));
Assert.IsTrue(AlternateDataStreamExists(restoredDirPath, "hiddenstream"));
Assert.AreEqual("hidden content", ReadAlternateDataStream(restoredDirPath, "hiddenstream"));
}
[Test]
[Category("Windows")]
public async Task DisableAdsRestoreSkipsAlternateDataStreamsOnDirectoryLegacyAsync()
{
if (!OperatingSystem.IsWindows())
{
Assert.Ignore("This test is Windows-only.");
return;
}
var dirPath = Path.Combine(DATAFOLDER, "testdir");
Directory.CreateDirectory(dirPath);
CreateAlternateDataStream(dirPath, "hiddenstream", "hidden content");
await File.WriteAllTextAsync(Path.Combine(DATAFOLDER, "onefile.txt"), "content");
var backupOptions = new Dictionary<string, string>(TestOptions)
{
["enable-ads-backup"] = "true"
};
using (var c = new Controller("file://" + TARGETFOLDER, backupOptions, null))
TestUtils.AssertResults(await c.BackupAsync(new[] { DATAFOLDER }));
var restoreOptions = new Dictionary<string, string>(TestOptions)
{
["restore-path"] = RESTOREFOLDER,
["disable-ads-restore"] = "true",
["restore-legacy"] = "true"
};
using (var c = new Controller("file://" + TARGETFOLDER, restoreOptions, null))
TestUtils.AssertResults(await c.RestoreAsync(new[] { "*" }));
var restoredDirPath = Path.Combine(RESTOREFOLDER, "testdir");
Assert.IsTrue(Directory.Exists(restoredDirPath));
Assert.IsFalse(AlternateDataStreamExists(restoredDirPath, "hiddenstream"));
}
[Test]
[Category("Windows")]
public async Task BackupWithoutEnableAdsBackupIgnoresStreamsOnDirectoryAsync()
{
if (!OperatingSystem.IsWindows())
{
Assert.Ignore("This test is Windows-only.");
return;
}
var dirPath = Path.Combine(DATAFOLDER, "testdir");
Directory.CreateDirectory(dirPath);
CreateAlternateDataStream(dirPath, "hiddenstream", "hidden content");
await File.WriteAllTextAsync(Path.Combine(DATAFOLDER, "onefile.txt"), "content");
var backupOptions = new Dictionary<string, string>(TestOptions);
// enable-ads-backup is not set (defaults to false)
using (var c = new Controller("file://" + TARGETFOLDER, backupOptions, null))
TestUtils.AssertResults(await c.BackupAsync(new[] { DATAFOLDER }));
var restoreOptions = new Dictionary<string, string>(TestOptions)
{
["restore-path"] = RESTOREFOLDER
};
using (var c = new Controller("file://" + TARGETFOLDER, restoreOptions, null))
TestUtils.AssertResults(await c.RestoreAsync(new[] { "*" }));
var restoredDirPath = Path.Combine(RESTOREFOLDER, "testdir");
Assert.IsTrue(Directory.Exists(restoredDirPath));
Assert.IsFalse(AlternateDataStreamExists(restoredDirPath, "hiddenstream"));
}
}
}
Duplicati/UnitTest/ProblematicPathTests.cs
+19 -54
View File
@@ -89,31 +89,23 @@ namespace Duplicati.UnitTest
// Backup all files.
var options = new Dictionary<string, string>(this.TestOptions);
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
var backupResults = await c.BackupAsync(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER }));
// Restore all files.
var restoreOptions = new Dictionary<string, string>(options) { ["restore-path"] = this.RESTOREFOLDER };
using (var c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
{
var restoreResults = await c.RestoreAsync(null);
Assert.AreEqual(0, restoreResults.Errors.Count());
Assert.AreEqual(0, restoreResults.Warnings.Count());
TestUtils.AssertResults(await c.RestoreAsync(null));
TestUtils.AssertDirectoryTreesAreEquivalent(this.DATAFOLDER, this.RESTOREFOLDER, true, "Restore");
// List results using * should return a match for each directory.
var listResults = await c.ListAsync(SystemIO.IO_OS.PathCombine(dirWithAsterisk, file));
Assert.AreEqual(0, listResults.Errors.Count());
Assert.AreEqual(0, listResults.Warnings.Count());
TestUtils.AssertResults(listResults);
Assert.AreEqual(directories.Count, listResults.Files.Count());
listResults = await c.ListAsync(SystemIO.IO_OS.PathCombine(dirWithQuestionMark, file));
Assert.AreEqual(0, listResults.Errors.Count());
Assert.AreEqual(0, listResults.Warnings.Count());
TestUtils.AssertResults(listResults);
// List results using ? should return 3 matches in Linux,
// one for the directory with '*' and one for the directory
// with '?', plus one for directory 'X'; but should return
@@ -134,14 +126,11 @@ namespace Duplicati.UnitTest
// Verify that list result using verbatim identifier contains only one file.
var listResults = await c.ListAsync(verbatimFilePath);
Assert.AreEqual(0, listResults.Errors.Count());
Assert.AreEqual(0, listResults.Warnings.Count());
TestUtils.AssertResults(listResults);
Assert.AreEqual(1, listResults.Files.Count());
Assert.AreEqual(expectedFilePath, listResults.Files.Single().Path);
var restoreResults = await c.RestoreAsync(new[] { verbatimFilePath });
Assert.AreEqual(0, restoreResults.Errors.Count());
Assert.AreEqual(0, restoreResults.Warnings.Count());
TestUtils.AssertResults(await c.RestoreAsync(new[] { verbatimFilePath }));
var fileName = SystemIO.IO_OS.PathGetFileName(expectedFilePath);
var restoredFilePath = SystemIO.IO_OS.PathCombine(this.RESTOREFOLDER, fileName);
@@ -157,8 +146,7 @@ namespace Duplicati.UnitTest
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
var backupResults = await c.BackupAsync(new[] { this.DATAFOLDER }, filter);
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
TestUtils.AssertResults(backupResults);
Assert.AreEqual(directories.Count, backupResults.ExaminedFiles);
}
@@ -171,8 +159,7 @@ namespace Duplicati.UnitTest
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
var backupResults = await c.BackupAsync(new[] { this.DATAFOLDER }, filter);
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
TestUtils.AssertResults(backupResults);
Assert.AreEqual(verbatimAsteriskShouldMatchCount, backupResults.ExaminedFiles);
}
}
@@ -213,17 +200,12 @@ namespace Duplicati.UnitTest
var options = new Dictionary<string, string>(this.TestOptions);
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
var backupResults = await c.BackupAsync(new[] { this.DATAFOLDER }, filter);
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER }, filter));
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
var listResults = await c.ListAsync("*");
Assert.AreEqual(0, listResults.Errors.Count());
Assert.AreEqual(0, listResults.Warnings.Count());
TestUtils.AssertResults(listResults);
var backedUpPaths = listResults.Files.Select(x => x.Path).ToArray();
Assert.AreEqual(2, backedUpPaths.Length);
@@ -246,28 +228,18 @@ namespace Duplicati.UnitTest
var options = new Dictionary<string, string>(this.TestOptions);
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
var backupResults = await c.BackupAsync(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER }));
var restoreOptions = new Dictionary<string, string>(this.TestOptions) { ["restore-path"] = this.RESTOREFOLDER };
using (var c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
{
var restoreResults = await c.RestoreAsync(new[] { filePath });
Assert.AreEqual(0, restoreResults.Errors.Count());
Assert.AreEqual(0, restoreResults.Warnings.Count());
}
TestUtils.AssertResults(await c.RestoreAsync(new[] { filePath }));
var restoreFilePath = SystemIO.IO_OS.PathCombine(this.RESTOREFOLDER, fileName);
Assert.IsTrue(SystemIO.IO_OS.FileExists(restoreFilePath));
var restoredStream = new MemoryStream();
using (FileStream fileStream = SystemIO.IO_OS.FileOpenRead(restoreFilePath))
{
using (var fileStream = SystemIO.IO_OS.FileOpenRead(restoreFilePath))
Utility.CopyStream(fileStream, restoredStream);
}
Assert.AreEqual(fileBytes, restoredStream.ToArray());
}
@@ -283,6 +255,7 @@ namespace Duplicati.UnitTest
{
if (OperatingSystem.IsWindows() && skipOnWindows)
{
Assert.Ignore("This test is not for Windows.");
return;
}
@@ -296,20 +269,16 @@ namespace Duplicati.UnitTest
var options = new Dictionary<string, string>(this.TestOptions);
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
var backupResults = await c.BackupAsync(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
var r = await c.BackupAsync(new[] { this.DATAFOLDER });
Assert.AreNotEqual(r.AddedFiles, 0, "No files added");
TestUtils.AssertResults(r);
}
var restoreOptions = new Dictionary<string, string>(this.TestOptions) { ["restore-path"] = this.RESTOREFOLDER };
// Restore just the file.
using (var c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
{
var restoreResults = await c.RestoreAsync(new[] { filePath });
Assert.AreEqual(0, restoreResults.Errors.Count());
Assert.AreEqual(0, restoreResults.Warnings.Count());
}
TestUtils.AssertResults(await c.RestoreAsync(new[] { filePath }));
var restoreFilePath = SystemIO.IO_OS.PathCombine(this.RESTOREFOLDER, pathComponent);
TestUtils.AssertFilesAreEqual(filePath, restoreFilePath, true, pathComponent);
@@ -318,11 +287,7 @@ namespace Duplicati.UnitTest
// Restore the entire directory.
var pathSpec = $"[{Regex.Escape(Util.AppendDirSeparator(this.DATAFOLDER))}.*]";
using (var c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
{
var restoreResults = await c.RestoreAsync(new[] { pathSpec });
Assert.AreEqual(0, restoreResults.Errors.Count());
Assert.AreEqual(0, restoreResults.Warnings.Count());
}
TestUtils.AssertResults(await c.RestoreAsync(new[] { pathSpec }));
TestUtils.AssertDirectoryTreesAreEquivalent(this.DATAFOLDER, this.RESTOREFOLDER, true, pathComponent);
}
Duplicati/UnitTest/RestorePathTraversalTests.cs
+17 -4
View File
@@ -16,7 +16,7 @@ namespace Duplicati.UnitTest
{
[Test]
[Category("RestoreHandler")]
public async Task RestoreToSymlinkTargetAsync()
public async Task RestoreToSymlinkTargetAsync([Values(true, false)] bool legacyRestore)
{
if (OperatingSystem.IsWindows())
Assert.Ignore("Symlink tests are not supported on Windows");
@@ -53,12 +53,25 @@ namespace Duplicati.UnitTest
var restoreOptions = new Dictionary<string, string>(this.TestOptions);
restoreOptions["restore-path"] = restoreTarget;
restoreOptions["overwrite"] = "true";
restoreOptions["restore-legacy"] = legacyRestore.ToString();
restoreOptions["unittest-mode"] = "false";
using (var c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
{
// We expect a UserInformationException due to path traversal detection
var ex = NUnit.Framework.Assert.ThrowsAsync<UserInformationException>(() => c.RestoreAsync(null));
NUnit.Framework.Assert.That(ex.Message.Contains("Path traversal detected"), "Expected path traversal error message");
// Verify that path traversal is detected
var restoreResults = await c.RestoreAsync(null);
var foundError = restoreResults.Errors.Any(e => e.Contains("Path traversal detected"));
if (!foundError)
{
Console.WriteLine("Errors found:");
foreach (var e in restoreResults.Errors)
Console.WriteLine(e);
Console.WriteLine("Warnings found:");
foreach (var e in restoreResults.Warnings)
Console.WriteLine(e);
}
Assert.IsTrue(foundError, "Expected path traversal error message");
}
// 5. Verify vulnerability