haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updated file backend

Browse Source
This commit is contained in:
Kenneth Skovhede
2025-03-21 11:09:23 +01:00
parent bf27f3d4d3
commit dbe3a4eeac
3 changed files with 84 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/Library/Backend/File/Duplicati.Library.Backend.File.csproj
+2
View File
@@ -5,6 +5,8 @@
<OutputType>Library</OutputType>
<RootNamespace>Duplicati.Library.Backend</RootNamespace>
<Copyright>Copyright © 2025 Team Duplicati, MIT license</Copyright>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>
<ItemGroup>
Duplicati/Library/Backend/File/FileBackend.cs
+62 -61
View File
@@ -21,14 +21,11 @@
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Duplicati.Library.Utility;
using Duplicati.Library.Utility.Options;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Versioning;
using System.Threading;
using System.Threading.Tasks;
namespace Duplicati.Library.Backend
{
@@ -43,71 +40,65 @@ namespace Duplicati.Library.Backend
private const string OPTION_DISABLE_LENGTH_VERIFICATION = "disable-length-verification";
private readonly string m_path;
private string m_username;
private string m_password;
private string? m_username;
private string? m_password;
private readonly bool m_moveFile;
private bool m_hasAutenticated;
private readonly bool m_forceReauth;
private readonly bool m_verifyDestinationLength;
private readonly TimeoutOptionsHelper.Timeouts m_timeouts;
private static readonly ISystemIO systemIO = SystemIO.IO_OS;
public File()
{
m_path = null!;
m_timeouts = null!;
}
public File(string url, Dictionary<string, string> options)
public File(string url, Dictionary<string, string?> options)
{
var uri = new Utility.Uri(url);
m_path = uri.HostAndPath;
var path = uri.HostAndPath;
var auth = AuthOptionsHelper.Parse(options, uri);
m_timeouts = TimeoutOptionsHelper.Parse(options);
m_username = auth.Username;
m_password = auth.Password;
if (options.ContainsKey("auth-username"))
m_username = options["auth-username"];
if (options.ContainsKey("auth-password"))
m_password = options["auth-password"];
if (!string.IsNullOrEmpty(uri.Username))
m_username = uri.Username;
if (!string.IsNullOrEmpty(uri.Password))
m_password = uri.Password;
if (!Path.IsPathRooted(path))
path = systemIO.PathGetFullPath(path);
if (!System.IO.Path.IsPathRooted(m_path))
m_path = systemIO.PathGetFullPath(m_path);
if (options.ContainsKey(OPTION_ALTERNATE_PATHS))
var altPaths = options.GetValueOrDefault(OPTION_ALTERNATE_PATHS);
if (!string.IsNullOrWhiteSpace(altPaths))
{
List<string> paths =
[
m_path,
.. options[OPTION_ALTERNATE_PATHS].Split(new string[] { System.IO.Path.PathSeparator.ToString() }, StringSplitOptions.RemoveEmptyEntries),
path,
.. altPaths.Split(new string[] {Path.PathSeparator.ToString() }, StringSplitOptions.RemoveEmptyEntries),
];
//On windows we expand the drive letter * to all drives
if (OperatingSystem.IsWindows())
{
System.IO.DriveInfo[] drives = System.IO.DriveInfo.GetDrives();
for (int i = 0; i < paths.Count; i++)
var drives = DriveInfo.GetDrives();
for (var i = 0; i < paths.Count; i++)
{
if (paths[i].StartsWith("*:", StringComparison.Ordinal))
{
string rpl_path = paths[i].Substring(1);
var rpl_path = paths[i].Substring(1);
paths.RemoveAt(i);
i--;
foreach (System.IO.DriveInfo di in drives)
foreach (var di in drives)
paths.Insert(++i, di.Name[0] + rpl_path);
}
}
}
string markerfile = null;
//If there is a marker file, we do not allow the primary target path
// to be accepted, unless it contains the marker file
if (options.ContainsKey(OPTION_DESTINATION_MARKER))
{
markerfile = options[OPTION_DESTINATION_MARKER];
m_path = null;
}
var markerfile = options.GetValueOrDefault(OPTION_DESTINATION_MARKER);
if (!string.IsNullOrWhiteSpace(markerfile))
path = null;
foreach (string p in paths)
{
@@ -115,7 +106,7 @@ namespace Duplicati.Library.Backend
{
if (systemIO.DirectoryExists(p) && (markerfile == null || systemIO.FileExists(systemIO.PathCombine(p, markerfile))))
{
m_path = p;
path = p;
break;
}
}
@@ -124,10 +115,11 @@ namespace Duplicati.Library.Backend
}
}
if (m_path == null)
if (string.IsNullOrWhiteSpace(path))
throw new UserInformationException(Strings.FileBackend.NoDestinationWithMarkerFileError(markerfile, paths.ToArray()), "NoDestinationWithMarker");
}
m_path = path;
m_moveFile = Utility.Utility.ParseBoolOption(options, OPTION_MOVE_FILE);
m_forceReauth = Utility.Utility.ParseBoolOption(options, OPTION_FORCE_REAUTH);
m_verifyDestinationLength = !Utility.Utility.ParseBoolOption(options, OPTION_DISABLE_LENGTH_VERIFICATION);
@@ -174,14 +166,19 @@ namespace Duplicati.Library.Backend
}
/// <inheritdoc />
public IAsyncEnumerable<IFileEntry> ListAsync(CancellationToken cancelToken)
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancelToken)
{
PreAuthenticate();
if (!systemIO.DirectoryExists(m_path))
throw new FolderMissingException(Strings.FileBackend.FolderMissingError(m_path));
return systemIO.EnumerateFileEntries(m_path).ToAsyncEnumerable();
var res = await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, _ =>
systemIO.EnumerateFileEntries(m_path)
).ConfigureAwait(false);
foreach (var entry in res)
yield return entry;
}
#if DEBUG_RETRY
@@ -201,17 +198,19 @@ namespace Duplicati.Library.Backend
string targetFilePath = GetRemoteName(targetFilename);
long copiedBytes = 0;
using (var targetStream = systemIO.FileCreate(targetFilePath))
copiedBytes = await Utility.Utility.CopyStreamAsync(sourceStream, targetStream, true, cancelToken).ConfigureAwait(false);
using (var timeoutStream = targetStream.ObserveWriteTimeout(m_timeouts.ReadWriteTimeout))
copiedBytes = await Utility.Utility.CopyStreamAsync(sourceStream, timeoutStream, true, cancelToken).ConfigureAwait(false);
VerifyMatchingSize(targetFilePath, sourceStream, copiedBytes);
}
#endif
public async Task GetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken)
public async Task GetAsync(string remotename, Stream stream, CancellationToken cancelToken)
{
// FileOpenRead has flags System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read
using (var readstream = systemIO.FileOpenRead(GetRemoteName(remotename)))
await Utility.Utility.CopyStreamAsync(readstream, stream, true, cancelToken).ConfigureAwait(false);
using (var timeoutStream = readstream.ObserveReadTimeout(m_timeouts.ReadWriteTimeout))
await Utility.Utility.CopyStreamAsync(timeoutStream, stream, true, cancelToken).ConfigureAwait(false);
}
public Task PutAsync(string targetFilename, string sourceFilePath, CancellationToken cancelToken)
@@ -245,10 +244,11 @@ namespace Duplicati.Library.Backend
return Task.CompletedTask;
}
public Task DeleteAsync(string remotename, CancellationToken cancelToken)
public async Task DeleteAsync(string remotename, CancellationToken cancelToken)
{
systemIO.FileDelete(GetRemoteName(remotename));
return Task.CompletedTask;
await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _
=> systemIO.FileDelete(GetRemoteName(remotename))
).ConfigureAwait(false);
}
public IList<ICommandLineArgument> SupportedCommands
@@ -258,8 +258,7 @@ namespace Duplicati.Library.Backend
var lst = new List<ICommandLineArgument>();
if (OperatingSystem.IsWindows())
lst.AddRange([
new CommandLineArgument("auth-username", CommandLineArgument.ArgumentType.String, Strings.FileBackend.DescriptionAuthUsernameShort, Strings.FileBackend.DescriptionAuthUsernameLong),
new CommandLineArgument("auth-password", CommandLineArgument.ArgumentType.Password, Strings.FileBackend.DescriptionAuthPasswordShort, Strings.FileBackend.DescriptionAuthPasswordLong),
.. AuthOptionsHelper.GetOptions(),
new CommandLineArgument(OPTION_FORCE_REAUTH, CommandLineArgument.ArgumentType.Boolean, Strings.FileBackend.ForceReauthShort, Strings.FileBackend.ForceReauthLong)
]);
@@ -268,6 +267,7 @@ namespace Duplicati.Library.Backend
new CommandLineArgument(OPTION_ALTERNATE_PATHS, CommandLineArgument.ArgumentType.Path, Strings.FileBackend.AlternateTargetPathsShort, Strings.FileBackend.AlternateTargetPathsLong(OPTION_DESTINATION_MARKER, System.IO.Path.PathSeparator)),
new CommandLineArgument(OPTION_MOVE_FILE, CommandLineArgument.ArgumentType.Boolean, Strings.FileBackend.UseMoveForPutShort, Strings.FileBackend.UseMoveForPutLong),
new CommandLineArgument(OPTION_DISABLE_LENGTH_VERIFICATION, CommandLineArgument.ArgumentType.Boolean, Strings.FileBackend.DisableLengthVerificationShort, Strings.FileBackend.DisableLengthVerificationLong),
.. TimeoutOptionsHelper.GetOptions()
]);
return lst;
@@ -285,13 +285,14 @@ namespace Duplicati.Library.Backend
public Task TestAsync(CancellationToken cancelToken)
=> this.TestListAsync(cancelToken);
public Task CreateFolderAsync(CancellationToken cancelToken)
public async Task CreateFolderAsync(CancellationToken cancelToken)
{
if (systemIO.DirectoryExists(m_path))
throw new FolderAreadyExistedException();
systemIO.DirectoryCreate(m_path);
return Task.CompletedTask;
await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _
=> systemIO.DirectoryCreate(m_path)
).ConfigureAwait(false);
}
#endregion
@@ -306,7 +307,7 @@ namespace Duplicati.Library.Backend
#endregion
private System.IO.DriveInfo GetDrive()
private DriveInfo? GetDrive()
{
string root;
if (!OperatingSystem.IsWindows())
@@ -314,7 +315,7 @@ namespace Duplicati.Library.Backend
string path = Util.AppendDirSeparator(systemIO.PathGetFullPath(m_path));
// If the built-in .NET DriveInfo works, use it
try { return new System.IO.DriveInfo(path); }
try { return new DriveInfo(path); }
catch { }
root = "/";
@@ -322,7 +323,7 @@ namespace Duplicati.Library.Backend
//Find longest common prefix from mounted devices
//TODO: Can trick this with symlinks, where the symlink is on one mounted volume,
// and the actual storage is on another
foreach (System.IO.DriveInfo di in System.IO.DriveInfo.GetDrives())
foreach (var di in DriveInfo.GetDrives())
if (path.StartsWith(Util.AppendDirSeparator(di.Name), StringComparison.Ordinal) && di.Name.Length > root.Length)
root = di.Name;
}
@@ -337,7 +338,7 @@ namespace Duplicati.Library.Backend
{
try
{
return new System.IO.DriveInfo(root);
return new DriveInfo(root);
}
catch (ArgumentException)
{
@@ -348,7 +349,7 @@ namespace Duplicati.Library.Backend
return null;
}
public Task<IQuotaInfo> GetQuotaInfoAsync(CancellationToken cancelToken)
public Task<IQuotaInfo?> GetQuotaInfoAsync(CancellationToken cancelToken)
{
var driveInfo = this.GetDrive();
if (driveInfo != null)
@@ -357,7 +358,7 @@ namespace Duplicati.Library.Backend
// If the drive actually has a total size of 0, this should be obvious immediately due to write errors
if (driveInfo.TotalSize > 0)
{
return Task.FromResult<IQuotaInfo>(new QuotaInfo(driveInfo.TotalSize, driveInfo.AvailableFreeSpace));
return Task.FromResult<IQuotaInfo?>(new QuotaInfo(driveInfo.TotalSize, driveInfo.AvailableFreeSpace));
}
}
@@ -365,10 +366,10 @@ namespace Duplicati.Library.Backend
{
// If we can't get the DriveInfo on Windows, fallback to GetFreeDiskSpaceEx
// https://stackoverflow.com/questions/2050343/programmatically-determining-space-available-from-unc-path
return Task.FromResult<IQuotaInfo>(GetDiskFreeSpace(m_path));
return Task.FromResult<IQuotaInfo?>(GetDiskFreeSpace(m_path));
}
return null;
return Task.FromResult<IQuotaInfo?>(null);
}
public Task<string[]> GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(Array.Empty<string>());
@@ -391,7 +392,7 @@ namespace Duplicati.Library.Backend
/// <returns>Quota info</returns>
[System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)]
[SupportedOSPlatform("windows")]
public static QuotaInfo GetDiskFreeSpace(string directory)
public static QuotaInfo? GetDiskFreeSpace(string directory)
{
ulong available;
ulong total;
@@ -439,7 +440,7 @@ namespace Duplicati.Library.Backend
}
}
private static void VerifyMatchingSize(string targetFilePath, Stream sourceStream, long? expectedLength)
private static void VerifyMatchingSize(string targetFilePath, Stream? sourceStream, long? expectedLength)
{
try
{
Duplicati/Library/Backend/File/Strings.cs
+20 -24
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Localization.Short;
namespace Duplicati.Library.Backend.Strings
@@ -29,12 +29,8 @@ namespace Duplicati.Library.Backend.Strings
public static string AlternateDestinationMarkerShort { get { return LC.L(@"Look for a file in the destination folder"); } }
public static string AlternateTargetPathsLong(string optionname, char pathseparator) { return LC.L(@"This option allows multiple targets to be specified. The primary target path is placed before the list of paths supplied with this option. Before starting the backup, each folder in the list is checked for existence and optionally the presence of the marker file supplied by --{0}. The first existing path that optionally contains the marker file is then used as the destination. Multiple destinations are separated with a ""{1}"". On Windows, the path may be a UNC path, and the drive letter may be substituted with an asterisk (*), e.g: ""*:\backup"", which will examine all drive letters. If a username and password is supplied, the same credentials are used for all destinations.", optionname, pathseparator); }
public static string AlternateTargetPathsShort { get { return LC.L(@"A list of secondary target paths"); } }
public static string DescriptionAuthPasswordLong { get { return LC.L(@"The password used to connect to the server. This may also be supplied as the environment variable ""AUTH_PASSWORD""."); } }
public static string DescriptionAuthPasswordShort { get { return LC.L(@"Supply the password used to connect to the server"); } }
public static string DescriptionAuthUsernameLong { get { return LC.L(@"The username used to connect to the server. This may also be supplied as the environment variable ""AUTH_USERNAME""."); } }
public static string DescriptionAuthUsernameShort { get { return LC.L(@"Supply the username used to connect to the server"); } }
public static string FolderMissingError(string foldername) { return LC.L(@"The folder {0} does not exist", foldername); }
public static string NoDestinationWithMarkerFileError(string markername, string[] folders) { return LC.L(@"The marker file ""{0}"" was not found in any of the examined destinations: {1}", markername, string.Join(", ", folders)); }
public static string NoDestinationWithMarkerFileError(string? markername, string[] folders) { return LC.L(@"The marker file ""{0}"" was not found in any of the examined destinations: {1}", markername, string.Join(", ", folders)); }
public static string UseMoveForPutLong { get { return LC.L(@"When storing the file, the standard operation is to copy the file and delete the original. This sequence ensures that the operation can be retried if something goes wrong. Activating this option may cause the retry operation to fail. This option has no effect unless the option --{0} is activated.", "disable-streaming-transfers"); } }
public static string UseMoveForPutShort { get { return LC.L(@"Move the file instead of copying it"); } }
public static string ForceReauthLong { get { return LC.L(@"If this option is set, any existing authentication against the remote share is dropped before attempting to authenticate."); } }