haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #6057 from duplicati/feature/unified-timeout-and-options

Browse Source 
Unified options
This commit is contained in:
Kenneth Skovhede
2025-03-19 21:20:53 +01:00
committed by GitHub
parent af7bbb4164 8f38c82f1e
commit 7d94d86643
15 changed files with 686 additions and 273 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/Library/Backend/FTP/AlternateFTPBackend.cs
+6 -4
View File
@@ -22,6 +22,7 @@ using System;
using System.Collections.Generic;
using System.Security.Authentication;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility.Options;
using FluentFTP;
namespace Duplicati.Library.Backend;
@@ -68,9 +69,8 @@ public class AlternateFTPBackend : FTP
/// Overriden so that for the alternate FTP backend, we need to support the alternate config keys starting with "aftp"
/// </summary>
public override IList<ICommandLineArgument> SupportedCommands =>
new List<ICommandLineArgument>([
new CommandLineArgument("auth-password", CommandLineArgument.ArgumentType.Password, Strings.DescriptionAuthPasswordShort, Strings.DescriptionAuthPasswordLong),
new CommandLineArgument("auth-username", CommandLineArgument.ArgumentType.String, Strings.DescriptionAuthUsernameShort, Strings.DescriptionAuthUsernameLong),
[
.. AuthOptionsHelper.GetOptions(),
new CommandLineArgument(CONFIG_KEY_DISABLE_UPLOAD_VERIFY, CommandLineArgument.ArgumentType.Boolean, Strings.DescriptionDisableUploadVerifyShort, Strings.DescriptionDisableUploadVerifyLong),
new CommandLineArgument(CONFIG_KEY_FTP_RELATIVE_PATH, CommandLineArgument.ArgumentType.Boolean, Strings.DescriptionAbsolutePathShort, Strings.DescriptionAbsolutePathLong),
new CommandLineArgument(CONFIG_KEY_FTP_USE_CWD_NAMES, CommandLineArgument.ArgumentType.Boolean, Strings.DescriptionUseCwdNamesShort, Strings.DescriptionUseCwdNamesLong),
@@ -80,5 +80,7 @@ public class AlternateFTPBackend : FTP
new CommandLineArgument(CONFIG_KEY_FTP_UPLOAD_DELAY, CommandLineArgument.ArgumentType.Timespan, Strings.DescriptionUploadDelayShort, Strings.DescriptionUploadDelayLong, DEFAULT_UPLOAD_DELAY_STRING),
new CommandLineArgument(CONFIG_KEY_FTP_LOGTOCONSOLE, CommandLineArgument.ArgumentType.Boolean, Strings.DescriptionLogToConsoleShort, Strings.DescriptionLogToConsoleLong),
new CommandLineArgument(CONFIG_KEY_FTP_LOGPRIVATEINFOTOCONSOLE, CommandLineArgument.ArgumentType.Boolean, Strings.DescriptionLogPrivateInfoToConsoleShort, Strings.DescriptionLogPrivateInfoToConsoleLong, "false"),
]);
.. SslOptionsHelper.GetCertOnlyOptions(),
.. TimeoutOptionsHelper.GetOptions(),
];
}
Duplicati/Library/Backend/FTP/FTPBackend.cs
+85 -78
View File
@@ -1,4 +1,4 @@
// Copyright (C) 2025, The Duplicati Team
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
@@ -36,6 +36,7 @@ using System.Threading.Tasks;
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
using Duplicati.Library.Utility.Options;
using FluentFTP;
using FluentFTP.Client.BaseClient;
using FluentFTP.Exceptions;
@@ -61,15 +62,6 @@ namespace Duplicati.Library.Backend
/// The credentials used to authenticate with the FTP server
/// </summary>
private readonly NetworkCredential? _userInfo;
/// <summary>
/// Th option used to accept a specific SSL certificate hash
/// </summary>
private const string OPTION_ACCEPT_SPECIFIED_CERTIFICATE = "accept-specified-ssl-hash"; // Global option
/// <summary>
/// The option used to accept any SSL certificate
/// </summary>
private const string OPTION_ACCEPT_ANY_CERTIFICATE = "accept-any-ssl-certificate"; // Global option
/// <summary>
/// The default data connection type
/// </summary>
@@ -207,14 +199,13 @@ namespace Duplicati.Library.Backend
/// </summary>
private readonly bool _diagnosticsLog;
/// <summary>
/// The flag to indicate if all certificates should be accepted
/// The ssl certificate options to use
/// </summary>
private readonly bool _accepAllCertificates;
private readonly SslOptionsHelper.SslCertificateOptions _sslOptions;
/// <summary>
/// The valid certificate hashes
/// The timeout options to use
/// </summary>
private readonly string[] _validHashes;
private readonly TimeoutOptionsHelper.Timeouts _timeouts;
/// <summary>
/// The localized name to display for this backend
/// </summary>
@@ -237,9 +228,8 @@ namespace Duplicati.Library.Backend
/// <inheritdoc />
public virtual IList<ICommandLineArgument> SupportedCommands =>
new List<ICommandLineArgument>([
new CommandLineArgument("auth-password", CommandLineArgument.ArgumentType.Password, Strings.DescriptionAuthPasswordShort, Strings.DescriptionAuthPasswordLong),
new CommandLineArgument("auth-username", CommandLineArgument.ArgumentType.String, Strings.DescriptionAuthUsernameShort, Strings.DescriptionAuthUsernameLong),
[
.. AuthOptionsHelper.GetOptions(),
new CommandLineArgument(CONFIG_KEY_DISABLE_UPLOAD_VERIFY, CommandLineArgument.ArgumentType.Boolean, Strings.DescriptionDisableUploadVerifyShort, Strings.DescriptionDisableUploadVerifyLong),
new CommandLineArgument(CONFIG_KEY_FTP_ABSOLUTE_PATH, CommandLineArgument.ArgumentType.Boolean, Strings.DescriptionAbsolutePathShort, Strings.DescriptionAbsolutePathLong),
new CommandLineArgument(CONFIG_KEY_FTP_USE_CWD_NAMES, CommandLineArgument.ArgumentType.Boolean, Strings.DescriptionUseCwdNamesShort, Strings.DescriptionUseCwdNamesLong),
@@ -253,7 +243,9 @@ namespace Duplicati.Library.Backend
new CommandLineArgument(CONFIG_KEY_FTP_LEGACY_FTPPASSIVE, CommandLineArgument.ArgumentType.Boolean, Strings.DescriptionFTPPassiveShort, Strings.DescriptionFTPPassiveLong, "false", null, null, Strings.FtpPassiveDeprecated),
new CommandLineArgument(CONFIG_KEY_FTP_LEGACY_FTPREGULAR, CommandLineArgument.ArgumentType.Boolean, Strings.DescriptionFTPActiveShort, Strings.DescriptionFTPActiveLong, "true", null, null, Strings.FtpActiveDeprecated),
new CommandLineArgument(CONFIG_KEY_FTP_LEGACY_USESSL, CommandLineArgument.ArgumentType.Boolean, Strings.DescriptionUseSSLShort, Strings.DescriptionUseSSLLong, "false", null, null, Strings.UseSslDeprecated),
]);
.. SslOptionsHelper.GetCertOnlyOptions(),
.. TimeoutOptionsHelper.GetOptions(),
];
/// <summary>
/// Initialize a new instance.
@@ -261,7 +253,8 @@ namespace Duplicati.Library.Backend
public FTP()
{
// TODO: Remove this constructor once static properties are introduced on IBackend
_validHashes = null!;
_sslOptions = null!;
_timeouts = null!;
_ftpConfig = null!;
_url = null!;
_client = null!;
@@ -275,11 +268,7 @@ namespace Duplicati.Library.Backend
[SuppressMessage("ReSharper", "VirtualMemberCallInConstructor")] // The behavior of accessing the virtual properties is as expected
public FTP(string url, Dictionary<string, string?> options)
{
_accepAllCertificates = CoreUtility.ParseBoolOption(options, OPTION_ACCEPT_ANY_CERTIFICATE);
options.TryGetValue(OPTION_ACCEPT_SPECIFIED_CERTIFICATE, out var certHash);
_validHashes = certHash?.Split([",", ";"], StringSplitOptions.RemoveEmptyEntries) ?? [];
_sslOptions = SslOptionsHelper.SslCertificateOptions.Parse(options);
var u = new Utility.Uri(url);
u.RequireHost();
@@ -352,6 +341,7 @@ namespace Duplicati.Library.Backend
_logToConsole = CoreUtility.ParseBoolOption(options, CONFIG_KEY_FTP_LOGTOCONSOLE);
_logPrivateInfoToConsole = CoreUtility.ParseBoolOption(options, CONFIG_KEY_FTP_LOGPRIVATEINFOTOCONSOLE);
_diagnosticsLog = CoreUtility.ParseBoolOption(options, CONFIG_KEY_FTP_LOGDIAGNOSTICS);
_timeouts = TimeoutOptionsHelper.Timeouts.Parse(options);
_ftpConfig = new FtpConfig
{
@@ -359,7 +349,7 @@ namespace Duplicati.Library.Backend
EncryptionMode = encryptionMode,
SslProtocols = sslProtocols,
LogToConsole = _logToConsole,
ValidateAnyCertificate = _accepAllCertificates,
ValidateAnyCertificate = _sslOptions.AcceptAllCertificates,
Noop = true
};
@@ -372,9 +362,12 @@ namespace Duplicati.Library.Backend
FtpListItem[] items;
try
{
var client = CreateClient(CancellationToken.None).Await();
var client = await CreateClient(cancelToken).ConfigureAwait(false);
var remotePath = PreparePathForClient(null);
items = await client.GetListing(remotePath, FtpListOption.Modify | FtpListOption.Size).ConfigureAwait(false);
items = await Utility.Utility.WithTimeout(_timeouts.ListTimeout, cancelToken, ct =>
client.GetListing(remotePath, FtpListOption.Modify | FtpListOption.Size, ct)
).ConfigureAwait(false);
}
catch (Exception e)
{
@@ -445,7 +438,8 @@ namespace Duplicati.Library.Backend
try { streamLen = input.Length; }
catch (NotSupportedException) { }
var status = await client.UploadStream(input, clientRemoteName, createRemoteDir: false, token: cancelToken, progress: null).ConfigureAwait(false);
using var timeoutStream = input.ObserveReadTimeout(_timeouts.ReadWriteTimeout, false);
var status = await client.UploadStream(timeoutStream, clientRemoteName, createRemoteDir: false, token: cancelToken, progress: null).ConfigureAwait(false);
if (status != FtpStatus.Success)
throw new UserInformationException(Strings.ErrorWriteFile(remotename, $"Status is {status}"), "FtpPutFailure");
@@ -485,6 +479,7 @@ namespace Duplicati.Library.Backend
var client = await CreateClient(cancelToken).ConfigureAwait(false);
var clientRemoteName = PreparePathForClient(remotename);
await using var inputStream = await client.OpenRead(clientRemoteName, token: cancelToken);
await using var timeoutStream = inputStream.ObserveReadTimeout(_timeouts.ReadWriteTimeout);
await CoreUtility.CopyStreamAsync(inputStream, output, false, cancelToken).ConfigureAwait(false);
}
catch (Exception e)
@@ -510,7 +505,9 @@ namespace Duplicati.Library.Backend
{
var client = await CreateClient(cancelToken).ConfigureAwait(false);
var clientRemoteName = PreparePathForClient(remotename);
await client.DeleteFile(clientRemoteName, cancelToken);
await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancelToken, ct =>
client.DeleteFile(clientRemoteName, ct)
).ConfigureAwait(false);
}
catch (Exception e)
{
@@ -535,11 +532,14 @@ namespace Duplicati.Library.Backend
try
{
var client = await CreateClient(cancellationToken).ConfigureAwait(false);
if (!_useCwdNames)
await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct =>
{
var folderpath = PreparePathForClient(null);
await client.SetWorkingDirectory(folderpath, cancellationToken).ConfigureAwait(false);
}
if (!_useCwdNames)
{
var folderpath = PreparePathForClient(null);
await client.SetWorkingDirectory(folderpath, ct).ConfigureAwait(false);
}
}).ConfigureAwait(false);
}
catch (Exception e)
{
@@ -618,28 +618,32 @@ namespace Duplicati.Library.Backend
{
try
{
// Try to create the directory
var client = await CreateClient(cancellationToken, false).ConfigureAwait(false);
var clientPath = PreparePathForClient(null, false);
if (_useCwdNames && clientPath.Contains('/') && clientPath != "/")
{
// Go to the parent folder and create the folder
var parentPath = clientPath.Substring(0, clientPath.LastIndexOf('/'));
var folderName = clientPath.Substring(clientPath.LastIndexOf('/') + 1);
await client.SetWorkingDirectory(parentPath, cancellationToken).ConfigureAwait(false);
await client.CreateDirectory(folderName, true, cancellationToken).ConfigureAwait(false);
// Reset the client and check that it works
_client = null;
client = await CreateClient(cancellationToken).ConfigureAwait(false);
var cwd = await client.GetWorkingDirectory(cancellationToken).ConfigureAwait(false);
if (!string.Equals(cwd?.TrimEnd('/'), clientPath, StringComparison.OrdinalIgnoreCase))
throw new UserInformationException(Strings.ErrorCreateFolder(clientPath, cwd), "CreateFolderError");
}
else
await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct =>
{
await client.CreateDirectory(clientPath, true, cancellationToken);
}
// Try to create the directory
if (_useCwdNames && clientPath.Contains('/') && clientPath != "/")
{
// Go to the parent folder and create the folder
var parentPath = clientPath.Substring(0, clientPath.LastIndexOf('/'));
var folderName = clientPath.Substring(clientPath.LastIndexOf('/') + 1);
await client.SetWorkingDirectory(parentPath, ct).ConfigureAwait(false);
await client.CreateDirectory(folderName, true, ct).ConfigureAwait(false);
// Reset the client and check that it works
_client = null;
client = await CreateClient(ct).ConfigureAwait(false);
var cwd = await client.GetWorkingDirectory(ct).ConfigureAwait(false);
if (!string.Equals(cwd?.TrimEnd('/'), clientPath, StringComparison.OrdinalIgnoreCase))
throw new UserInformationException(Strings.ErrorCreateFolder(clientPath, cwd), "CreateFolderError");
}
else
{
await client.CreateDirectory(clientPath, true, ct);
}
}).ConfigureAwait(false);
}
catch (Exception ex)
{
@@ -665,33 +669,36 @@ namespace Duplicati.Library.Backend
{
if (_client == null)
{
var client = new AsyncFtpClient
_client = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct =>
{
Host = _url.Host,
Port = _url.Port == -1 ? 21 : _url.Port,
Credentials = _userInfo,
Config = _ftpConfig,
Logger = _diagnosticsLog ? new DiagnosticsLogger() : null
};
var client = new AsyncFtpClient
{
Host = _url.Host,
Port = _url.Port == -1 ? 21 : _url.Port,
Credentials = _userInfo,
Config = _ftpConfig,
Logger = _diagnosticsLog ? new DiagnosticsLogger() : null
};
client.ValidateCertificate += HandleValidateCertificate;
await client.Connect(cancellationToken).ConfigureAwait(false);
client.ValidateCertificate += HandleValidateCertificate;
await client.Connect(ct).ConfigureAwait(false);
// Set up for relative paths
if (_relativePaths)
{
_initialCwd = await client.GetWorkingDirectory(cancellationToken).ConfigureAwait(false);
_initialCwd = _initialCwd?.TrimEnd('/');
}
// Set up for relative paths
if (_relativePaths)
{
_initialCwd = await client.GetWorkingDirectory(ct).ConfigureAwait(false);
_initialCwd = _initialCwd?.TrimEnd('/');
}
// Setup the initial working directory, if needed
if (cwdFlag ?? _useCwdNames)
{
var clientPath = PreparePathForClient(null, false, client);
await client.SetWorkingDirectory(clientPath, cancellationToken).ConfigureAwait(false);
}
// Setup the initial working directory, if needed
if (cwdFlag ?? _useCwdNames)
{
var clientPath = PreparePathForClient(null, false, client);
await client.SetWorkingDirectory(clientPath, ct).ConfigureAwait(false);
}
_client = client;
return client;
}).ConfigureAwait(false);
}
return _client;
}
@@ -703,7 +710,7 @@ namespace Duplicati.Library.Backend
/// <param name="e">The event arguments.</param>
private void HandleValidateCertificate(BaseFtpClient control, FtpSslValidationEventArgs e)
{
if (e.PolicyErrors == SslPolicyErrors.None || _accepAllCertificates)
if (e.PolicyErrors == SslPolicyErrors.None || _sslOptions.AcceptAllCertificates)
{
e.Accept = true;
return;
@@ -712,8 +719,8 @@ namespace Duplicati.Library.Backend
e.Accept = false;
try
{
var certHash = (_validHashes != null && _validHashes.Length > 0) ? e.Certificate?.GetCertHashString() : null;
if (certHash != null && _validHashes != null && _validHashes.Any(hash => !string.IsNullOrEmpty(hash) && certHash.Equals(hash, StringComparison.OrdinalIgnoreCase)))
var certHash = (_sslOptions.AcceptSpecificCertificateHashes != null && _sslOptions.AcceptSpecificCertificateHashes.Length > 0) ? e.Certificate?.GetCertHashString() : null;
if (certHash != null && _sslOptions.AcceptSpecificCertificateHashes != null && _sslOptions.AcceptSpecificCertificateHashes.Any(hash => !string.IsNullOrEmpty(hash) && certHash.Equals(hash, StringComparison.OrdinalIgnoreCase)))
e.Accept = true;
}
catch
@@ -721,7 +728,7 @@ namespace Duplicati.Library.Backend
}
if (e.Accept == false && e.Certificate != null)
throw new SslCertificateValidator.InvalidCertificateException(e.Certificate?.GetCertHashString(), e.PolicyErrors);
throw new SslCertificateValidator.InvalidCertificateException(e.Certificate?.GetCertHashString() ?? "<unknown>", e.PolicyErrors);
}
/// <summary>
Duplicati/Library/Backend/FTP/Strings.cs
-4
View File
@@ -31,10 +31,6 @@ namespace Duplicati.Library.Backend
public static string DescriptionAlternate { get { return LC.L(@"This backend can read and write data to an FTP based backend. Allowed formats are ""aftp://hostname/folder"" and ""aftp://username:password@hostname/folder""."); } }
public static string DisplayName { get { return LC.L(@"FTP"); } }
public static string DisplayNameAlternate { get { return LC.L(@"Alternative FTP"); } }
public static string DescriptionAuthPasswordLong { get { return LC.L(@"The password used to connect to the server. This may also be supplied as the environment variable ""AUTH_PASSWORD""."); } }
public static string DescriptionAuthPasswordShort { get { return LC.L(@"Supply the password used to connect to the server"); } }
public static string DescriptionAuthUsernameLong { get { return LC.L(@"The username used to connect to the server. This may also be supplied as the environment variable ""AUTH_USERNAME""."); } }
public static string DescriptionAuthUsernameShort { get { return LC.L(@"Supply the username used to connect to the server"); } }
public static string DescriptionLogToConsoleLong { get { return LC.L(@"Use this option to log FTP dialog to terminal console for debugging purposes."); } }
public static string DescriptionLogToConsoleShort { get { return LC.L(@"Log FTP dialog to terminal console"); } }
public static string DescriptionLogPrivateInfoToConsoleLong { get { return LC.L(@"Use this option to log FTP PRIVATE info (username, password) to console for debugging purposes (DO NOT POST THIS TO THE INTERNET!)"); } }
Duplicati/Library/Backend/WEBDAV/Strings.cs
+7 -14
View File
@@ -19,30 +19,23 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Localization.Short;
namespace Duplicati.Library.Backend.Strings {
internal static class WEBDAV {
namespace Duplicati.Library.Backend.Strings
{
internal static class WEBDAV
{
public static string Description { get { return LC.L(@"This backend can read and write data to a WEBDAV enabled web server, using the HTTP protocol. Allowed formats are ""webdav://hostname/folder"" and ""webdav://username:password@hostname/folder""."); } }
public static string DisplayName { get { return LC.L(@"WebDAV"); } }
public static string DescriptionForceDigestLong { get { return LC.L(@"Using the HTTP Digest authentication method allows the user to authenticate with the server, without sending the password in clear. However, a man-in-the-middle attack is easy, because the HTTP protocol specifies a fallback to Basic authentication, which will make the client send the password to the attacker. Using this option, the client does not accept this, and always uses Digest authentication or fails to connect."); } }
public static string DescriptionForceDigestShort { get { return LC.L(@"Force the use of the HTTP Digest authentication method"); } }
public static string DescriptionAuthPasswordLong { get { return LC.L(@"The password used to connect to the server. This may also be supplied as the environment variable ""AUTH_PASSWORD""."); } }
public static string DescriptionAuthPasswordShort { get { return LC.L(@"Supply the password used to connect to the server"); } }
public static string DescriptionAuthUsernameLong { get { return LC.L(@"The username used to connect to the server. This may also be supplied as the environment variable ""AUTH_USERNAME""."); } }
public static string DescriptionAuthUsernameShort { get { return LC.L(@"Supply the username used to connect to the server"); } }
public static string DescriptionIntegratedAuthenticationLong { get { return LC.L(@"If the server and client both supports integrated authentication, this option enables that authentication method. This is likely only available with windows servers and clients."); } }
public static string DescriptionIntegratedAuthenticationShort { get { return LC.L(@"Use windows integrated authentication to connect to the server"); } }
public static string MethodNotAllowedError(System.Net.HttpStatusCode statuscode) { return LC.L(@"The server returned the error code {0} ({1}), indicating that the server does not support WebDAV connections", (int)statuscode, statuscode); }
public static string MissingFolderError(string foldername, string message) { return LC.L(@"The folder {0} was not found, message: {1}", foldername, message); }
public static string SeenThenNotFoundError(string foldername, string filename, string extension, string errormessage) { return LC.L(@"When listing the folder {0} the file {1} was listed, but the server now reports that the file is not found.
public static string MissingFolderError(string foldername, string message) { return LC.L(@"The folder {0} was not found, message: {1}", foldername, message); }
public static string SeenThenNotFoundError(string foldername, string filename, string extension, string errormessage) { return LC.L(@"When listing the folder {0} the file {1} was listed, but the server now reports that the file is not found.
This can be because the file is deleted or unavailable, but it can also be because the file extension {2} is blocked by the web server. IIS blocks unknown extensions by default.
Error message: {3}", foldername, filename, extension, errormessage); }
public static string DescriptionUseSSLLong { get { return LC.L(@"Use this option to communicate using Secure Socket Layer (SSL) over http (https)."); } }
public static string DescriptionUseSSLShort { get { return LC.L(@"Instruct Duplicati to use an SSL (https) connection"); } }
public static string DescriptionDebugPropfindLong { get { return LC.L(@"To aid in debugging issues, it is possible to set a path to a file that will be overwritten with the PROPFIND response."); } }
public static string DescriptionDebugPropfindShort { get { return LC.L(@"Dump the PROPFIND response"); } }
public static string DescriptionAcceptAnyCertificateLong { get { return LC.L(@"Use this option to accept any server certificate, regardless of what errors it may have. Please use --{0} instead, whenever possible.", "accept-specified-ssl-hash"); } }
public static string DescriptionAcceptAnyCertificateShort { get { return LC.L(@"Accept any server certificate"); } }
public static string DescriptionAcceptHashLong2 { get { return LC.L(@"If your server certificate is reported as invalid (e.g. with self-signed certificates), you can supply the certificate hash (SHA1) to approve it anyway. The hash value must be entered in hex format without spaces or colons. You can enter multiple hashes separated by commas."); } }
public static string DescriptionAcceptHashShort { get { return LC.L(@"Optionally accept a known SSL certificate"); } }
public static string UsernameAndPasswordRequired { get { return LC.L(@"Non-integrated authentication requires both a username and a password"); } }
}
}
Duplicati/Library/Backend/WEBDAV/WEBDAV.cs
+166 -137
View File
@@ -19,9 +19,12 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
using Duplicati.Library.Utility.Options;
using System;
using System.Collections.Generic;
using System.IO;
@@ -35,48 +38,61 @@ namespace Duplicati.Library.Backend
{
public class WEBDAV : IStreamingBackend
{
private record RequestResources : IDisposable
{
public RequestResources(HttpClient httpClient, HttpRequestMessage requestMessage)
{
HttpClient = httpClient;
RequestMessage = requestMessage;
}
public HttpRequestMessage RequestMessage { get; init; }
public HttpClient HttpClient { get; init; }
public void Dispose()
{
try
{
RequestMessage?.Dispose();
}
catch { }
try
{
HttpClient?.Dispose();
}
catch { }
}
}
private readonly NetworkCredential m_userInfo;
/// <summary>
/// The credentials to use for the connection
/// </summary>
private readonly NetworkCredential? m_userInfo;
/// <summary>
/// The URL to connect to
/// </summary>
private readonly string m_url;
/// <summary>
/// The path to use for all operations
/// </summary>
private readonly string m_path;
/// <summary>
/// The sanitized URL to use for listing
/// </summary>
private readonly string m_sanitizedUrl;
/// <summary>
/// The reverse protocol URL to use for listing
/// </summary>
private readonly string m_reverseProtocolUrl;
/// <summary>
/// The raw URL to use for listing
/// </summary>
private readonly string m_rawurl;
/// <summary>
/// The raw URL with port to use for listing
/// </summary>
private readonly string m_rawurlPort;
/// <summary>
/// The DNS name used for the connection
/// </summary>
private readonly string m_dnsName;
/// <summary>
/// Flag to indicate if integrated authentication should be used
/// </summary>
private readonly bool m_useIntegratedAuthentication = false;
/// <summary>
/// Flag to indicate if digest authentication should be used
/// </summary>
private readonly bool m_forceDigestAuthentication = false;
private readonly bool m_useSSL = false;
/// <summary>
/// The timeouts to use for the operations
/// </summary>
private readonly TimeoutOptionsHelper.Timeouts m_timeouts;
/// <summary>
/// The options for the SSL certificate validation
/// </summary>
private readonly SslOptionsHelper.SslCertificateOptions m_certificateOptions;
/// <summary>
/// A list of files seen in the last List operation.
/// It is used to detect a problem with IIS where a file is listed,
/// but IIS responds 404 because the file mapping is incorrect.
/// </summary>
private HashSet<string> m_filenamelist = null;
private HashSet<string>? m_filenamelist = null;
// According to the WEBDAV standard, the "allprop" request should return all properties, however this seems to fail on some servers (box.net).
// I've found this description: http://www.webdav.org/specs/rfc2518.html#METHOD_PROPFIND
@@ -86,34 +102,29 @@ namespace Duplicati.Library.Backend
private static readonly byte[] PROPFIND_BODY = new byte[0];
/// <summary>
/// Option to accept any SSL certificate
/// The HTTP client to use for all operations
/// </summary>
private readonly bool m_acceptAnyCertificate;
/// <summary>
/// Specific hashes to be accepted by the certificate validator
/// </summary>
private readonly string[] m_acceptSpecificCertificates;
/// <summary>
/// The default timeout in seconds for List operations
/// </summary>
private const int LIST_OPERATION_TIMEOUT_SECONDS = 600;
/// <summary>
/// The default timeout in seconds for Delete/CreateFolder operations
/// </summary>
private const int SHORT_OPERATION_TIMEOUT_SECONDS = 30;
private HttpClient? m_httpClient = null;
public WEBDAV()
{
m_url = string.Empty;
m_path = string.Empty;
m_sanitizedUrl = string.Empty;
m_reverseProtocolUrl = string.Empty;
m_rawurl = string.Empty;
m_rawurlPort = string.Empty;
m_dnsName = string.Empty;
m_certificateOptions = null!;
m_timeouts = null!;
}
public WEBDAV(string url, Dictionary<string, string> options)
public WEBDAV(string url, Dictionary<string, string?> options)
{
var u = new Utility.Uri(url);
u.RequireHost();
m_dnsName = u.Host;
var auth = AuthOptionsHelper.AuthOptions.Parse(options);
if (!string.IsNullOrEmpty(u.Username))
{
@@ -121,17 +132,17 @@ namespace Duplicati.Library.Backend
m_userInfo.UserName = u.Username;
if (!string.IsNullOrEmpty(u.Password))
m_userInfo.Password = u.Password;
else if (options.ContainsKey("auth-password"))
m_userInfo.Password = options["auth-password"];
else if (auth.HasPassword)
m_userInfo.Password = auth.Password;
}
else
{
if (options.ContainsKey("auth-username"))
if (auth.HasUsername)
{
m_userInfo = new NetworkCredential();
m_userInfo.UserName = options["auth-username"];
if (options.ContainsKey("auth-password"))
m_userInfo.Password = options["auth-password"];
m_userInfo.UserName = auth.Username;
if (auth.HasPassword)
m_userInfo.Password = auth.Password;
}
}
@@ -139,11 +150,14 @@ namespace Duplicati.Library.Backend
if (m_userInfo != null)
m_userInfo.Domain = "";
m_certificateOptions = SslOptionsHelper.SslCertificateOptions.Parse(options);
m_useIntegratedAuthentication = Utility.Utility.ParseBoolOption(options, "integrated-authentication");
m_forceDigestAuthentication = Utility.Utility.ParseBoolOption(options, "force-digest-authentication");
m_useSSL = Utility.Utility.ParseBoolOption(options, "use-ssl");
m_url = u.SetScheme(m_useSSL ? "https" : "http").SetCredentials(null, null).SetQuery(null).ToString();
if (!m_useIntegratedAuthentication && m_userInfo == null)
throw new UserInformationException(Strings.WEBDAV.UsernameAndPasswordRequired, "UsernameAndPasswordRequired");
m_url = u.SetScheme(m_certificateOptions.UseSSL ? "https" : "http").SetCredentials(null, null).SetQuery(null).ToString();
m_url = Util.AppendDirSeparator(m_url, "/");
m_path = u.Path;
@@ -152,17 +166,53 @@ namespace Duplicati.Library.Backend
m_path = Util.AppendDirSeparator(m_path, "/");
m_path = Utility.Uri.UrlDecode(m_path);
m_rawurl = new Utility.Uri(m_useSSL ? "https" : "http", u.Host, m_path).ToString();
m_rawurl = new Utility.Uri(m_certificateOptions.UseSSL ? "https" : "http", u.Host, m_path).ToString();
int port = u.Port;
if (port <= 0)
port = m_useSSL ? 443 : 80;
port = m_certificateOptions.UseSSL ? 443 : 80;
m_rawurlPort = new Utility.Uri(m_useSSL ? "https" : "http", u.Host, m_path, null, null, null, port).ToString();
m_sanitizedUrl = new Utility.Uri(m_useSSL ? "https" : "http", u.Host, m_path).ToString();
m_reverseProtocolUrl = new Utility.Uri(m_useSSL ? "http" : "https", u.Host, m_path).ToString();
m_acceptAnyCertificate = options.ContainsKey("accept-any-ssl-certificate") && Utility.Utility.ParseBoolOption(options, "accept-any-ssl-certificate");
m_acceptSpecificCertificates = options.ContainsKey("accept-specified-ssl-hash") ? options["accept-specified-ssl-hash"].Split([",", ";"], StringSplitOptions.RemoveEmptyEntries) : null;
m_rawurlPort = new Utility.Uri(m_certificateOptions.UseSSL ? "https" : "http", u.Host, m_path, null, null, null, port).ToString();
m_sanitizedUrl = new Utility.Uri(m_certificateOptions.UseSSL ? "https" : "http", u.Host, m_path).ToString();
m_reverseProtocolUrl = new Utility.Uri(m_certificateOptions.UseSSL ? "http" : "https", u.Host, m_path).ToString();
m_timeouts = TimeoutOptionsHelper.Timeouts.Parse(options);
}
/// <summary>
/// Gets the HTTP client to use for all operations
/// </summary>
/// <returns>The HTTP client to use</returns>
private HttpClient GetHttpClient()
{
if (m_httpClient == null)
{
var httpHandler = m_certificateOptions.CreateHandler();
if (m_useIntegratedAuthentication)
{
httpHandler.UseDefaultCredentials = true;
}
else if (m_forceDigestAuthentication)
{
httpHandler.Credentials = new CredentialCache
{
{ new System.Uri(m_url), "Digest", m_userInfo! }
};
}
m_httpClient = HttpClientHelper.CreateClient(httpHandler);
m_httpClient.Timeout = Timeout.InfiniteTimeSpan;
if (!m_useIntegratedAuthentication && !m_forceDigestAuthentication)
{
m_httpClient.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue(
"Basic",
Convert.ToBase64String(System.Text.Encoding.ASCII.GetBytes($"{m_userInfo!.UserName}:{m_userInfo.Password}"))
);
}
}
return m_httpClient;
}
#region IBackend Members
@@ -176,22 +226,27 @@ namespace Duplicati.Library.Backend
///<inheritdoc/>
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancelToken)
{
using var timeoutToken = new CancellationTokenSource();
timeoutToken.CancelAfter(TimeSpan.FromSeconds(LIST_OPERATION_TIMEOUT_SECONDS));
using var requestResources = CreateRequest(string.Empty, new HttpMethod("PROPFIND"));
requestResources.RequestMessage.Headers.Add("Depth", "1");
requestResources.RequestMessage.Content = new StreamContent(new MemoryStream(PROPFIND_BODY));
requestResources.RequestMessage.Content.Headers.ContentLength = PROPFIND_BODY.Length;
using var request = CreateRequest(string.Empty, new HttpMethod("PROPFIND"));
request.Headers.Add("Depth", "1");
request.Content = new StreamContent(new MemoryStream(PROPFIND_BODY));
request.Content.Headers.ContentLength = PROPFIND_BODY.Length;
var doc = new System.Xml.XmlDocument();
try
{
using var response = await requestResources.HttpClient.SendAsync(requestResources.RequestMessage, HttpCompletionOption.ResponseContentRead, timeoutToken.Token).ConfigureAwait(false);
using var response = await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, ct =>
GetHttpClient().SendAsync(request, HttpCompletionOption.ResponseContentRead, ct)
).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
using var stream = await response.Content.ReadAsStreamAsync(cancelToken).ConfigureAwait(false);
doc.Load(stream);
using var sourceStream = await Utility.Utility.WithTimeout(m_timeouts.ReadWriteTimeout, cancelToken, ct =>
response.Content.ReadAsStreamAsync(ct)
).ConfigureAwait(false);
using var timeoutStream = sourceStream.ObserveReadTimeout(m_timeouts.ReadWriteTimeout, false);
doc.Load(timeoutStream);
}
catch (HttpRequestException wex)
{
@@ -208,7 +263,14 @@ namespace Duplicati.Library.Backend
nm.AddNamespace("D", "DAV:");
var filenamelist = new HashSet<string>();
foreach (System.Xml.XmlNode n in doc.SelectNodes("D:multistatus/D:response/D:href", nm))
var lst = doc.SelectNodes("D:multistatus/D:response/D:href", nm);
if (lst == null)
{
m_filenamelist = filenamelist;
yield break;
}
foreach (System.Xml.XmlNode n in lst)
{
//IIS uses %20 for spaces and %2B for +
//Apache uses %20 for spaces and + for +
@@ -245,10 +307,10 @@ namespace Duplicati.Library.Backend
DateTime lastModified = new DateTime();
bool isCollection = false;
System.Xml.XmlNode stat = n.ParentNode.SelectSingleNode("D:propstat/D:prop", nm);
System.Xml.XmlNode? stat = n.ParentNode?.SelectSingleNode("D:propstat/D:prop", nm);
if (stat != null)
{
System.Xml.XmlNode s = stat.SelectSingleNode("D:getcontentlength", nm);
System.Xml.XmlNode? s = stat.SelectSingleNode("D:getcontentlength", nm);
if (s != null)
size = long.Parse(s.InnerText);
s = stat.SelectSingleNode("D:getlastmodified", nm);
@@ -298,16 +360,15 @@ namespace Duplicati.Library.Backend
{
try
{
using var timeoutToken = new CancellationTokenSource();
timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS));
using var combinedTokens = CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancelToken);
await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, async ct =>
{
using var request = CreateRequest(remotename);
request.Method = HttpMethod.Delete;
using var requestResources = CreateRequest(remotename);
requestResources.RequestMessage.Method = HttpMethod.Delete;
var response = await GetHttpClient().SendAsync(request, ct).ConfigureAwait(false);
var response = await requestResources.HttpClient.SendAsync(requestResources.RequestMessage, combinedTokens.Token).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
response.EnsureSuccessStatusCode();
}).ConfigureAwait(false);
}
catch (HttpRequestException wex)
@@ -323,15 +384,14 @@ namespace Duplicati.Library.Backend
{
get
{
return new List<ICommandLineArgument>(new ICommandLineArgument[] {
new CommandLineArgument("auth-password", CommandLineArgument.ArgumentType.Password, Strings.WEBDAV.DescriptionAuthPasswordShort, Strings.WEBDAV.DescriptionAuthPasswordLong),
new CommandLineArgument("auth-username", CommandLineArgument.ArgumentType.String, Strings.WEBDAV.DescriptionAuthUsernameShort, Strings.WEBDAV.DescriptionAuthUsernameLong),
return
[
.. AuthOptionsHelper.GetOptions(),
new CommandLineArgument("integrated-authentication", CommandLineArgument.ArgumentType.Boolean, Strings.WEBDAV.DescriptionIntegratedAuthenticationShort, Strings.WEBDAV.DescriptionIntegratedAuthenticationLong),
new CommandLineArgument("force-digest-authentication", CommandLineArgument.ArgumentType.Boolean, Strings.WEBDAV.DescriptionForceDigestShort, Strings.WEBDAV.DescriptionForceDigestLong),
new CommandLineArgument("use-ssl", CommandLineArgument.ArgumentType.Boolean, Strings.WEBDAV.DescriptionUseSSLShort, Strings.WEBDAV.DescriptionUseSSLLong),
new CommandLineArgument("accept-any-ssl-certificate", CommandLineArgument.ArgumentType.Boolean, Strings.WEBDAV.DescriptionAcceptAnyCertificateShort, Strings.WEBDAV.DescriptionAcceptAnyCertificateLong),
new CommandLineArgument("accept-specified-ssl-hash", CommandLineArgument.ArgumentType.String, Strings.WEBDAV.DescriptionAcceptHashShort, Strings.WEBDAV.DescriptionAcceptHashLong2)
});
.. SslOptionsHelper.GetOptions(),
.. TimeoutOptionsHelper.GetOptions()
];
}
}
@@ -348,15 +408,12 @@ namespace Duplicati.Library.Backend
///<inheritdoc/>
public async Task CreateFolderAsync(CancellationToken cancelToken)
{
using var timeoutToken = new CancellationTokenSource();
timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS));
using var combinedTokens = CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancelToken);
using var requestResources = CreateRequest(string.Empty, new HttpMethod("MKCOL"));
using var response = await requestResources.HttpClient.SendAsync(requestResources.RequestMessage, combinedTokens.Token).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, async ct =>
{
using var request = CreateRequest(string.Empty, new HttpMethod("MKCOL"));
using var response = await GetHttpClient().SendAsync(request, ct).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
}).ConfigureAwait(false);
}
#endregion
@@ -370,36 +427,8 @@ namespace Duplicati.Library.Backend
#endregion
private RequestResources CreateRequest(string remotename, HttpMethod method = null)
private HttpRequestMessage CreateRequest(string remotename, HttpMethod? method = null)
{
HttpClient httpClient;
HttpClientHandler httpHandler = new HttpClientHandler();
HttpClientHelper.ConfigureHandlerCertificateValidator(httpHandler, m_acceptAnyCertificate, m_acceptSpecificCertificates);
if (m_useIntegratedAuthentication)
{
httpHandler.UseDefaultCredentials = true;
httpClient = HttpClientHelper.CreateClient(httpHandler);
}
else if (m_forceDigestAuthentication)
{
httpHandler.Credentials = new CredentialCache
{
{ new System.Uri(m_url), "Digest", m_userInfo }
};
httpClient = HttpClientHelper.CreateClient(httpHandler);
}
else
{
httpClient = HttpClientHelper.CreateClient(httpHandler);
httpClient.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue(
"Basic",
Convert.ToBase64String(System.Text.Encoding.ASCII.GetBytes($"{m_userInfo.UserName}:{m_userInfo.Password}"))
);
}
httpClient.Timeout = Timeout.InfiniteTimeSpan;
var request = new HttpRequestMessage(HttpMethod.Get, $"{m_url}{Utility.Uri.UrlEncode(remotename).Replace("+", "%20")}");
request.Headers.Add(HttpRequestHeader.UserAgent.ToString(), "Duplicati WEBDAV Client v" + System.Reflection.Assembly.GetExecutingAssembly().GetName().Version);
@@ -408,7 +437,7 @@ namespace Duplicati.Library.Backend
if (method != null)
request.Method = method;
return new RequestResources(httpClient, request);
return request;
}
@@ -419,14 +448,14 @@ namespace Duplicati.Library.Backend
{
try
{
using var requestResources = CreateRequest(remotename, HttpMethod.Put);
using var request = CreateRequest(remotename, HttpMethod.Put);
using var timeoutStream = stream.ObserveReadTimeout(m_timeouts.ReadWriteTimeout, false);
request.Content = new StreamContent(timeoutStream);
request.Content.Headers.ContentLength = stream.Length;
request.Content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/octet-stream");
request.Version = HttpVersion.Version11;
requestResources.RequestMessage.Content = new StreamContent(stream);
requestResources.RequestMessage.Content.Headers.ContentLength = stream.Length;
requestResources.RequestMessage.Content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/octet-stream");
requestResources.RequestMessage.Version = HttpVersion.Version11;
using var response = await requestResources.HttpClient.SendAsync(requestResources.RequestMessage, HttpCompletionOption.ResponseHeadersRead, cancelToken).ConfigureAwait(false);
using var response = await GetHttpClient().SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancelToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
}
@@ -444,9 +473,9 @@ namespace Duplicati.Library.Backend
{
try
{
using var requestResources = CreateRequest(remotename, HttpMethod.Get);
await requestResources.HttpClient.DownloadFile(requestResources.RequestMessage, stream, null, cancelToken).ConfigureAwait(false);
using var request = CreateRequest(remotename, HttpMethod.Get);
using var timeoutStream = stream.ObserveWriteTimeout(m_timeouts.ReadWriteTimeout, false);
await GetHttpClient().DownloadFile(request, timeoutStream, null, cancelToken).ConfigureAwait(false);
}
catch (HttpRequestException wex)
{
Duplicati/Library/Main/Controller.cs
+2 -2
View File
@@ -815,7 +815,7 @@ namespace Duplicati.Library.Main
if (l != null)
foreach (ICommandLineArgument a in l)
{
if (supportedOptions.ContainsKey(a.Name) && !Options.KnownDuplicates.Contains(a.Name, StringComparer.OrdinalIgnoreCase))
if (supportedOptions.ContainsKey(a.Name) && !Options.KnownDuplicates.Contains(a.Name))
Logging.Log.WriteWarningMessage(LOGTAG, "DuplicateOption", null, Strings.Controller.DuplicateOptionNameWarning(a.Name));
supportedOptions[a.Name] = a;
@@ -823,7 +823,7 @@ namespace Duplicati.Library.Main
if (a.Aliases != null)
foreach (string s in a.Aliases)
{
if (supportedOptions.ContainsKey(s) && !Options.KnownDuplicates.Contains(s, StringComparer.OrdinalIgnoreCase))
if (supportedOptions.ContainsKey(s) && !Options.KnownDuplicates.Contains(s))
Logging.Log.WriteWarningMessage(LOGTAG, "DuplicateOption", null, Strings.Controller.DuplicateOptionNameWarning(s));
supportedOptions[s] = a;
Duplicati/Library/Main/Options.cs
+6 -1
View File
@@ -26,6 +26,7 @@ using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
using System.Globalization;
using System.Threading;
using Duplicati.Library.Utility.Options;
namespace Duplicati.Library.Main
{
@@ -281,7 +282,11 @@ namespace Duplicati.Library.Main
/// <summary>
/// Returns a list of options that are intentionally duplicate
/// </summary>
public static string[] KnownDuplicates => ["auth-password", "auth-username", "accept-any-ssl-certificate", "accept-specified-ssl-hash"];
public static readonly IReadOnlySet<string> KnownDuplicates =
AuthOptionsHelper.GetOptions().Select(x => x.Name)
.Concat(SslOptionsHelper.GetOptions().Select(x => x.Name))
.Concat(TimeoutOptionsHelper.GetOptions().Select(x => x.Name))
.ToHashSet(StringComparer.OrdinalIgnoreCase);
/// <summary>
/// A default backup name
Duplicati/Library/Utility/Duplicati.Library.Utility.csproj
+1 -1
View File
@@ -8,7 +8,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Duplicati.StreamUtil" Version="1.0.0" />
<PackageReference Include="Duplicati.StreamUtil" Version="1.0.2" />
<PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
<PackageReference Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
<PackageReference Include="System.Management" Version="8.0.0" />
Duplicati/Library/Utility/HttpClientHelper.cs
+1 -1
View File
@@ -76,7 +76,7 @@ public static class HttpClientHelper
/// <param name="handler">The HttpClientHandler to be configured</param>
/// <param name="acceptAllCertificates">If all certificates will be considered valid</param>
/// <param name="acceptSpecificCertificateHashes">List of hashes of certificates that will be accepted</param>
public static void ConfigureHandlerCertificateValidator(HttpClientHandler handler, bool acceptAllCertificates, string[] acceptSpecificCertificateHashes)
public static void ConfigureHandlerCertificateValidator(HttpClientHandler handler, bool acceptAllCertificates, string[]? acceptSpecificCertificateHashes)
{
var validator = new SslCertificateValidator(acceptAllCertificates, acceptSpecificCertificateHashes);
handler.ServerCertificateCustomValidationCallback = validator.ValidateServerCertificate;
Duplicati/Library/Utility/Options/AuthOptionsHelper.cs
+90
View File
@@ -0,0 +1,90 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
using System.Collections.Generic;
using Duplicati.Library.Interface;
namespace Duplicati.Library.Utility.Options;
/// <summary>
/// Helper class for unified use of authentication settings
/// </summary>
public static class AuthOptionsHelper
{
/// <summary>
/// Gets the authentication options
/// </summary>
/// <param name="prefix">An optional prefix for the options</param>
/// <returns>The authentication options</returns>
public static CommandLineArgument[] GetOptions(string? prefix = null) =>
[
new CommandLineArgument($"{prefix}auth-username", CommandLineArgument.ArgumentType.String, Strings.AuthSettingsHelper.DescriptionAuthUsernameShort, Strings.AuthSettingsHelper.DescriptionAuthUsernameLong),
new CommandLineArgument($"{prefix}auth-password", CommandLineArgument.ArgumentType.Password, Strings.AuthSettingsHelper.DescriptionAuthPasswordShort, Strings.AuthSettingsHelper.DescriptionAuthPasswordLong)
];
/// <summary>
/// Structure to hold the authentication options
/// </summary>
/// <param name="Username">The username</param>
/// <param name="Password">The password</param>
public sealed record AuthOptions(string? Username, string? Password)
{
/// <summary>
/// Checks if the username has a value
/// </summary>
public bool HasUsername => !string.IsNullOrWhiteSpace(Username);
/// <summary>
/// Checks if the password has a value
/// </summary>
public bool HasPassword => !string.IsNullOrWhiteSpace(Password);
/// <summary>
/// Checks if the username and password are set
/// </summary>
/// <returns>True if the username and password are set</returns>
public bool IsValid() => HasUsername && HasPassword;
/// <summary>
/// Throws an exception if the username and password are not set
/// </summary>
public void RequireCredentials()
{
if (!IsValid())
throw new UserInformationException(Strings.AuthSettingsHelper.UsernameAndPasswordRequired, "UsernameAndPasswordRequired");
}
/// <summary>
/// Parses the authentication options from a dictionary
/// </summary>
/// <param name="options">The dictionary to parse</param>
/// <param name="prefix">An optional prefix for the options</param>
/// <returns>The parsed authentication options</returns>
public static AuthOptions Parse(IReadOnlyDictionary<string, string?> options, string? prefix = null)
=> new AuthOptions(
options.GetValueOrDefault($"{prefix}auth-username"),
options.GetValueOrDefault($"{prefix}auth-password")
);
}
}
Duplicati/Library/Utility/Options/SslOptionsHelper.cs
+103
View File
@@ -0,0 +1,103 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
using System;
using System.Collections.Generic;
using System.Net.Http;
using Duplicati.Library.Interface;
namespace Duplicati.Library.Utility.Options;
/// <summary>
/// Helper class for unified use of SSL certificate options
/// </summary>
public static class SslOptionsHelper
{
/// <summary>
/// Gets the SSL certificate options for certificate only
/// </summary>
/// <param name="prefix">An optional prefix for the options</param>
/// <returns>The SSL certificate options</returns>
public static CommandLineArgument[] GetCertOnlyOptions(string? prefix = null) =>
[
new CommandLineArgument($"{prefix}accept-any-ssl-certificate", CommandLineArgument.ArgumentType.Boolean, Strings.SslOptionsHelper.DescriptionAcceptAnyCertificateShort, Strings.SslOptionsHelper.DescriptionAcceptAnyCertificateLong, "false"),
new CommandLineArgument($"{prefix}accept-specified-ssl-hash", CommandLineArgument.ArgumentType.String, Strings.SslOptionsHelper.DescriptionAcceptHashShort, Strings.SslOptionsHelper.DescriptionAcceptHashLong)
];
/// <summary>
/// Gets the SSL certificate options
/// </summary>
/// <param name="prefix">An optional prefix for the options</param>
/// <returns>The SSL certificate options</returns>
public static CommandLineArgument[] GetOptions(string? prefix = null) =>
[
new CommandLineArgument($"{prefix}use-ssl", CommandLineArgument.ArgumentType.Timespan, Strings.SslOptionsHelper.DescriptionUseSSLShort, Strings.SslOptionsHelper.DescriptionUseSSLLong),
.. GetCertOnlyOptions(prefix)
];
/// <summary>
/// Structure to hold the SSL certificate options
/// </summary>
/// <param name="UseSSL">Flag to indicate if SSL is used</param>
/// <param name="AcceptAllCertificates">Flag to accept all certificates</param>
/// <param name="AcceptSpecificCertificateHashes">Array of specific certificate hashes to accept</param>
public sealed record SslCertificateOptions(bool UseSSL, bool AcceptAllCertificates, string[] AcceptSpecificCertificateHashes)
{
/// <summary>
/// Parses the SSL certificate options from a dictionary
/// </summary>
/// <param name="options">The dictionary to parse</param>
/// <param name="prefix">An optional prefix for the options</param>
/// <returns>The parsed SSL certificate options</returns>
public static SslCertificateOptions Parse(IReadOnlyDictionary<string, string?> options, string? prefix = null)
{
var acceptSpecificCertificatesString = options.TryGetValue($"{prefix}accept-specified-ssl-hash", out var value) ? value : null;
return new SslCertificateOptions(
Utility.ParseBoolOption(options, $"{prefix}use-ssl"),
Utility.ParseBoolOption(options, $"{prefix}accept-any-ssl-certificate"),
string.IsNullOrWhiteSpace(acceptSpecificCertificatesString) ? [] : acceptSpecificCertificatesString.Split([",", ";"], StringSplitOptions.RemoveEmptyEntries)
);
}
/// <summary>
/// Creates a handler with the SSL certificate options
/// </summary>
/// <returns>The created handler</returns>
public HttpClientHandler CreateHandler()
{
var handler = new HttpClientHandler();
return ConfigureHandler(handler);
}
/// <summary>
/// Configures an existing handler with the SSL certificate options
/// </summary>
/// <param name="handler">The handler to configure</param>
/// <returns>The configured handler</returns>
public HttpClientHandler ConfigureHandler(HttpClientHandler handler)
{
HttpClientHelper.ConfigureHandlerCertificateValidator(handler, AcceptAllCertificates, AcceptSpecificCertificateHashes);
return handler;
}
}
}
Duplicati/Library/Utility/Options/TimeoutOptionsHelper.cs
+86
View File
@@ -0,0 +1,86 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
using System;
using System.Collections.Generic;
using Duplicati.Library.Interface;
namespace Duplicati.Library.Utility.Options;
/// <summary>
/// Helper class for unified use of timeout settings
/// </summary>
public static class TimeoutOptionsHelper
{
/// <summary>
/// The default timeout in seconds for List operations
/// </summary>
public const string DEFAULT_LIST_OPERATION_TIMEOUT = "10m";
/// <summary>
/// The default timeout in seconds for Delete/CreateFolder operations
/// </summary>
public const string DEFAULT_SHORT_OPERATION_TIMEOUT = "30s";
/// <summary>
/// The timeout for no activity during transfers
/// </summary>
public const string DEFAULT_READ_WRITE_TIMEOUT = "30s";
/// <summary>
/// Gets the timeout options
/// </summary>
/// <param name="prefix">An optional prefix for the options</param>
/// <returns>The timeout options</returns>
public static CommandLineArgument[] GetOptions(string? prefix = null) =>
[
new CommandLineArgument($"{prefix}short-timeout", CommandLineArgument.ArgumentType.Timespan, Strings.TimeoutSettingsHelper.DescriptionShortTimeoutShort, Strings.TimeoutSettingsHelper.DescriptionShortTimeoutLong, DEFAULT_SHORT_OPERATION_TIMEOUT),
new CommandLineArgument($"{prefix}list-timeout", CommandLineArgument.ArgumentType.Timespan, Strings.TimeoutSettingsHelper.DescriptionListTimeoutShort, Strings.TimeoutSettingsHelper.DescriptionListTimeoutLong, DEFAULT_LIST_OPERATION_TIMEOUT),
new CommandLineArgument($"{prefix}read-write-timeout", CommandLineArgument.ArgumentType.Timespan, Strings.TimeoutSettingsHelper.DescriptionReadWriteTimeoutShort, Strings.TimeoutSettingsHelper.DescriptionReadWriteTimeoutLong, DEFAULT_READ_WRITE_TIMEOUT)
];
/// <summary>
/// Structure for timeout configuration
/// </summary>
/// <param name="ShortTimeout">The timeout in seconds for short operations like delete and create folder</param>
/// <param name="ListTimeout">The timeout in seconds for listing files and folders</param>
/// <param name="ReadWriteTimeout">The timeout in seconds for read and write operations.</param>
public sealed record Timeouts(TimeSpan ShortTimeout, TimeSpan ListTimeout, TimeSpan ReadWriteTimeout)
{
/// <summary>
/// Parses the timeout options from an options dictionary
/// </summary>
/// <param name="options">The options dictionary</param>
/// <param name="prefix">An optional prefix for the options</param>
/// <returns>The timeout configuration</returns>
public static Timeouts Parse(IReadOnlyDictionary<string, string?> options, string? prefix = null)
=> new Timeouts(
Utility.ParseTimespanOption(options, $"{prefix}short-timeout", DEFAULT_SHORT_OPERATION_TIMEOUT),
Utility.ParseTimespanOption(options, $"{prefix}list-timeout", DEFAULT_LIST_OPERATION_TIMEOUT),
Utility.ParseTimespanOption(options, $"{prefix}read-write-timeout", DEFAULT_READ_WRITE_TIMEOUT)
);
}
}
Duplicati/Library/Utility/SslCertificateValidator.cs
+15 -12
View File
@@ -19,6 +19,8 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
using System;
using System.Linq;
using System.Net.Security;
@@ -26,7 +28,7 @@ using System.Security.Cryptography.X509Certificates;
namespace Duplicati.Library.Utility;
public class SslCertificateValidator(bool acceptAll, string[] validHashes)
public class SslCertificateValidator(bool acceptAll, string[]? validHashes)
{
[Serializable]
public class InvalidCertificateException(string certificate, SslPolicyErrors error)
@@ -39,26 +41,26 @@ public class SslCertificateValidator(bool acceptAll, string[] validHashes)
public SslPolicyErrors SslError => _mErrors;
}
public bool ValidateServerCertificate(object sender, X509Certificate cert, X509Chain chain, SslPolicyErrors sslPolicyErrors)
public bool ValidateServerCertificate(object sender, X509Certificate? cert, X509Chain? chain, SslPolicyErrors sslPolicyErrors)
{
if (acceptAll)
return true;
try
{
DateTime now = DateTime.Now;
using var certificate = cert as X509Certificate2 ?? new X509Certificate2(cert);
using var certificate = cert as X509Certificate2 ?? new X509Certificate2(cert ?? throw new ArgumentNullException(nameof(cert)));
if (!IsDateValid(certificate, now))
return false;
if (validHashes != null)
{
// Check main certificate hash
if (IsTrustedHash(Utility.ByteArrayAsHexString(certificate.GetCertHash())))
return true;
// Check chain certificate from root for the hash (this allows custom CA certificates hashes to be added)
if (chain?.ChainElements != null)
if (chain.ChainElements.Any(element => IsTrustedHash(Utility.ByteArrayAsHexString(element.Certificate.GetCertHash())) && IsDateValid(element.Certificate, now)))
@@ -73,11 +75,12 @@ public class SslCertificateValidator(bool acceptAll, string[] validHashes)
throw new Exception(Strings.SslCertificateValidator.VerifyCertificateHashError(ex, sslPolicyErrors), ex);
}
}
private bool IsTrustedHash(string hash) =>
!string.IsNullOrWhiteSpace(hash) &&
validHashes.Any(validHash => !string.IsNullOrEmpty(validHash) &&
!string.IsNullOrWhiteSpace(hash) &&
validHashes != null &&
validHashes.Any(validHash => !string.IsNullOrEmpty(validHash) &&
hash.Equals(validHash, StringComparison.OrdinalIgnoreCase));
private bool IsDateValid(X509Certificate2 cert, DateTime now) => now <= cert.NotAfter && now >= cert.NotBefore;
}
Duplicati/Library/Utility/Strings.cs
+48 -19
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Localization.Short;
using System;
@@ -66,4 +66,33 @@ namespace Duplicati.Library.Utility.Strings
{
public static string UnknownFilterGroup(string filterSet) { return LC.L(@"The string {0} does not represent a known filter group name. Valid values are: {1}", filterSet, string.Join(", ", Enum.GetNames(typeof(FilterGroup)))); }
}
internal static class TimeoutSettingsHelper
{
public static string DescriptionShortTimeoutLong { get { return LC.L(@"The timeout in seconds for short operations like delete and create folder"); } }
public static string DescriptionShortTimeoutShort { get { return LC.L(@"Short operation timeout"); } }
public static string DescriptionListTimeoutLong { get { return LC.L(@"The timeout in seconds for listing files and folders"); } }
public static string DescriptionListTimeoutShort { get { return LC.L(@"List operation timeout"); } }
public static string DescriptionReadWriteTimeoutLong { get { return LC.L(@"The timeout in seconds for read and write operations. If no activity is detected in this interval, a timeout error is raised"); } }
public static string DescriptionReadWriteTimeoutShort { get { return LC.L(@"Read/write operation timeout"); } }
}
internal static class SslOptionsHelper
{
public static string DescriptionAcceptAnyCertificateLong { get { return LC.L(@"Use this option to accept any server certificate, regardless of what errors it may have. Please use --{0} instead, whenever possible.", "accept-specified-ssl-hash"); } }
public static string DescriptionAcceptAnyCertificateShort { get { return LC.L(@"Accept any server certificate"); } }
public static string DescriptionAcceptHashLong { get { return LC.L(@"If your server certificate is reported as invalid (e.g. with self-signed certificates), you can supply the certificate hash (SHA1) to approve it anyway. The hash value must be entered in hex format without spaces or colons. You can enter multiple hashes separated by commas."); } }
public static string DescriptionAcceptHashShort { get { return LC.L(@"Optionally accept a known SSL certificate"); } }
public static string DescriptionUseSSLLong { get { return LC.L(@"Use this option to communicate using Secure Socket Layer (SSL) over http (https)."); } }
public static string DescriptionUseSSLShort { get { return LC.L(@"Instruct Duplicati to use an SSL (https) connection"); } }
}
internal static class AuthSettingsHelper
{
public static string DescriptionAuthPasswordLong { get { return LC.L(@"The password used to connect to the server. This may also be supplied as the environment variable ""AUTH_PASSWORD""."); } }
public static string DescriptionAuthPasswordShort { get { return LC.L(@"Supply the password used to connect to the server"); } }
public static string DescriptionAuthUsernameLong { get { return LC.L(@"The username used to connect to the server. This may also be supplied as the environment variable ""AUTH_USERNAME""."); } }
public static string DescriptionAuthUsernameShort { get { return LC.L(@"Supply the username used to connect to the server"); } }
public static string UsernameAndPasswordRequired { get { return LC.L(@"Authentication requires both a username and a password"); } }
}
}
Duplicati/Library/Utility/Utility.cs
+70
View File
@@ -37,6 +37,7 @@ using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Duplicati.Library.Common.IO;
using Duplicati.StreamUtil;
namespace Duplicati.Library.Utility
{
@@ -1667,5 +1668,74 @@ namespace Duplicati.Library.Utility
/// <returns>The formatted string</returns>
public static string FormatInvariant(this FormattableString formattable)
=> formattable.ToString(CultureInfo.InvariantCulture);
/// <summary>
/// Performs the function with an additional timeout
/// </summary>
/// <param name="timeout">The timeout to observe</param>
/// <param name="token">The cancellation token</param>
/// <param name="func">The function to invoke</param>
/// <returns>The task</returns>
public static async Task WithTimeout(TimeSpan timeout, CancellationToken token, Func<CancellationToken, Task> func)
{
using var cts = CancellationTokenSource.CreateLinkedTokenSource(token);
cts.CancelAfter(timeout);
await func(cts.Token);
}
/// <summary>
/// Performs the function with an additional timeout
/// </summary>
/// <typeparam name="T">The return type</typeparam>
/// <param name="timeout">The timeout to observe</param>
/// <param name="token">The cancellation token</param>
/// <param name="func">The function to invoke</param>
/// <returns>The task</returns>
public static async Task<T> WithTimeout<T>(TimeSpan timeout, CancellationToken token, Func<CancellationToken, Task<T>> func)
{
using var cts = CancellationTokenSource.CreateLinkedTokenSource(token);
cts.CancelAfter(timeout);
return await func(cts.Token);
}
/// <summary>
/// Wraps the stream in a timeout observing stream
/// </summary>
/// <param name="stream">The stream to wrap</param>
/// <param name="timeout">The timeout to observe</param>
/// <param name="disposeBaseStream">A flag indicating if the base stream should be disposed</param>
/// <returns>The wrapped stream</returns>
public static Stream ObserveReadTimeout(this Stream stream, TimeSpan timeout, bool disposeBaseStream = true)
{
if (stream == null)
throw new ArgumentNullException(nameof(stream));
return new TimeoutObservingStream(stream, disposeBaseStream)
{
ReadTimeout = timeout.Ticks > 0 && timeout != Timeout.InfiniteTimeSpan
? (int)timeout.TotalMilliseconds
: Timeout.Infinite
};
}
/// <summary>
/// Wraps the stream in a timeout observing stream
/// </summary>
/// <param name="stream">The stream to wrap</param>
/// <param name="timeout">The timeout to observe</param>
/// <param name="disposeBaseStream">A flag indicating if the base stream should be disposed</param>
/// <returns>The wrapped stream</returns>
public static Stream ObserveWriteTimeout(this Stream stream, TimeSpan timeout, bool disposeBaseStream = true)
{
if (stream == null)
throw new ArgumentNullException(nameof(stream));
return new TimeoutObservingStream(stream, disposeBaseStream)
{
WriteTimeout = timeout.Ticks > 0 && timeout != Timeout.InfiniteTimeSpan
? (int)timeout.TotalMilliseconds
: Timeout.Infinite
};
}
}
}