From 6cad935656fb1ab97b6b7f4ce0a54c8f39a97d37 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 19 Mar 2025 22:40:09 +0100 Subject: [PATCH 01/48] Common parsing for auth options --- Duplicati/Library/Backend/FTP/FTPBackend.cs | 29 +++++---------- Duplicati/Library/Backend/WEBDAV/WEBDAV.cs | 27 +++----------- .../Utility/Options/AuthOptionsHelper.cs | 35 +++++++++++++++++++ 3 files changed, 48 insertions(+), 43 deletions(-) diff --git a/Duplicati/Library/Backend/FTP/FTPBackend.cs b/Duplicati/Library/Backend/FTP/FTPBackend.cs index 9bcdc477c..cc9977df9 100644 --- a/Duplicati/Library/Backend/FTP/FTPBackend.cs +++ b/Duplicati/Library/Backend/FTP/FTPBackend.cs @@ -273,31 +273,18 @@ namespace Duplicati.Library.Backend var u = new Utility.Uri(url); u.RequireHost(); - if (!string.IsNullOrEmpty(u.Username)) + var auth = AuthOptionsHelper.AuthOptions.Parse(options); + if (auth.HasUsername) { - _userInfo = new NetworkCredential + _userInfo = new NetworkCredential() { - UserName = u.Username + UserName = auth.Username, + Domain = "" }; - if (!string.IsNullOrEmpty(u.Password)) - _userInfo.Password = u.Password; - else if (options.ContainsKey("auth-password")) - _userInfo.Password = options["auth-password"]; - } - else - { - if (options.ContainsKey("auth-username")) - { - _userInfo = new NetworkCredential(); - _userInfo.UserName = options["auth-username"]; - if (options.ContainsKey("auth-password")) - _userInfo.Password = options["auth-password"]; - } - } - //Bugfix, see http://connect.microsoft.com/VisualStudio/feedback/details/695227/networkcredential-default-constructor-leaves-domain-null-leading-to-null-object-reference-exceptions-in-framework-code - if (_userInfo != null) - _userInfo.Domain = ""; + if (auth.HasPassword) + _userInfo.Password = auth.Password; + } var parsedurl = u.SetScheme("ftp").SetQuery(null).SetCredentials(null, null).ToString(); parsedurl = Util.AppendDirSeparator(parsedurl, "/"); diff --git a/Duplicati/Library/Backend/WEBDAV/WEBDAV.cs b/Duplicati/Library/Backend/WEBDAV/WEBDAV.cs index 98c7464a0..184bfeb8a 100644 --- a/Duplicati/Library/Backend/WEBDAV/WEBDAV.cs +++ b/Duplicati/Library/Backend/WEBDAV/WEBDAV.cs @@ -124,31 +124,14 @@ namespace Duplicati.Library.Backend var u = new Utility.Uri(url); u.RequireHost(); m_dnsName = u.Host; - var auth = AuthOptionsHelper.AuthOptions.Parse(options); - - if (!string.IsNullOrEmpty(u.Username)) + var auth = AuthOptionsHelper.AuthOptions.Parse(options, u); + if (auth.HasUsername) { - m_userInfo = new NetworkCredential(); - m_userInfo.UserName = u.Username; - if (!string.IsNullOrEmpty(u.Password)) - m_userInfo.Password = u.Password; - else if (auth.HasPassword) + m_userInfo = new NetworkCredential() { Domain = "" }; + m_userInfo.UserName = auth.Username; + if (auth.HasPassword) m_userInfo.Password = auth.Password; } - else - { - if (auth.HasUsername) - { - m_userInfo = new NetworkCredential(); - m_userInfo.UserName = auth.Username; - if (auth.HasPassword) - m_userInfo.Password = auth.Password; - } - } - - //Bugfix, see http://connect.microsoft.com/VisualStudio/feedback/details/695227/networkcredential-default-constructor-leaves-domain-null-leading-to-null-object-reference-exceptions-in-framework-code - if (m_userInfo != null) - m_userInfo.Domain = ""; m_certificateOptions = SslOptionsHelper.SslCertificateOptions.Parse(options); m_useIntegratedAuthentication = Utility.Utility.ParseBoolOption(options, "integrated-authentication"); diff --git a/Duplicati/Library/Utility/Options/AuthOptionsHelper.cs b/Duplicati/Library/Utility/Options/AuthOptionsHelper.cs index ef5ecfafd..b28b67a8a 100644 --- a/Duplicati/Library/Utility/Options/AuthOptionsHelper.cs +++ b/Duplicati/Library/Utility/Options/AuthOptionsHelper.cs @@ -85,6 +85,41 @@ public static class AuthOptionsHelper options.GetValueOrDefault($"{prefix}auth-username"), options.GetValueOrDefault($"{prefix}auth-password") ); + + /// + /// Parses the authentication options from a dictionary + /// + /// The dictionary to parse + /// The URI to get the default values from + /// An optional prefix for the options + /// The parsed authentication options + public static AuthOptions Parse(IReadOnlyDictionary options, Uri uri, string? prefix = null) + { + var optionUsername = options.GetValueOrDefault($"{prefix}auth-username"); + var optionPassword = options.GetValueOrDefault($"{prefix}auth-password"); + + string? username = null; + string? password = null; + if (!string.IsNullOrEmpty(uri.Username)) + { + username = uri.Username; + if (!string.IsNullOrEmpty(uri.Password)) + password = uri.Password; + else if (!string.IsNullOrWhiteSpace(optionPassword)) + password = optionPassword; + } + else + { + if (!string.IsNullOrWhiteSpace(optionUsername)) + { + username = optionUsername; + if (!string.IsNullOrWhiteSpace(optionPassword)) + password = optionPassword; + } + } + + return new AuthOptions(username, password); + } } } From 7405bab68f9b9457f4ce601f39ae6c01a360c3f5 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 19 Mar 2025 23:16:37 +0100 Subject: [PATCH 02/48] Simplified working with the options helpers --- Duplicati/Library/Backend/FTP/FTPBackend.cs | 6 +- Duplicati/Library/Backend/WEBDAV/WEBDAV.cs | 6 +- .../Utility/Options/AuthOptionsHelper.cs | 150 ++++++++++++------ .../Utility/Options/SslOptionsHelper.cs | 32 ++-- .../Utility/Options/TimeoutOptionsHelper.cs | 31 ++-- 5 files changed, 139 insertions(+), 86 deletions(-) diff --git a/Duplicati/Library/Backend/FTP/FTPBackend.cs b/Duplicati/Library/Backend/FTP/FTPBackend.cs index cc9977df9..23a777161 100644 --- a/Duplicati/Library/Backend/FTP/FTPBackend.cs +++ b/Duplicati/Library/Backend/FTP/FTPBackend.cs @@ -268,12 +268,12 @@ namespace Duplicati.Library.Backend [SuppressMessage("ReSharper", "VirtualMemberCallInConstructor")] // The behavior of accessing the virtual properties is as expected public FTP(string url, Dictionary options) { - _sslOptions = SslOptionsHelper.SslCertificateOptions.Parse(options); + _sslOptions = SslOptionsHelper.Parse(options); var u = new Utility.Uri(url); u.RequireHost(); - var auth = AuthOptionsHelper.AuthOptions.Parse(options); + var auth = AuthOptionsHelper.Parse(options); if (auth.HasUsername) { _userInfo = new NetworkCredential() @@ -328,7 +328,7 @@ namespace Duplicati.Library.Backend _logToConsole = CoreUtility.ParseBoolOption(options, CONFIG_KEY_FTP_LOGTOCONSOLE); _logPrivateInfoToConsole = CoreUtility.ParseBoolOption(options, CONFIG_KEY_FTP_LOGPRIVATEINFOTOCONSOLE); _diagnosticsLog = CoreUtility.ParseBoolOption(options, CONFIG_KEY_FTP_LOGDIAGNOSTICS); - _timeouts = TimeoutOptionsHelper.Timeouts.Parse(options); + _timeouts = TimeoutOptionsHelper.Parse(options); _ftpConfig = new FtpConfig { diff --git a/Duplicati/Library/Backend/WEBDAV/WEBDAV.cs b/Duplicati/Library/Backend/WEBDAV/WEBDAV.cs index 184bfeb8a..e0006bca8 100644 --- a/Duplicati/Library/Backend/WEBDAV/WEBDAV.cs +++ b/Duplicati/Library/Backend/WEBDAV/WEBDAV.cs @@ -124,7 +124,7 @@ namespace Duplicati.Library.Backend var u = new Utility.Uri(url); u.RequireHost(); m_dnsName = u.Host; - var auth = AuthOptionsHelper.AuthOptions.Parse(options, u); + var auth = AuthOptionsHelper.Parse(options, u); if (auth.HasUsername) { m_userInfo = new NetworkCredential() { Domain = "" }; @@ -133,7 +133,7 @@ namespace Duplicati.Library.Backend m_userInfo.Password = auth.Password; } - m_certificateOptions = SslOptionsHelper.SslCertificateOptions.Parse(options); + m_certificateOptions = SslOptionsHelper.Parse(options); m_useIntegratedAuthentication = Utility.Utility.ParseBoolOption(options, "integrated-authentication"); m_forceDigestAuthentication = Utility.Utility.ParseBoolOption(options, "force-digest-authentication"); @@ -158,7 +158,7 @@ namespace Duplicati.Library.Backend m_rawurlPort = new Utility.Uri(m_certificateOptions.UseSSL ? "https" : "http", u.Host, m_path, null, null, null, port).ToString(); m_sanitizedUrl = new Utility.Uri(m_certificateOptions.UseSSL ? "https" : "http", u.Host, m_path).ToString(); m_reverseProtocolUrl = new Utility.Uri(m_certificateOptions.UseSSL ? "http" : "https", u.Host, m_path).ToString(); - m_timeouts = TimeoutOptionsHelper.Timeouts.Parse(options); + m_timeouts = TimeoutOptionsHelper.Parse(options); } /// diff --git a/Duplicati/Library/Utility/Options/AuthOptionsHelper.cs b/Duplicati/Library/Utility/Options/AuthOptionsHelper.cs index b28b67a8a..94b0f1f34 100644 --- a/Duplicati/Library/Utility/Options/AuthOptionsHelper.cs +++ b/Duplicati/Library/Utility/Options/AuthOptionsHelper.cs @@ -43,6 +43,109 @@ public static class AuthOptionsHelper new CommandLineArgument($"{prefix}auth-password", CommandLineArgument.ArgumentType.Password, Strings.AuthSettingsHelper.DescriptionAuthPasswordShort, Strings.AuthSettingsHelper.DescriptionAuthPasswordLong) ]; + /// + /// Gets the authentication options + /// + /// The name of the username options + /// The name of the password options + /// The authentication options + public static CommandLineArgument[] GetOptionsWithAlias(string username, string password) => + [ + new CommandLineArgument(username, CommandLineArgument.ArgumentType.String, Strings.AuthSettingsHelper.DescriptionAuthUsernameShort, Strings.AuthSettingsHelper.DescriptionAuthUsernameLong, null, ["auth-username"]), + new CommandLineArgument(password, CommandLineArgument.ArgumentType.Password, Strings.AuthSettingsHelper.DescriptionAuthPasswordShort, Strings.AuthSettingsHelper.DescriptionAuthPasswordLong, null, ["auth-password"]) + ]; + + /// + /// Parses the authentication options from a dictionary + /// + /// The dictionary to parse + /// An optional prefix for the options + /// The parsed authentication options + public static AuthOptions Parse(IReadOnlyDictionary options, string? prefix = null) + => new AuthOptions( + options.GetValueOrDefault($"{prefix}auth-username"), + options.GetValueOrDefault($"{prefix}auth-password") + ); + + /// + /// Parses the authentication options from a dictionary + /// + /// The dictionary to parse + /// The name of the username options + /// The name of the password options + /// The parsed authentication options + public static AuthOptions ParseWithAlias(IReadOnlyDictionary options, string username, string password) + { + var optionUsername = options.GetValueOrDefault(username); + var optionPassword = options.GetValueOrDefault(password); + + if (string.IsNullOrWhiteSpace(optionUsername)) + optionUsername = options.GetValueOrDefault("auth-username"); + if (string.IsNullOrWhiteSpace(optionPassword)) + optionPassword = options.GetValueOrDefault("auth-password"); + + return new AuthOptions(optionUsername, optionPassword); + } + + /// + /// Parses the authentication options from a dictionary + /// + /// The dictionary to parse + /// The URI to get the default values from + /// The name of the username options + /// The name of the password options + /// The parsed authentication options + public static AuthOptions ParseWithAlias(IReadOnlyDictionary options, Uri uri, string username, string password) + { + // Prefer the primary name, if set + var optionUsername = options.GetValueOrDefault(username); + var optionPassword = options.GetValueOrDefault(password); + var parsedOptions = Parse(options, uri); + + if (string.IsNullOrWhiteSpace(optionUsername)) + optionUsername = parsedOptions.Username; + if (string.IsNullOrWhiteSpace(optionPassword)) + optionPassword = parsedOptions.Password; + + return new AuthOptions(optionUsername, optionPassword); + } + + /// + /// Parses the authentication options from a dictionary + /// + /// The dictionary to parse + /// The URI to get the default values from + /// An optional prefix for the options + /// The parsed authentication options + public static AuthOptions Parse(IReadOnlyDictionary options, Uri uri, string? prefix = null) + { + var optionUsername = options.GetValueOrDefault($"{prefix}auth-username"); + var optionPassword = options.GetValueOrDefault($"{prefix}auth-password"); + + // Prefer the URL values, if set + string? username = null; + string? password = null; + if (!string.IsNullOrEmpty(uri.Username)) + { + username = uri.Username; + if (!string.IsNullOrEmpty(uri.Password)) + password = uri.Password; + else if (!string.IsNullOrWhiteSpace(optionPassword)) + password = optionPassword; + } + else + { + if (!string.IsNullOrWhiteSpace(optionUsername)) + { + username = optionUsername; + if (!string.IsNullOrWhiteSpace(optionPassword)) + password = optionPassword; + } + } + + return new AuthOptions(username, password); + } + /// /// Structure to hold the authentication options /// @@ -73,53 +176,6 @@ public static class AuthOptionsHelper if (!IsValid()) throw new UserInformationException(Strings.AuthSettingsHelper.UsernameAndPasswordRequired, "UsernameAndPasswordRequired"); } - - /// - /// Parses the authentication options from a dictionary - /// - /// The dictionary to parse - /// An optional prefix for the options - /// The parsed authentication options - public static AuthOptions Parse(IReadOnlyDictionary options, string? prefix = null) - => new AuthOptions( - options.GetValueOrDefault($"{prefix}auth-username"), - options.GetValueOrDefault($"{prefix}auth-password") - ); - - /// - /// Parses the authentication options from a dictionary - /// - /// The dictionary to parse - /// The URI to get the default values from - /// An optional prefix for the options - /// The parsed authentication options - public static AuthOptions Parse(IReadOnlyDictionary options, Uri uri, string? prefix = null) - { - var optionUsername = options.GetValueOrDefault($"{prefix}auth-username"); - var optionPassword = options.GetValueOrDefault($"{prefix}auth-password"); - - string? username = null; - string? password = null; - if (!string.IsNullOrEmpty(uri.Username)) - { - username = uri.Username; - if (!string.IsNullOrEmpty(uri.Password)) - password = uri.Password; - else if (!string.IsNullOrWhiteSpace(optionPassword)) - password = optionPassword; - } - else - { - if (!string.IsNullOrWhiteSpace(optionUsername)) - { - username = optionUsername; - if (!string.IsNullOrWhiteSpace(optionPassword)) - password = optionPassword; - } - } - - return new AuthOptions(username, password); - } } } diff --git a/Duplicati/Library/Utility/Options/SslOptionsHelper.cs b/Duplicati/Library/Utility/Options/SslOptionsHelper.cs index 508b65996..3932121bd 100644 --- a/Duplicati/Library/Utility/Options/SslOptionsHelper.cs +++ b/Duplicati/Library/Utility/Options/SslOptionsHelper.cs @@ -55,6 +55,22 @@ public static class SslOptionsHelper .. GetCertOnlyOptions(prefix) ]; + /// + /// Parses the SSL certificate options from a dictionary + /// + /// The dictionary to parse + /// An optional prefix for the options + /// The parsed SSL certificate options + public static SslCertificateOptions Parse(IReadOnlyDictionary options, string? prefix = null) + { + var acceptSpecificCertificatesString = options.TryGetValue($"{prefix}accept-specified-ssl-hash", out var value) ? value : null; + return new SslCertificateOptions( + Utility.ParseBoolOption(options, $"{prefix}use-ssl"), + Utility.ParseBoolOption(options, $"{prefix}accept-any-ssl-certificate"), + string.IsNullOrWhiteSpace(acceptSpecificCertificatesString) ? [] : acceptSpecificCertificatesString.Split([",", ";"], StringSplitOptions.RemoveEmptyEntries) + ); + } + /// /// Structure to hold the SSL certificate options /// @@ -63,22 +79,6 @@ public static class SslOptionsHelper /// Array of specific certificate hashes to accept public sealed record SslCertificateOptions(bool UseSSL, bool AcceptAllCertificates, string[] AcceptSpecificCertificateHashes) { - /// - /// Parses the SSL certificate options from a dictionary - /// - /// The dictionary to parse - /// An optional prefix for the options - /// The parsed SSL certificate options - public static SslCertificateOptions Parse(IReadOnlyDictionary options, string? prefix = null) - { - var acceptSpecificCertificatesString = options.TryGetValue($"{prefix}accept-specified-ssl-hash", out var value) ? value : null; - return new SslCertificateOptions( - Utility.ParseBoolOption(options, $"{prefix}use-ssl"), - Utility.ParseBoolOption(options, $"{prefix}accept-any-ssl-certificate"), - string.IsNullOrWhiteSpace(acceptSpecificCertificatesString) ? [] : acceptSpecificCertificatesString.Split([",", ";"], StringSplitOptions.RemoveEmptyEntries) - ); - } - /// /// Creates a handler with the SSL certificate options /// diff --git a/Duplicati/Library/Utility/Options/TimeoutOptionsHelper.cs b/Duplicati/Library/Utility/Options/TimeoutOptionsHelper.cs index cf3d68cd1..9bcb4b05d 100644 --- a/Duplicati/Library/Utility/Options/TimeoutOptionsHelper.cs +++ b/Duplicati/Library/Utility/Options/TimeoutOptionsHelper.cs @@ -60,27 +60,24 @@ public static class TimeoutOptionsHelper new CommandLineArgument($"{prefix}read-write-timeout", CommandLineArgument.ArgumentType.Timespan, Strings.TimeoutSettingsHelper.DescriptionReadWriteTimeoutShort, Strings.TimeoutSettingsHelper.DescriptionReadWriteTimeoutLong, DEFAULT_READ_WRITE_TIMEOUT) ]; + /// + /// Parses the timeout options from an options dictionary + /// + /// The options dictionary + /// An optional prefix for the options + /// The timeout configuration + public static Timeouts Parse(IReadOnlyDictionary options, string? prefix = null) + => new Timeouts( + Utility.ParseTimespanOption(options, $"{prefix}short-timeout", DEFAULT_SHORT_OPERATION_TIMEOUT), + Utility.ParseTimespanOption(options, $"{prefix}list-timeout", DEFAULT_LIST_OPERATION_TIMEOUT), + Utility.ParseTimespanOption(options, $"{prefix}read-write-timeout", DEFAULT_READ_WRITE_TIMEOUT) + ); + /// /// Structure for timeout configuration /// /// The timeout in seconds for short operations like delete and create folder /// The timeout in seconds for listing files and folders /// The timeout in seconds for read and write operations. - public sealed record Timeouts(TimeSpan ShortTimeout, TimeSpan ListTimeout, TimeSpan ReadWriteTimeout) - { - /// - /// Parses the timeout options from an options dictionary - /// - /// The options dictionary - /// An optional prefix for the options - /// The timeout configuration - public static Timeouts Parse(IReadOnlyDictionary options, string? prefix = null) - => new Timeouts( - Utility.ParseTimespanOption(options, $"{prefix}short-timeout", DEFAULT_SHORT_OPERATION_TIMEOUT), - Utility.ParseTimespanOption(options, $"{prefix}list-timeout", DEFAULT_LIST_OPERATION_TIMEOUT), - Utility.ParseTimespanOption(options, $"{prefix}read-write-timeout", DEFAULT_READ_WRITE_TIMEOUT) - ); - } - - + public sealed record Timeouts(TimeSpan ShortTimeout, TimeSpan ListTimeout, TimeSpan ReadWriteTimeout); } \ No newline at end of file From 133237e56c692f22b7e3f6abc0347242e4775dde Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 19 Mar 2025 23:18:48 +0100 Subject: [PATCH 03/48] Patched up AliyunOSS and AzureBlob with timeouts and new options helpers --- .../Backend/AliyunOSS/AliyunOSSBackend.cs | 83 +++++++++---------- .../Backend/AzureBlob/AzureBlobBackend.cs | 62 +++++--------- .../Backend/AzureBlob/AzureBlobWrapper.cs | 39 ++++++--- .../Library/Backend/AzureBlob/Strings.cs | 7 +- 4 files changed, 87 insertions(+), 104 deletions(-) diff --git a/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs b/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs index 53dcb37b7..fdb37170f 100644 --- a/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs +++ b/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs @@ -22,6 +22,8 @@ using Aliyun.OSS; using Aliyun.OSS.Common; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; +using Duplicati.Library.Utility; +using Duplicati.Library.Utility.Options; using System; using System.Collections.Generic; using System.IO; @@ -46,7 +48,8 @@ namespace Duplicati.Library.Backend.AliyunOSS private const string OSS_ACCESS_KEY_ID = "oss-access-key-id"; private const string OSS_ACCESS_KEY_SECRET = "oss-access-key-secret"; - private AliyunOSSOptions _ossOptions; + private readonly AliyunOSSOptions _ossOptions; + private readonly TimeoutOptionsHelper.Timeouts _timeouts; public OSS() { } @@ -54,34 +57,25 @@ namespace Duplicati.Library.Backend.AliyunOSS public OSS(string url, Dictionary options) { _ossOptions = new AliyunOSSOptions(); + _timeouts = TimeoutOptionsHelper.Parse(options); var uri = new Utility.Uri(url?.Trim()); var prefix = uri.HostAndPath?.TrimPath(); if (!string.IsNullOrEmpty(prefix)) - { _ossOptions.Path = prefix; - } - if (options.ContainsKey(OSS_ACCESS_KEY_ID)) - { - _ossOptions.AccessKeyId = options[OSS_ACCESS_KEY_ID]; - } + var auth = AuthOptionsHelper.ParseWithAlias(options, uri, OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY_SECRET); + auth.RequireCredentials(); - if (options.ContainsKey(OSS_ACCESS_KEY_SECRET)) - { - _ossOptions.AccessKeySecret = options[OSS_ACCESS_KEY_SECRET]; - } + _ossOptions.AccessKeyId = auth.Username; + _ossOptions.AccessKeySecret = auth.Password; if (options.ContainsKey(OSS_BUCKET_NAME)) - { _ossOptions.BucketName = options[OSS_BUCKET_NAME]; - } if (options.ContainsKey(OSS_ENDPOINT)) - { _ossOptions.Endpoint = options[OSS_ENDPOINT]; - } } private OssClient GetClient(bool isUseNewServiceClient = true) @@ -122,11 +116,11 @@ namespace Duplicati.Library.Backend.AliyunOSS ObjectListing result; try { - result = await Task.Run(() => client.ListObjects(listObjectsRequest), cancelToken).ConfigureAwait(ConfigureAwaitOptions.ForceYielding); + result = await Utility.Utility.WithTimeout(_timeouts.ListTimeout, cancelToken, async ct => await Task.Run(() => client.ListObjects(listObjectsRequest), ct).ConfigureAwait(ConfigureAwaitOptions.ForceYielding)) + .ConfigureAwait(false); + if (result.HttpStatusCode != HttpStatusCode.OK) - { throw new Exception(result.HttpStatusCode.ToString()); - } } catch (OssException ex) @@ -164,29 +158,27 @@ namespace Duplicati.Library.Backend.AliyunOSS await GetAsync(remotename, fs, cancelToken).ConfigureAwait(false); } - public Task DeleteAsync(string remotename, CancellationToken cancelToken) + public async Task DeleteAsync(string remotename, CancellationToken cancelToken) { try { var client = GetClient(); - var bucketName = _ossOptions.BucketName; - var objectName = $"{_ossOptions.Path.TrimPath()}/{remotename}".TrimPath(); - var res = client.DeleteObject(bucketName, objectName); - if (res?.HttpStatusCode != HttpStatusCode.OK) + await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancelToken, ct => { - Logging.Log.WriteInformationMessage(LOGTAG, "Delete", "Delete object failed. it may have been deleted"); - } + var res = client.DeleteObject(bucketName, objectName); + if (res?.HttpStatusCode != HttpStatusCode.OK) + Logging.Log.WriteInformationMessage(LOGTAG, "Delete", "Delete object failed. it may have been deleted"); + return Task.CompletedTask; + }); } catch (Exception ex) { Logging.Log.WriteErrorMessage(LOGTAG, "Delete", ex, "Delete object failed. {0}", ex.Message); throw; } - - return Task.CompletedTask; } public Task TestAsync(CancellationToken cancelToken) @@ -212,8 +204,9 @@ namespace Duplicati.Library.Backend.AliyunOSS var client = GetClient(); try { + using var timeoutStream = stream.ObserveReadTimeout(_timeouts.ReadWriteTimeout, false); var objectResult = await Task.Factory.FromAsync( - (cb, state) => client.BeginPutObject(bucketName, objectName, stream, cb, state), + (cb, state) => client.BeginPutObject(bucketName, objectName, timeoutStream, cb, state), client.EndPutObject, null).ConfigureAwait(false); @@ -230,23 +223,24 @@ namespace Duplicati.Library.Backend.AliyunOSS public async Task GetAsync(string remotename, Stream stream, CancellationToken cancelToken) { var bucketName = _ossOptions.BucketName; - var objectName = $"{_ossOptions.Path.TrimPath()}/{remotename}".TrimPath(); - var client = GetClient(false); try { - var obj = await Task.Factory.FromAsync( - (cb, state) => client.BeginGetObject(bucketName, objectName, null, null), - client.EndGetObject, - null).ConfigureAwait(false); + var obj = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancelToken, async ct + => await Task.Factory.FromAsync( + (cb, state) => client.BeginGetObject(bucketName, objectName, null, null), + client.EndGetObject, + null).ConfigureAwait(false) + ).ConfigureAwait(false); if (obj.HttpStatusCode != HttpStatusCode.OK) throw new Exception("Get failed"); using (var requestStream = obj.Content) - await Library.Utility.Utility.CopyStreamAsync(requestStream, stream, cancelToken).ConfigureAwait(false); + using (var timeoutStream = requestStream.ObserveWriteTimeout(_timeouts.ReadWriteTimeout, false)) + await Utility.Utility.CopyStreamAsync(requestStream, timeoutStream, cancelToken).ConfigureAwait(false); } catch (Exception ex) { @@ -274,15 +268,15 @@ namespace Duplicati.Library.Backend.AliyunOSS // copy file var req = new CopyObjectRequest(sourceBucket, sourceObject, targetBucket, targetObject); - var res = await Task.Factory.FromAsync( - (cb, state) => client.BeginCopyObject(req, cb, state), - client.EndCopyResult, - null).ConfigureAwait(false); + var res = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancelToken, async ct + => await Task.Factory.FromAsync( + (cb, state) => client.BeginCopyObject(req, cb, state), + client.EndCopyResult, + null).ConfigureAwait(false) + ).ConfigureAwait(false); if (res?.HttpStatusCode != HttpStatusCode.OK) - { throw new Exception("file rename failed"); - } // del old file await DeleteAsync(oldname, cancelToken).ConfigureAwait(false); @@ -310,10 +304,11 @@ namespace Duplicati.Library.Backend.AliyunOSS get { return new List([ - new CommandLineArgument(OSS_ACCESS_KEY_ID, CommandLineArgument.ArgumentType.String, Strings.OSSBackend.OSSAccessKeyIdDescriptionShort, Strings.OSSBackend.OSSAccessKeyIdDescriptionLong), - new CommandLineArgument(OSS_ACCESS_KEY_SECRET, CommandLineArgument.ArgumentType.Password, Strings.OSSBackend.OSSAccessKeySecretDescriptionShort, Strings.OSSBackend.OSSAccessKeySecretDescriptionLong), + new CommandLineArgument(OSS_ACCESS_KEY_ID, CommandLineArgument.ArgumentType.String, Strings.OSSBackend.OSSAccessKeyIdDescriptionShort, Strings.OSSBackend.OSSAccessKeyIdDescriptionLong, null, ["auth-username"]), + new CommandLineArgument(OSS_ACCESS_KEY_SECRET, CommandLineArgument.ArgumentType.Password, Strings.OSSBackend.OSSAccessKeySecretDescriptionShort, Strings.OSSBackend.OSSAccessKeySecretDescriptionLong, null, ["auth-password"]), new CommandLineArgument(OSS_BUCKET_NAME, CommandLineArgument.ArgumentType.String, Strings.OSSBackend.OSSBucketNameDescriptionShort, Strings.OSSBackend.OSSBucketNameDescriptionLong), - new CommandLineArgument(OSS_ENDPOINT, CommandLineArgument.ArgumentType.String, Strings.OSSBackend.OSSEndpointDescriptionShort, Strings.OSSBackend.OSSEndpointDescriptionLong) + new CommandLineArgument(OSS_ENDPOINT, CommandLineArgument.ArgumentType.String, Strings.OSSBackend.OSSEndpointDescriptionShort, Strings.OSSBackend.OSSEndpointDescriptionLong), + .. TimeoutOptionsHelper.GetOptions() ]); } } diff --git a/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs b/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs index d82962eed..129f0c41b 100644 --- a/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs +++ b/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs @@ -22,6 +22,7 @@ using Azure; using Azure.Storage.Blobs.Models; using Duplicati.Library.Interface; +using Duplicati.Library.Utility.Options; namespace Duplicati.Library.Backend.AzureBlob { @@ -44,38 +45,19 @@ namespace Duplicati.Library.Backend.AzureBlob var uri = new Utility.Uri(url); uri.RequireHost(); - string storageAccountName = null; - string accessKey = null; - string sasToken = null; - string containerName = uri.Host.ToLowerInvariant(); + var containerName = uri.Host.ToLowerInvariant(); - if (options.ContainsKey("auth-username")) - storageAccountName = options["auth-username"]; - if (options.ContainsKey("auth-password")) - accessKey = options["auth-password"]; + var auth = AuthOptionsHelper.ParseWithAlias(options, uri, "azure-account-name", "azure-access-key"); + var timeouts = TimeoutOptionsHelper.Parse(options); - if (options.ContainsKey("azure-account-name")) - storageAccountName = options["azure-account-name"]; - if (options.ContainsKey("azure-access-key")) - accessKey = options["azure-access-key"]; - if (options.ContainsKey("azure-access-sas-token")) - sasToken = options["azure-access-sas-token"]; - - if (!string.IsNullOrEmpty(uri.Username)) - storageAccountName = uri.Username; - if (!string.IsNullOrEmpty(uri.Password)) - accessKey = uri.Password; - - if (string.IsNullOrWhiteSpace(storageAccountName)) - { + var sasToken = options.GetValueOrDefault("azure-access-sas-token"); + if (!auth.HasUsername) throw new UserInformationException(Strings.AzureBlobBackend.NoStorageAccountName, "AzureNoAccountName"); - } - if (string.IsNullOrWhiteSpace(accessKey) && string.IsNullOrWhiteSpace(sasToken)) - { + + if (!auth.HasPassword && string.IsNullOrWhiteSpace(sasToken)) throw new UserInformationException(Strings.AzureBlobBackend.NoAccessKeyOrSasToken, "AzureNoAccessKeyOrSasToken"); - } - - _azureBlob = new AzureBlobWrapper(storageAccountName, accessKey, sasToken, containerName); + + _azureBlob = new AzureBlobWrapper(auth.Username, auth.Password, sasToken, containerName, timeouts); } public string DisplayName => Strings.AzureBlobBackend.DisplayName; @@ -119,29 +101,25 @@ namespace Duplicati.Library.Backend.AzureBlob { get { - return new List(new ICommandLineArgument[] { + return [ new CommandLineArgument("azure-account-name", CommandLineArgument.ArgumentType.String, Strings.AzureBlobBackend.StorageAccountNameDescriptionShort, - Strings.AzureBlobBackend.StorageAccountNameDescriptionLong), + Strings.AzureBlobBackend.StorageAccountNameDescriptionLong, + null, + ["auth-username"]), new CommandLineArgument("azure-access-key", CommandLineArgument.ArgumentType.Password, Strings.AzureBlobBackend.AccessKeyDescriptionShort, - Strings.AzureBlobBackend.AccessKeyDescriptionLong), + Strings.AzureBlobBackend.AccessKeyDescriptionLong, + null, + ["auth-password"]), new CommandLineArgument("azure-access-sas-token", CommandLineArgument.ArgumentType.Password, Strings.AzureBlobBackend.SasTokenDescriptionShort, Strings.AzureBlobBackend.SasTokenDescriptionLong), - new CommandLineArgument("auth-password", - CommandLineArgument.ArgumentType.Password, - Strings.AzureBlobBackend.AuthPasswordDescriptionShort, - Strings.AzureBlobBackend.AuthPasswordDescriptionLong), - new CommandLineArgument("auth-username", - CommandLineArgument.ArgumentType.String, - Strings.AzureBlobBackend.AuthUsernameDescriptionShort, - Strings.AzureBlobBackend.AuthUsernameDescriptionLong) - }); - + .. TimeoutOptionsHelper.GetOptions() + ]; } } @@ -156,7 +134,7 @@ namespace Duplicati.Library.Backend.AzureBlob { return WrapWithExceptionHandler(_azureBlob.AddContainerAsync(cancellationToken)); } - + /// /// Wraps the task with exception handling /// diff --git a/Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs b/Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs index 769d06f21..bf164edd2 100644 --- a/Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs +++ b/Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs @@ -25,6 +25,8 @@ using System.Runtime.CompilerServices; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using Uri = System.Uri; +using Duplicati.Library.Utility.Options; +using Duplicati.Library.Utility; namespace Duplicati.Library.Backend.AzureBlob { @@ -34,6 +36,7 @@ namespace Duplicati.Library.Backend.AzureBlob public class AzureBlobWrapper { private readonly BlobContainerClient _container; + private readonly TimeoutOptionsHelper.Timeouts _timeouts; /// /// Gets an array of DNS names associated with the blob container. @@ -56,7 +59,8 @@ namespace Duplicati.Library.Backend.AzureBlob /// The access key for the storage account. /// The Shared Access Signature (SAS) token for authentication. /// The name of the blob container. - public AzureBlobWrapper(string accountName, string accessKey, string sasToken, string containerName) + /// The timeout options. + public AzureBlobWrapper(string accountName, string accessKey, string sasToken, string containerName, TimeoutOptionsHelper.Timeouts timeouts) { BlobServiceClient blobServiceClient; if (sasToken != null) @@ -71,6 +75,7 @@ namespace Duplicati.Library.Backend.AzureBlob } _container = blobServiceClient.GetBlobContainerClient(containerName); + _timeouts = timeouts; } /// @@ -80,8 +85,10 @@ namespace Duplicati.Library.Backend.AzureBlob /// A task that represents the asynchronous operation. public async Task AddContainerAsync(CancellationToken cancellationToken) { - // Even though PublicAccessType.None is by default, we set it explicitly to highlight it. - await _container.CreateAsync(PublicAccessType.None, cancellationToken: cancellationToken).ConfigureAwait(false); + await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct => + // Even though PublicAccessType.None is by default, we set it explicitly to highlight it. + await _container.CreateAsync(PublicAccessType.None, cancellationToken: ct).ConfigureAwait(false) + ).ConfigureAwait(false); } /// @@ -94,6 +101,7 @@ namespace Duplicati.Library.Backend.AzureBlob public async Task GetFileStreamAsync(string keyName, Stream target, CancellationToken cancellationToken) { var blobClient = _container.GetBlobClient(keyName); + using var timeoutStream = target.ObserveWriteTimeout(_timeouts.ReadWriteTimeout, false); await blobClient.DownloadToAsync(target, cancellationToken).ConfigureAwait(false); } @@ -107,7 +115,8 @@ namespace Duplicati.Library.Backend.AzureBlob public async Task AddFileStream(string keyName, Stream source, CancellationToken cancelToken) { var blobClient = _container.GetBlobClient(keyName); - await blobClient.UploadAsync(source, true, cancelToken).ConfigureAwait(false); + using var timeoutStream = source.ObserveReadTimeout(_timeouts.ReadWriteTimeout, false); + await blobClient.UploadAsync(timeoutStream, true, cancelToken).ConfigureAwait(false); } /// @@ -118,9 +127,11 @@ namespace Duplicati.Library.Backend.AzureBlob public async Task DeleteObjectAsync(string keyName, CancellationToken cancelToken) { var blobClient = _container.GetBlobClient(keyName); - await blobClient.DeleteIfExistsAsync(cancellationToken: cancelToken).ConfigureAwait(false); + await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancelToken, async ct => + await blobClient.DeleteIfExistsAsync(cancellationToken: ct).ConfigureAwait(false) + ).ConfigureAwait(false); } - + /// /// List container files. /// @@ -130,29 +141,31 @@ namespace Duplicati.Library.Backend.AzureBlob public virtual async IAsyncEnumerable ListContainerEntriesAsync([EnumeratorCancellation] CancellationToken cancelToken) { await using var blobEnumerator = _container.GetBlobsAsync().GetAsyncEnumerator(cancelToken); - + while (true) { bool hasNext; - + try { - hasNext = await blobEnumerator.MoveNextAsync().ConfigureAwait(false); + hasNext = await Utility.Utility.WithTimeout(_timeouts.ListTimeout, cancelToken, async ct + => await blobEnumerator.MoveNextAsync().ConfigureAwait(false) + ).ConfigureAwait(false); } catch (Azure.RequestFailedException ex) when (ex.Status == 404) { throw new FolderMissingException(ex); } - + if (!hasNext) break; - + cancelToken.ThrowIfCancellationRequested(); - + if (blobEnumerator.Current is { } blob) { var blobName = Uri.UnescapeDataString(blob.Name.Replace("+", "%2B")); var lastModified = blob.Properties.LastModified?.UtcDateTime ?? DateTime.UtcNow; - + yield return new FileEntry( blobName, blob.Properties.ContentLength ?? 0, diff --git a/Duplicati/Library/Backend/AzureBlob/Strings.cs b/Duplicati/Library/Backend/AzureBlob/Strings.cs index e0693f8d2..9c4be64c2 100644 --- a/Duplicati/Library/Backend/AzureBlob/Strings.cs +++ b/Duplicati/Library/Backend/AzureBlob/Strings.cs @@ -23,7 +23,8 @@ using Duplicati.Library.Localization.Short; namespace Duplicati.Library.Backend.AzureBlob.Strings; -internal static class AzureBlobBackend { +internal static class AzureBlobBackend +{ public static string DescriptionV2 => LC.L(@"This backend can read and write data to Azure blob storage. Allowed format is ""azure://bucketname""."); public static string DisplayName => LC.L(@"Azure blob"); public static string ContainerNameDescriptionLong => LC.L(@"All files will be written to the container specified."); @@ -36,8 +37,4 @@ internal static class AzureBlobBackend { public static string SasTokenDescriptionLong => LC.L(@"The Azure shared access signature (SAS) token which can be obtained by selecting the ""Shared access signature"" blade on the storage account dashboard, or inside a container blade."); public static string SasTokenDescriptionShort => LC.L(@"The SAS token"); public static string NoAccessKeyOrSasToken => LC.L(@"No Azure access key or SAS token given"); - public static string AuthPasswordDescriptionLong => LC.L(@"The password used to connect to the server. This may also be supplied as the environment variable ""AUTH_PASSWORD""."); - public static string AuthPasswordDescriptionShort => LC.L(@"Supply the password used to connect to the server"); - public static string AuthUsernameDescriptionLong => LC.L(@"The username used to connect to the server. This may also be supplied as the environment variable ""AUTH_USERNAME""."); - public static string AuthUsernameDescriptionShort => LC.L(@"Supply the username used to connect to the server"); } \ No newline at end of file From 58a9d14d4d20634cdcee5e90954f062050a6186e Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 19 Mar 2025 23:29:28 +0100 Subject: [PATCH 04/48] Updated Backblaze --- Duplicati/Library/Backend/Backblaze/B2.cs | 153 ++++++++---------- .../Library/Backend/Backblaze/Strings.cs | 54 +++---- 2 files changed, 90 insertions(+), 117 deletions(-) diff --git a/Duplicati/Library/Backend/Backblaze/B2.cs b/Duplicati/Library/Backend/Backblaze/B2.cs index 4ed35e91b..5a343688f 100644 --- a/Duplicati/Library/Backend/Backblaze/B2.cs +++ b/Duplicati/Library/Backend/Backblaze/B2.cs @@ -28,6 +28,7 @@ using System.Reflection; using Duplicati.Library.Backend.Backblaze.Model; using FileEntry = Duplicati.Library.Common.IO.FileEntry; using System.Runtime.CompilerServices; +using Duplicati.Library.Utility.Options; namespace Duplicati.Library.Backend.Backblaze; @@ -36,11 +37,6 @@ namespace Duplicati.Library.Backend.Backblaze; /// public class B2 : IStreamingBackend, ITimeoutExemptBackend { - /// - /// The default timeout in seconds for LIST/CreateFolder operations - /// - private const int SHORT_OPERATION_TIMEOUT_SECONDS = 30; - /// /// The option key for specifying the Backblaze B2 account ID /// @@ -121,6 +117,11 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend /// private readonly B2AuthHelper _b2AuthHelper; + /// + /// The timeout options for the backend + /// + private readonly TimeoutOptionsHelper.Timeouts _timeouts; + /// /// Cached upload URL and authorization token for file uploads /// @@ -177,28 +178,12 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend if (options.TryGetValue(B2_CREATE_BUCKET_TYPE_OPTION, out var option1)) _bucketType = option1; - string accountId = null; - string accountKey = null; + var auth = AuthOptionsHelper.ParseWithAlias(options, uri, B2_ID_OPTION, B2_KEY_OPTION); - // Takes the account ID and key from the options, or from the URL with the cascading precedence - - if (options.TryGetValue("auth-username", out var authUsernameOption)) - accountId = authUsernameOption; - if (options.TryGetValue("auth-password", out var authPasswordOption)) - accountKey = authPasswordOption; - if (options.TryGetValue(B2_ID_OPTION, out var accountIdOption)) - accountId = accountIdOption; - if (options.TryGetValue(B2_KEY_OPTION, out var accountKeyOption)) - accountKey = accountKeyOption; - if (!string.IsNullOrEmpty(uri.Username)) - accountId = uri.Username; - if (!string.IsNullOrEmpty(uri.Password)) - accountKey = uri.Password; - - if (string.IsNullOrEmpty(accountId)) + if (!auth.HasUsername) throw new UserInformationException(Strings.B2.NoB2UserIDError, "B2MissingUserID"); - if (string.IsNullOrEmpty(accountKey)) + if (!auth.HasPassword) throw new UserInformationException(Strings.B2.NoB2KeyError, "B2MissingKey"); _pageSize = DEFAULT_PAGE_SIZE; @@ -216,7 +201,8 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend _httpClient = HttpClientHelper.CreateClient(); _httpClient.Timeout = Timeout.InfiniteTimeSpan; - _b2AuthHelper = new B2AuthHelper(accountId, accountKey, _httpClient); + _b2AuthHelper = new B2AuthHelper(auth.Username, auth.Password, _httpClient); + _timeouts = TimeoutOptionsHelper.Parse(options); } @@ -238,14 +224,12 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend try { - using var timeoutToken = new CancellationTokenSource(); - timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS)); - using var combinedCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken); - - var buckets = await _b2AuthHelper.PostAndGetJsonDataAsync( - $"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_list_buckets", - new ListBucketsRequest(accountId: _b2AuthHelper.AccountId), - combinedCancellationToken.Token).ConfigureAwait(false); + var buckets = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct + => await _b2AuthHelper.PostAndGetJsonDataAsync( + $"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_list_buckets", + new ListBucketsRequest(accountId: _b2AuthHelper.AccountId), + ct).ConfigureAwait(false) + ).ConfigureAwait(false); return _bucket ??= buckets?.Buckets?.FirstOrDefault(x => string.Equals(x.BucketName, _bucketName, StringComparison.OrdinalIgnoreCase)) @@ -266,14 +250,11 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend if (_uploadUrl != null) return _uploadUrl; - using var timeoutToken = new CancellationTokenSource(); - timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS)); - using var combinedCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken); - - _uploadUrl = await _b2AuthHelper.PostAndGetJsonDataAsync( - $"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_get_upload_url", - new UploadUrlRequest { BucketID = (await GetBucketAsync(cancellationToken)).BucketID }, - combinedCancellationToken.Token + _uploadUrl = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct + => await _b2AuthHelper.PostAndGetJsonDataAsync( + $"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_get_upload_url", + new UploadUrlRequest { BucketID = (await GetBucketAsync(cancellationToken)).BucketID }, + ct).ConfigureAwait(false) ).ConfigureAwait(false); return _uploadUrl; @@ -311,11 +292,10 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend ["auth-password"], null), new CommandLineArgument(B2_KEY_OPTION, CommandLineArgument.ArgumentType.Password, Strings.B2.B2applicationkeyDescriptionShort, Strings.B2.B2applicationkeyDescriptionLong, null, ["auth-username"], null), - new CommandLineArgument("auth-password", CommandLineArgument.ArgumentType.Password, Strings.B2.AuthPasswordDescriptionShort, Strings.B2.AuthPasswordDescriptionLong), - new CommandLineArgument("auth-username", CommandLineArgument.ArgumentType.String, Strings.B2.AuthUsernameDescriptionShort, Strings.B2.AuthUsernameDescriptionLong), new CommandLineArgument(B2_CREATE_BUCKET_TYPE_OPTION, CommandLineArgument.ArgumentType.String, Strings.B2.B2createbuckettypeDescriptionShort, Strings.B2.B2createbuckettypeDescriptionLong, DEFAULT_BUCKET_TYPE), new CommandLineArgument(B2_PAGESIZE_OPTION, CommandLineArgument.ArgumentType.Integer, Strings.B2.B2pagesizeDescriptionShort, Strings.B2.B2pagesizeDescriptionLong, DEFAULT_PAGE_SIZE.ToString()), - new CommandLineArgument(B2_DOWNLOAD_URL_OPTION, CommandLineArgument.ArgumentType.String, Strings.B2.B2downloadurlDescriptionShort, Strings.B2.B2downloadurlDescriptionLong) + new CommandLineArgument(B2_DOWNLOAD_URL_OPTION, CommandLineArgument.ArgumentType.String, Strings.B2.B2downloadurlDescriptionShort, Strings.B2.B2downloadurlDescriptionLong), + ..TimeoutOptionsHelper.GetOptions() ]); public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken) @@ -339,7 +319,7 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend // Compute the hash using (var hashalg = HashFactory.CreateHasher("SHA1")) - sha1 = Library.Utility.Utility.ByteArrayAsHexString(hashalg.ComputeHash(measure)); + sha1 = Utility.Utility.ByteArrayAsHexString(hashalg.ComputeHash(measure)); measure.Position = p; } @@ -367,14 +347,15 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend // For PutAsync, no timeout is specified. The only thing that can stop it is the cancellation token using var request = new HttpRequestMessage(HttpMethod.Post, uploadUrlData.UploadUrl); + using var timeoutStream = stream.ObserveReadTimeout(_timeouts.ReadWriteTimeout, false); request.Headers.TryAddWithoutValidation("Authorization", uploadUrlData.AuthorizationToken); request.Headers.Add("X-Bz-Content-Sha1", sha1); request.Headers.Add("X-Bz-File-Name", _urlencodedPrefix + Utility.Uri.UrlPathEncode(remotename)); - request.Content = new StreamContent(stream, B2_RECOMMENDED_CHUNK_SIZE); + request.Content = new StreamContent(timeoutStream, B2_RECOMMENDED_CHUNK_SIZE); request.Content.Headers.Add("Content-Type", "application/octet-stream"); - request.Content.Headers.Add("Content-Length", stream.Length.ToString()); + request.Content.Headers.Add("Content-Length", timeoutStream.Length.ToString()); var response = await _httpClient.UploadStream(request, cancelToken).ConfigureAwait(false); var rdata = await response.Content.ReadAsStreamAsync(cancelToken).ConfigureAwait(false); @@ -444,7 +425,8 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend .ConfigureAwait(false); response.EnsureSuccessStatusCode(); await using var responseStream = await response.Content.ReadAsStreamAsync(cancellationToken); - await responseStream.CopyToAsync(stream, cancellationToken).ConfigureAwait(false); + using var timeoutStream = responseStream.ObserveReadTimeout(_timeouts.ReadWriteTimeout, false); + await timeoutStream.CopyToAsync(stream, cancellationToken).ConfigureAwait(false); } catch (Exception ex) { @@ -478,22 +460,19 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend { try { - using var timeoutToken = new CancellationTokenSource(); - timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS)); - using var combinedCancellationToken = - CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken); - - var resp = await _b2AuthHelper.PostAndGetJsonDataAsync( - $"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_list_file_versions", - new ListFilesRequest - { - BucketID = (await GetBucketAsync(cancellationToken)).BucketID, - MaxFileCount = _pageSize, - Prefix = _prefix, - StartFileID = nextFileId, - StartFileName = nextFileName - }, - combinedCancellationToken.Token + var resp = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct + => await _b2AuthHelper.PostAndGetJsonDataAsync( + $"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_list_file_versions", + new ListFilesRequest + { + BucketID = (await GetBucketAsync(cancellationToken)).BucketID, + MaxFileCount = _pageSize, + Prefix = _prefix, + StartFileID = nextFileId, + StartFileName = nextFileName + }, + ct + ).ConfigureAwait(false) ).ConfigureAwait(false); nextFileId = resp.NextFileID; @@ -581,10 +560,6 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend { try { - using var timeoutToken = new CancellationTokenSource(); - timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS)); - using var combinedCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken); - if (_filecache == null || !_filecache.ContainsKey(remotename)) await RebuildFileCache(cancellationToken).ConfigureAwait(false); @@ -592,14 +567,16 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend throw new FileMissingException(); foreach (var n in value.OrderBy(x => x.UploadTimestamp)) - await _b2AuthHelper.PostAndGetJsonDataAsync( - $"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_delete_file_version", - new DeleteRequest - { - FileName = _prefix + remotename, - FileId = n.FileID - }, - combinedCancellationToken.Token + await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct + => await _b2AuthHelper.PostAndGetJsonDataAsync( + $"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_delete_file_version", + new DeleteRequest + { + FileName = _prefix + remotename, + FileId = n.FileID + }, + ct + ).ConfigureAwait(false) ).ConfigureAwait(false); _filecache[remotename].Clear(); @@ -626,19 +603,17 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend /// public async Task CreateFolderAsync(CancellationToken cancellationToken) { - using var timeoutToken = new CancellationTokenSource(); - timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS)); - using var combinedCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken); - - _bucket = await _b2AuthHelper.PostAndGetJsonDataAsync( - $"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_create_bucket", - new BucketEntity - { - AccountID = _b2AuthHelper.AccountId, - BucketName = _bucketName, - BucketType = _bucketType - }, - combinedCancellationToken.Token + _bucket = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct + => await _b2AuthHelper.PostAndGetJsonDataAsync( + $"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_create_bucket", + new BucketEntity + { + AccountID = _b2AuthHelper.AccountId, + BucketName = _bucketName, + BucketType = _bucketType + }, + ct + ).ConfigureAwait(false) ).ConfigureAwait(false); } diff --git a/Duplicati/Library/Backend/Backblaze/Strings.cs b/Duplicati/Library/Backend/Backblaze/Strings.cs index 83580fab0..a10b2881f 100644 --- a/Duplicati/Library/Backend/Backblaze/Strings.cs +++ b/Duplicati/Library/Backend/Backblaze/Strings.cs @@ -1,45 +1,43 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Localization.Short; -namespace Duplicati.Library.Backend.Strings { - internal static class B2 { +namespace Duplicati.Library.Backend.Strings +{ + internal static class B2 + { public static string Description { get { return LC.L(@"This backend can read and write data to the Backblaze B2 Cloud Storage. Allowed format is ""b2://bucketname/prefix""."); } } public static string DisplayName { get { return LC.L(@"B2 Cloud Storage"); } } public static string B2applicationkeyDescriptionLong { get { return LC.L(@"B2 Cloud Storage Application Key can be obtained after logging into your Backblaze account. This can also be supplied through the option --{0}.", "auth-password"); } } public static string B2applicationkeyDescriptionShort { get { return LC.L(@"B2 Cloud Storage Application Key"); } } public static string B2accountidDescriptionLong { get { return LC.L(@"B2 Cloud Storage Account ID can be obtained after logging into your Backblaze account. This can also be supplied through the option --{0}.", "auth-username"); } } public static string B2accountidDescriptionShort { get { return LC.L(@"B2 Cloud Storage Account ID"); } } - public static string AuthPasswordDescriptionLong { get { return LC.L(@"The password used to connect to the server. This may also be supplied as the environment variable ""AUTH_PASSWORD""."); } } - public static string AuthPasswordDescriptionShort { get { return LC.L(@"Supply the password used to connect to the server"); } } - public static string AuthUsernameDescriptionLong { get { return LC.L(@"The username used to connect to the server. This may also be supplied as the environment variable ""AUTH_USERNAME""."); } } - public static string AuthUsernameDescriptionShort { get { return LC.L(@"Supply the username used to connect to the server"); } } public static string NoB2KeyError { get { return LC.L(@"No B2 Cloud Storage Application Key given"); } } public static string NoB2UserIDError { get { return LC.L(@"No B2 Cloud Storage Account ID given"); } } public static string B2createbuckettypeDescriptionLong { get { return LC.L(@"By default, a private bucket is created. Use this option to set the bucket type. Refer to the B2 documentation for allowed types."); } } public static string B2createbuckettypeDescriptionShort { get { return LC.L(@"The bucket type used when creating a bucket"); } } - public static string B2pagesizeDescriptionLong { get { return LC.L(@"Use this option to set the page size for listing contents of B2 buckets. A lower number means less data, but can increase the number of Class C transaction on B2. Suggested values are between 100 and 1000."); } } - public static string B2pagesizeDescriptionShort { get { return LC.L(@"The size of file-listing pages"); } } + public static string B2pagesizeDescriptionLong { get { return LC.L(@"Use this option to set the page size for listing contents of B2 buckets. A lower number means less data, but can increase the number of Class C transaction on B2. Suggested values are between 100 and 1000."); } } + public static string B2pagesizeDescriptionShort { get { return LC.L(@"The size of file-listing pages"); } } public static string B2downloadurlDescriptionLong { get { return LC.L(@"Change this if you want to use your custom domain to download files, and uploading will not be affected. The default download URL depends on your account and looks like ""https://f00X.backblazeb2.com""."); } } public static string B2downloadurlDescriptionShort { get { return LC.L(@"The base URL to use for downloading files"); } } public static string InvalidPageSizeError(string argname, string value) { return LC.L(@"The setting ""{0}"" is invalid for ""{1}"". It must be an integer larger than zero.", value, argname); } - } + } } From 8a77b2e6e581337e6e55a59447548868faf9e07a Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 20 Mar 2025 10:11:30 +0100 Subject: [PATCH 05/48] Improved error messages for Dropbox --- Duplicati/Library/Backend/Dropbox/Dropbox.cs | 53 +++++++------------ .../Library/Backend/Dropbox/DropboxHelper.cs | 42 ++++++++------- 2 files changed, 41 insertions(+), 54 deletions(-) diff --git a/Duplicati/Library/Backend/Dropbox/Dropbox.cs b/Duplicati/Library/Backend/Dropbox/Dropbox.cs index e929d9da9..b68302980 100644 --- a/Duplicati/Library/Backend/Dropbox/Dropbox.cs +++ b/Duplicati/Library/Backend/Dropbox/Dropbox.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; @@ -98,21 +98,6 @@ namespace Duplicati.Library.Backend return ife; } - private T HandleListExceptions(Func func) - { - try - { - return func(); - } - catch (DropboxException de) - { - if (de.errorJSON["error"][".tag"].ToString() == "path" && de.errorJSON["error"]["path"][".tag"].ToString() == "not_found") - throw new FolderMissingException(); - - throw; - } - } - private async Task HandleListExceptions(Func> func) { try diff --git a/Duplicati/Library/Backend/Dropbox/DropboxHelper.cs b/Duplicati/Library/Backend/Dropbox/DropboxHelper.cs index cb9d23c2a..c5e1bfe47 100644 --- a/Duplicati/Library/Backend/Dropbox/DropboxHelper.cs +++ b/Duplicati/Library/Backend/Dropbox/DropboxHelper.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Utility; using Newtonsoft.Json; @@ -276,8 +276,10 @@ namespace Duplicati.Library.Backend if (errJson != null) throw new DropboxException() { errorJSON = errJson }; + else if (exception.Response is HttpWebResponse wresp) + throw new InvalidDataException($"Non-json response (code: {wresp.StatusCode}, message: {wresp.StatusDescription}): {json}", ex); else - throw new InvalidDataException($"Non-json response: {json}"); + throw new InvalidDataException($"Non-json response: {json}", ex); } } } From d1659b7d4088c6196005b7cfb793f16c98601887 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 20 Mar 2025 10:15:27 +0100 Subject: [PATCH 06/48] Fixed FTP credentials not being parsed --- Duplicati/Library/Backend/FTP/FTPBackend.cs | 2 +- Duplicati/Library/Utility/Options/AuthOptionsHelper.cs | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Duplicati/Library/Backend/FTP/FTPBackend.cs b/Duplicati/Library/Backend/FTP/FTPBackend.cs index 23a777161..b9609801a 100644 --- a/Duplicati/Library/Backend/FTP/FTPBackend.cs +++ b/Duplicati/Library/Backend/FTP/FTPBackend.cs @@ -273,7 +273,7 @@ namespace Duplicati.Library.Backend var u = new Utility.Uri(url); u.RequireHost(); - var auth = AuthOptionsHelper.Parse(options); + var auth = AuthOptionsHelper.Parse(options, u); if (auth.HasUsername) { _userInfo = new NetworkCredential() diff --git a/Duplicati/Library/Utility/Options/AuthOptionsHelper.cs b/Duplicati/Library/Utility/Options/AuthOptionsHelper.cs index 94b0f1f34..9105c2667 100644 --- a/Duplicati/Library/Utility/Options/AuthOptionsHelper.cs +++ b/Duplicati/Library/Utility/Options/AuthOptionsHelper.cs @@ -55,6 +55,10 @@ public static class AuthOptionsHelper new CommandLineArgument(password, CommandLineArgument.ArgumentType.Password, Strings.AuthSettingsHelper.DescriptionAuthPasswordShort, Strings.AuthSettingsHelper.DescriptionAuthPasswordLong, null, ["auth-password"]) ]; + /* + + TODO: See if we can do without these, so we only have one way of parsing the auth options + /// /// Parses the authentication options from a dictionary /// @@ -86,6 +90,7 @@ public static class AuthOptionsHelper return new AuthOptions(optionUsername, optionPassword); } + */ /// /// Parses the authentication options from a dictionary From 0247f13ebf68c64de0432ce439bbc3565fa6d51b Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 20 Mar 2025 10:54:38 +0100 Subject: [PATCH 07/48] Updated box.com --- .../Backend/AliyunOSS/AliyunOSSBackend.cs | 4 +- Duplicati/Library/Backend/Box/BoxBackend.cs | 164 +++++++++++------- Duplicati/Library/Backend/Box/Strings.cs | 25 ++- .../Backend/OAuthHelper/OAuthHelper.cs | 104 ++++++----- .../Utility/Options/AuthIdOptionsHelper.cs | 78 +++++++++ .../Utility/Options/AuthOptionsHelper.cs | 4 +- Duplicati/Library/Utility/Strings.cs | 8 + 7 files changed, 254 insertions(+), 133 deletions(-) create mode 100644 Duplicati/Library/Utility/Options/AuthIdOptionsHelper.cs diff --git a/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs b/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs index fdb37170f..a7661a9ac 100644 --- a/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs +++ b/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs @@ -65,8 +65,8 @@ namespace Duplicati.Library.Backend.AliyunOSS if (!string.IsNullOrEmpty(prefix)) _ossOptions.Path = prefix; - var auth = AuthOptionsHelper.ParseWithAlias(options, uri, OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY_SECRET); - auth.RequireCredentials(); + var auth = AuthOptionsHelper.ParseWithAlias(options, uri, OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY_SECRET) + .RequireCredentials(); _ossOptions.AccessKeyId = auth.Username; _ossOptions.AccessKeySecret = auth.Password; diff --git a/Duplicati/Library/Backend/Box/BoxBackend.cs b/Duplicati/Library/Backend/Box/BoxBackend.cs index d256255d9..21eb991a1 100644 --- a/Duplicati/Library/Backend/Box/BoxBackend.cs +++ b/Duplicati/Library/Backend/Box/BoxBackend.cs @@ -19,8 +19,12 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +#nullable enable + using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; +using Duplicati.Library.Utility; +using Duplicati.Library.Utility.Options; using Newtonsoft.Json; using System; using System.Collections.Generic; @@ -37,10 +41,11 @@ namespace Duplicati.Library.Backend.Box public class BoxBackend : IBackend, IStreamingBackend { private static readonly string LOGTAG = Logging.Log.LogTagFromType(); - - private const string AUTHID_OPTION = "authid"; private const string REALLY_DELETE_OPTION = "box-delete-from-trash"; + private const string OAUTH_MODULE = "box.com"; + private static readonly string OAUTH_LOGIN_URL = OAuthHelper.OAUTH_LOGIN_URL(OAUTH_MODULE); + private const string BOX_API_URL = "https://api.box.com/2.0"; private const string BOX_UPLOAD_URL = "https://upload.box.com/api/2.0/files"; @@ -50,26 +55,28 @@ namespace Duplicati.Library.Backend.Box private readonly string m_path; private readonly bool m_deleteFromTrash; - private string m_currentfolder; + private readonly TimeoutOptionsHelper.Timeouts m_timeouts; + + private string? m_currentfolder; private readonly Dictionary m_filecache = new Dictionary(); private class BoxHelper : OAuthHelper { public BoxHelper(string authid) - : base(authid, "box.com") + : base(authid, OAUTH_MODULE) { AutoAuthHeader = true; } protected override void ParseException(Exception ex) { - Exception newex = null; + Exception? newex = null; try { if (ex is WebException exception && exception.Response is HttpWebResponse hs) { - string rawdata = null; - using (var rs = Library.Utility.AsyncHttpRequest.TrySetTimeout(hs.GetResponseStream())) + string? rawdata = null; + using (var rs = AsyncHttpRequest.TrySetTimeout(hs.GetResponseStream())) using (var sr = new System.IO.StreamReader(rs)) rawdata = sr.ReadToEnd(); @@ -79,7 +86,7 @@ namespace Duplicati.Library.Backend.Box newex = new Exception("Raw message: " + rawdata); var msg = JsonConvert.DeserializeObject(rawdata); - newex = new Exception(string.Format("{0} - {1}: {2}", msg.Status, msg.Code, msg.Message)); + newex = new Exception(string.Format("{0} - {1}: {2}", msg?.Status, msg?.Code, msg?.Message)); /*if (msg.ContextInfo != null && msg.ContextInfo.Length > 0) newex = new Exception(string.Format("{0} - {1}: {2}{3}{4}", msg.Status, msg.Code, msg.Message, Environment.NewLine, string.Join("; ", from n in msg.ContextInfo select n.Message))); @@ -100,56 +107,63 @@ namespace Duplicati.Library.Backend.Box // This constructor is needed by the BackendLoader. public BoxBackend() { + m_oauth = null!; + m_path = null!; + m_timeouts = null!; } // ReSharper disable once UnusedMember.Global // This constructor is needed by the BackendLoader. - public BoxBackend(string url, Dictionary options) + public BoxBackend(string url, Dictionary options) { var uri = new Utility.Uri(url); m_path = Util.AppendDirSeparator(uri.HostAndPath, "/"); - string authid = null; - if (options.ContainsKey(AUTHID_OPTION)) - authid = options[AUTHID_OPTION]; - + var authid = AuthIdOptionsHelper.Parse(options) + .RequireCredentials(OAUTH_LOGIN_URL); m_deleteFromTrash = Library.Utility.Utility.ParseBoolOption(options, REALLY_DELETE_OPTION); + m_timeouts = TimeoutOptionsHelper.Parse(options); - m_oauth = new BoxHelper(authid); + m_oauth = new BoxHelper(authid.AuthId!); } private async Task GetCurrentFolderWithCacheAsync(CancellationToken cancelToken) { if (m_currentfolder == null) - await GetCurrentFolderAsync(false, cancelToken).ConfigureAwait(false); + return await GetCurrentFolderAsync(false, cancelToken).ConfigureAwait(false); return m_currentfolder; } - private async Task GetCurrentFolderAsync(bool create, CancellationToken cancelToken) + private async Task GetCurrentFolderAsync(bool create, CancellationToken cancelToken) { var parentid = "0"; foreach (var p in m_path.Split(new string[] { "/" }, StringSplitOptions.RemoveEmptyEntries)) { - var el = (MiniFolder)await PagedFileListResponse(parentid, true, cancelToken).FirstOrDefaultAsync(x => x.Name == p).ConfigureAwait(false); + var el = (MiniFolder?)await PagedFileListResponse(parentid, true, cancelToken).FirstOrDefaultAsync(x => x.Name == p).ConfigureAwait(false); if (el == null) { if (!create) throw new FolderMissingException(); - el = await m_oauth.PostAndGetJSONDataAsync( - string.Format("{0}/folders", BOX_API_URL), - cancelToken, - new CreateItemRequest() { Name = p, Parent = new IDReference() { ID = parentid } } + el = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, async tr => + await m_oauth.PostAndGetJSONDataAsync( + string.Format("{0}/folders", BOX_API_URL), + tr, + new CreateItemRequest() { Name = p, Parent = new IDReference() { ID = parentid } } + ).ConfigureAwait(false) ).ConfigureAwait(false); } + if (string.IsNullOrWhiteSpace(el.ID)) + throw new Exception("Folder ID is empty?"); + parentid = el.ID; } - m_currentfolder = parentid; + return m_currentfolder = parentid; } private async Task GetFileIDAsync(string name, CancellationToken cancelToken) @@ -177,7 +191,9 @@ namespace Duplicati.Library.Backend.Box do { - var resp = await m_oauth.GetJSONDataAsync($"{BOX_API_URL}/folders/{parentid}/items?limit={PAGE_SIZE}&offset={offset}&fields=name,size,modified_at", cancelToken).ConfigureAwait(false); + var resp = await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, async ct + => await m_oauth.GetJSONDataAsync($"{BOX_API_URL}/folders/{parentid}/items?limit={PAGE_SIZE}&offset={offset}&fields=name,size,modified_at", ct).ConfigureAwait(false) + ).ConfigureAwait(false); if (resp.Entries == null || resp.Entries.Length == 0) break; @@ -191,6 +207,12 @@ namespace Duplicati.Library.Backend.Box } else { + if (string.IsNullOrWhiteSpace(f.ID) || string.IsNullOrWhiteSpace(f.Name)) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "BoxFileList", null, "Skipping invalid entry"); + continue; + } + if (!onlyfolders && f.Type == "file") m_filecache[f.Name] = f.ID; @@ -240,7 +262,12 @@ namespace Duplicati.Library.Backend.Box items.Add(new MultipartItem(stream, "file", remotename)); - var res = (await m_oauth.PostMultipartAndGetJSONDataAsync(url, null, cancelToken, items.ToArray())).Entries.First(); + // TODO: Add read-write-timeout when upgraded away from WebRequest + + var res = (await m_oauth.PostMultipartAndGetJSONDataAsync(url, null, cancelToken, items.ToArray())).Entries?.FirstOrDefault(); + if (string.IsNullOrWhiteSpace(res?.ID)) + throw new Exception("Failed to get ID from uploaded file"); + m_filecache[remotename] = res.ID; } catch @@ -253,9 +280,13 @@ namespace Duplicati.Library.Backend.Box public async Task GetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) { var fileId = await GetFileIDAsync(remotename, cancelToken).ConfigureAwait(false); - using (var resp = await m_oauth.GetResponseAsync(string.Format("{0}/files/{1}/content", BOX_API_URL, fileId), cancelToken).ConfigureAwait(false)) + using var resp = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, async ct + => await m_oauth.GetResponseAsync(string.Format("{0}/files/{1}/content", BOX_API_URL, fileId), cancelToken).ConfigureAwait(false) + ).ConfigureAwait(false); + using (var rs = Duplicati.Library.Utility.AsyncHttpRequest.TrySetTimeout(resp.GetResponseStream())) - await Library.Utility.Utility.CopyStreamAsync(rs, stream, cancelToken).ConfigureAwait(false); + using (var ts = rs.ObserveReadTimeout(m_timeouts.ReadWriteTimeout)) + await Library.Utility.Utility.CopyStreamAsync(ts, stream, cancelToken).ConfigureAwait(false); } #endregion @@ -286,12 +317,16 @@ namespace Duplicati.Library.Backend.Box var fileid = await GetFileIDAsync(remotename, cancelToken).ConfigureAwait(false); try { - using (var r = await m_oauth.GetResponseAsync(string.Format("{0}/files/{1}", BOX_API_URL, fileid), cancelToken, null, "DELETE").ConfigureAwait(false)) + using (var r = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, async ct => + await m_oauth.GetResponseAsync(string.Format("{0}/files/{1}", BOX_API_URL, fileid), ct, null, "DELETE").ConfigureAwait(false) + ).ConfigureAwait(false)) { } if (m_deleteFromTrash) - using (var r = await m_oauth.GetResponseAsync(string.Format("{0}/files/{1}/trash", BOX_API_URL, fileid), cancelToken, null, "DELETE").ConfigureAwait(false)) + using (var r = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, async ct + => await m_oauth.GetResponseAsync(string.Format("{0}/files/{1}/trash", BOX_API_URL, fileid), cancelToken, null, "DELETE").ConfigureAwait(false) + ).ConfigureAwait(false)) { } } @@ -330,10 +365,11 @@ namespace Duplicati.Library.Backend.Box { get { - return new List(new ICommandLineArgument[] { - new CommandLineArgument(AUTHID_OPTION, CommandLineArgument.ArgumentType.Password, Strings.Box.AuthidShort, Strings.Box.AuthidLong(OAuthHelper.OAUTH_LOGIN_URL("box.com"))), + return [ + .. AuthIdOptionsHelper.GetOptions(OAUTH_LOGIN_URL), new CommandLineArgument(REALLY_DELETE_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.Box.ReallydeleteShort, Strings.Box.ReallydeleteLong), - }); + .. TimeoutOptionsHelper.GetOptions() + ]; } } @@ -363,23 +399,23 @@ namespace Duplicati.Library.Backend.Box private class MiniUser : IDReference { [JsonProperty("type")] - public string Type { get; set; } + public string? Type { get; set; } [JsonProperty("name")] - public string Name { get; set; } + public string? Name { get; set; } [JsonProperty("login")] - public string Login { get; set; } + public string? Login { get; set; } } private class MiniFolder : IDReference { [JsonProperty("type")] - public string Type { get; set; } + public string? Type { get; set; } [JsonProperty("name")] - public string Name { get; set; } + public string? Name { get; set; } [JsonProperty("etag")] - public string ETag { get; set; } + public string? ETag { get; set; } [JsonProperty("sequence_id")] - public string SequenceID { get; set; } + public string? SequenceID { get; set; } } private class FileEntity : MiniFolder @@ -387,7 +423,7 @@ namespace Duplicati.Library.Backend.Box public FileEntity() { Size = -1; } [JsonProperty("sha1")] - public string SHA1 { get; set; } + public string? SHA1 { get; set; } [JsonProperty("size", NullValueHandling = NullValueHandling.Ignore)] public long Size { get; set; } @@ -400,7 +436,7 @@ namespace Duplicati.Library.Backend.Box [JsonProperty("total_count")] public long TotalCount { get; set; } [JsonProperty("entries")] - public MiniFolder[] Entries { get; set; } + public MiniFolder[]? Entries { get; set; } } private class FileList @@ -408,7 +444,7 @@ namespace Duplicati.Library.Backend.Box [JsonProperty("total_count")] public long TotalCount { get; set; } [JsonProperty("entries")] - public FileEntity[] Entries { get; set; } + public FileEntity[]? Entries { get; set; } [JsonProperty("offset")] public long Offset { get; set; } [JsonProperty("limit")] @@ -418,9 +454,9 @@ namespace Duplicati.Library.Backend.Box private class UploadEmail { [JsonProperty("access")] - public string Access { get; set; } + public string? Access { get; set; } [JsonProperty("email")] - public string Email { get; set; } + public string? Email { get; set; } } private class ListFolderResponse : MiniFolder @@ -430,80 +466,80 @@ namespace Duplicati.Library.Backend.Box [JsonProperty("modified_at")] public DateTime ModifiedAt { get; set; } [JsonProperty("description")] - public string Description { get; set; } + public string? Description { get; set; } [JsonProperty("size")] public long Size { get; set; } [JsonProperty("path_collection")] - public FolderList PathCollection { get; set; } + public FolderList? PathCollection { get; set; } [JsonProperty("created_by")] - public MiniUser CreatedBy { get; set; } + public MiniUser? CreatedBy { get; set; } [JsonProperty("modified_by")] - public MiniUser ModifiedBy { get; set; } + public MiniUser? ModifiedBy { get; set; } [JsonProperty("owned_by")] - public MiniUser OwnedBy { get; set; } + public MiniUser? OwnedBy { get; set; } [JsonProperty("shared_link")] - public MiniUser SharedLink { get; set; } + public MiniUser? SharedLink { get; set; } [JsonProperty("folder_upload_email")] - public UploadEmail FolderUploadEmail { get; set; } + public UploadEmail? FolderUploadEmail { get; set; } [JsonProperty("parent")] - public MiniFolder Parent { get; set; } + public MiniFolder? Parent { get; set; } [JsonProperty("item_status")] - public string ItemStatus { get; set; } + public string? ItemStatus { get; set; } [JsonProperty("item_collection")] - public FileList ItemCollection { get; set; } + public FileList? ItemCollection { get; set; } } private class OrderEntry { [JsonProperty("by")] - public string By { get; set; } + public string? By { get; set; } [JsonProperty("direction")] - public string Direction { get; set; } + public string? Direction { get; set; } } private class ShortListResponse : FileList { [JsonProperty("order")] - public OrderEntry[] Order { get; set; } + public OrderEntry[]? Order { get; set; } } private class IDReference { [JsonProperty("id")] - public string ID { get; set; } + public string? ID { get; set; } } private class CreateItemRequest { [JsonProperty("name")] - public string Name { get; set; } + public string? Name { get; set; } [JsonProperty("parent")] - public IDReference Parent { get; set; } + public IDReference? Parent { get; set; } } private class ErrorResponse { [JsonProperty("type")] - public string Type { get; set; } + public string? Type { get; set; } [JsonProperty("status")] public int Status { get; set; } [JsonProperty("code")] - public string Code { get; set; } + public string? Code { get; set; } [JsonProperty("help_url")] - public string HelpUrl { get; set; } + public string? HelpUrl { get; set; } [JsonProperty("message")] - public string Message { get; set; } + public string? Message { get; set; } [JsonProperty("request_id")] - public string RequestId { get; set; } + public string? RequestId { get; set; } } } diff --git a/Duplicati/Library/Backend/Box/Strings.cs b/Duplicati/Library/Backend/Box/Strings.cs index 648dd9b8f..2a1fc9b22 100644 --- a/Duplicati/Library/Backend/Box/Strings.cs +++ b/Duplicati/Library/Backend/Box/Strings.cs @@ -17,16 +17,15 @@ // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. -using Duplicati.Library.Localization.Short; -namespace Duplicati.Library.Backend.Strings -{ - internal static class Box { - public static string Description { get { return LC.L(@"This backend can read and write data to Box.com. Allowed format is ""box://folder/subfolder""."); } } - public static string DisplayName { get { return LC.L(@"Box.com"); } } - public static string AuthidLong(string url) { return LC.L(@"The authorization token retrieved from {0}", url); } - public static string AuthidShort { get { return LC.L(@"The authorization code"); } } - public static string ReallydeleteLong { get { return LC.L(@"After deleting a file, it may end up in the trash folder where it will be deleted after a grace period. Use this command to force immediate removal of delete files."); } } - public static string ReallydeleteShort { get { return LC.L(@"Force delete files"); } } - } -} +// DEALINGS IN THE SOFTWARE. +using Duplicati.Library.Localization.Short; +namespace Duplicati.Library.Backend.Strings +{ + internal static class Box + { + public static string Description { get { return LC.L(@"This backend can read and write data to Box.com. Allowed format is ""box://folder/subfolder""."); } } + public static string DisplayName { get { return LC.L(@"Box.com"); } } + public static string ReallydeleteLong { get { return LC.L(@"After deleting a file, it may end up in the trash folder where it will be deleted after a grace period. Use this command to force immediate removal of delete files."); } } + public static string ReallydeleteShort { get { return LC.L(@"Force delete files"); } } + } +} diff --git a/Duplicati/Library/Backend/OAuthHelper/OAuthHelper.cs b/Duplicati/Library/Backend/OAuthHelper/OAuthHelper.cs index 5e68b9e76..f25d400de 100644 --- a/Duplicati/Library/Backend/OAuthHelper/OAuthHelper.cs +++ b/Duplicati/Library/Backend/OAuthHelper/OAuthHelper.cs @@ -1,60 +1,58 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Net; using Duplicati.Library.Utility; -using System.Collections.Generic; -using System.Web; namespace Duplicati.Library { - /// - /// Class for providing call-context access to http settings - /// - public static class OAuthContextSettings - { - /// - /// The struct wrapping the OAuth settings - /// - private struct OAuthSettings - { - /// - /// The server url - /// - public string ServerURL; - } + /// + /// Class for providing call-context access to http settings + /// + public static class OAuthContextSettings + { + /// + /// The struct wrapping the OAuth settings + /// + private struct OAuthSettings + { + /// + /// The server url + /// + public string ServerURL; + } - /// - /// Starts the session. - /// - /// The session. - /// The url to use for the server. - public static IDisposable StartSession(string serverurl) - { + /// + /// Starts the session. + /// + /// The session. + /// The url to use for the server. + public static IDisposable StartSession(string serverurl) + { return CallContextSettings.StartContext(new OAuthSettings { ServerURL = serverurl }); - } + } - /// - /// Gets the server URL to use for OAuth. - /// + /// + /// Gets the server URL to use for OAuth. + /// public static string ServerURL { get @@ -63,7 +61,7 @@ namespace Duplicati.Library return string.IsNullOrWhiteSpace(r) ? OAuthHelper.DUPLICATI_OAUTH_SERVICE : r; } } - } + } public class OAuthHelper : JSONWebHelper { @@ -73,11 +71,11 @@ namespace Duplicati.Library public const string DUPLICATI_OAUTH_SERVICE = "https://duplicati-oauth-handler.appspot.com/refresh"; - public static string OAUTH_LOGIN_URL(string modulename) + public static string OAUTH_LOGIN_URL(string modulename) { var u = new Library.Utility.Uri(OAuthContextSettings.ServerURL); var addr = u.SetPath("").SetQuery((u.Query ?? "") + (string.IsNullOrWhiteSpace(u.Query) ? "" : "&") + "type={0}"); - return string.Format(addr.ToString(), modulename); + return string.Format(addr.ToString(), modulename); } /// @@ -106,7 +104,7 @@ namespace Duplicati.Library OAuthLoginUrl = OAUTH_LOGIN_URL(servicename); if (string.IsNullOrEmpty(authid)) - throw new Duplicati.Library.Interface.UserInformationException(Strings.OAuthHelper.MissingAuthID(OAuthLoginUrl), "MissingAuthID"); + throw new Interface.UserInformationException(Strings.OAuthHelper.MissingAuthID(OAuthLoginUrl), "MissingAuthID"); } public T GetTokenResponse() @@ -129,7 +127,7 @@ namespace Duplicati.Library if (!noAuthorization && AutoAuthHeader && !string.Equals(OAuthContextSettings.ServerURL, url)) r.Headers["Authorization"] = string.Format("Bearer {0}", AccessToken); return r; - } + } public string AccessToken { @@ -142,7 +140,7 @@ namespace Duplicati.Library { var retries = 0; - while(true) + while (true) { try { diff --git a/Duplicati/Library/Utility/Options/AuthIdOptionsHelper.cs b/Duplicati/Library/Utility/Options/AuthIdOptionsHelper.cs new file mode 100644 index 000000000..ff0b1c42d --- /dev/null +++ b/Duplicati/Library/Utility/Options/AuthIdOptionsHelper.cs @@ -0,0 +1,78 @@ + + +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable +using System.Collections.Generic; +using Duplicati.Library.Interface; + +namespace Duplicati.Library.Utility.Options; + +/// +/// Helper class for unified use of authentication settings +/// +public static class AuthIdOptionsHelper +{ + /// + /// Gets the authentication options + /// + /// The URL to use for the token + /// An optional prefix for the options + /// The authentication options + public static CommandLineArgument[] GetOptions(string tokenurl, string? prefix = null) => + [ + new CommandLineArgument($"{prefix}authid", CommandLineArgument.ArgumentType.String, Strings.AuthIdSettingsHelper.AuthidShort, Strings.AuthIdSettingsHelper.AuthidLong(tokenurl)) + ]; + + /// + /// Parses the authentication options from a dictionary + /// + /// The dictionary to parse + /// An optional prefix for the options + /// The parsed authentication options + public static AuthIdOptions Parse(IReadOnlyDictionary options, string? prefix = null) + => new AuthIdOptions(options.GetValueOrDefault($"{prefix}authid")); + + /// + /// Structure to hold the authentication options + /// + /// The Auth ID + public sealed record AuthIdOptions(string? AuthId) + { + /// + /// Checks if the authid is set + /// + public bool IsValid() => !string.IsNullOrEmpty(AuthId); + + /// + /// Throws an exception if the username and password are not set + /// + /// The URL to use for the token + public AuthIdOptions RequireCredentials(string tokenurl) + { + if (!IsValid()) + throw new UserInformationException(Strings.AuthIdSettingsHelper.MissingAuthID(tokenurl), "MissingAuthID"); + return this; + } + } + +} diff --git a/Duplicati/Library/Utility/Options/AuthOptionsHelper.cs b/Duplicati/Library/Utility/Options/AuthOptionsHelper.cs index 9105c2667..ef3c01006 100644 --- a/Duplicati/Library/Utility/Options/AuthOptionsHelper.cs +++ b/Duplicati/Library/Utility/Options/AuthOptionsHelper.cs @@ -176,10 +176,12 @@ public static class AuthOptionsHelper /// /// Throws an exception if the username and password are not set /// - public void RequireCredentials() + public AuthOptions RequireCredentials() { if (!IsValid()) throw new UserInformationException(Strings.AuthSettingsHelper.UsernameAndPasswordRequired, "UsernameAndPasswordRequired"); + + return this; } } diff --git a/Duplicati/Library/Utility/Strings.cs b/Duplicati/Library/Utility/Strings.cs index 64e4e5dea..78f279ed1 100644 --- a/Duplicati/Library/Utility/Strings.cs +++ b/Duplicati/Library/Utility/Strings.cs @@ -95,4 +95,12 @@ namespace Duplicati.Library.Utility.Strings public static string DescriptionAuthUsernameShort { get { return LC.L(@"Supply the username used to connect to the server"); } } public static string UsernameAndPasswordRequired { get { return LC.L(@"Authentication requires both a username and a password"); } } } + + internal static class AuthIdSettingsHelper + { + public static string AuthidLong(string url) { return LC.L(@"The authorization token retrieved from {0}", url); } + public static string AuthidShort { get { return LC.L(@"The authorization code"); } } + public static string MissingAuthID(string url) { return LC.L(@"You need an AuthID to use this destination. You can get it from: {0}", url); } + + } } From be8de849fbc969f5c21de86d72ea71c467a16be0 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 20 Mar 2025 12:02:09 +0100 Subject: [PATCH 08/48] Mapping timeouts to correct exceptions --- Duplicati/Library/Utility/Utility.cs | 69 +++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index bedf8ac8a..938290d49 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -1669,6 +1669,53 @@ namespace Duplicati.Library.Utility public static string FormatInvariant(this FormattableString formattable) => formattable.ToString(CultureInfo.InvariantCulture); + /// + /// Performs the function with an additional timeout + /// + /// The timeout to observe + /// The cancellation token + /// The function to invoke + /// The task + public static async Task WithTimeout(TimeSpan timeout, CancellationToken token, Action func) + { + using var cts = CancellationTokenSource.CreateLinkedTokenSource(token); + cts.CancelAfter(timeout); + try + { + await Task.Run(() => func(cts.Token), cts.Token).ConfigureAwait(false); + } + catch (TaskCanceledException) + { + if (cts.IsCancellationRequested) + throw new TimeoutException(); + throw; + } + } + + /// + /// Performs the function with an additional timeout + /// + /// The timeout to observe + /// The cancellation token + /// The function to invoke + /// The task + public static async Task WithTimeout(TimeSpan timeout, CancellationToken token, Func func) + { + using var cts = CancellationTokenSource.CreateLinkedTokenSource(token); + cts.CancelAfter(timeout); + try + { + return await Task.Run(() => func(cts.Token), cts.Token).ConfigureAwait(false); + } + catch (TaskCanceledException) + { + if (cts.IsCancellationRequested) + throw new TimeoutException(); + throw; + } + + } + /// /// Performs the function with an additional timeout /// @@ -1680,7 +1727,16 @@ namespace Duplicati.Library.Utility { using var cts = CancellationTokenSource.CreateLinkedTokenSource(token); cts.CancelAfter(timeout); - await func(cts.Token); + try + { + await func(cts.Token); + } + catch (TaskCanceledException) + { + if (cts.IsCancellationRequested) + throw new TimeoutException(); + throw; + } } /// @@ -1695,7 +1751,16 @@ namespace Duplicati.Library.Utility { using var cts = CancellationTokenSource.CreateLinkedTokenSource(token); cts.CancelAfter(timeout); - return await func(cts.Token); + try + { + return await func(cts.Token); + } + catch (TaskCanceledException) + { + if (cts.IsCancellationRequested) + throw new TimeoutException(); + throw; + } } /// From 1465d7c36817f85e22aad790588ca2b6ef4ece4e Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 20 Mar 2025 12:02:25 +0100 Subject: [PATCH 09/48] Updated CIFS --- Duplicati/Library/Backend/CIFS/CIFS.cs | 92 ++++--- .../CIFS/Model/SMBConnectionParameters.cs | 100 ++------ .../Backend/CIFS/SMBShareConnection.cs | 234 +++++++++++------- Duplicati/Library/Backend/CIFS/Strings.cs | 4 - .../Builtin/BackendSourceFileEntry.cs | 4 +- 5 files changed, 217 insertions(+), 217 deletions(-) diff --git a/Duplicati/Library/Backend/CIFS/CIFS.cs b/Duplicati/Library/Backend/CIFS/CIFS.cs index 53ec3264e..04603ec74 100644 --- a/Duplicati/Library/Backend/CIFS/CIFS.cs +++ b/Duplicati/Library/Backend/CIFS/CIFS.cs @@ -19,6 +19,8 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +#nullable enable + using System; using Duplicati.Library.Interface; using System.Collections.Generic; @@ -32,7 +34,7 @@ using SMBLibrary; using Duplicati.Library.SourceProvider; using System.Runtime.CompilerServices; using Duplicati.Library.Common.IO; -using System.Runtime.CompilerServices; +using Duplicati.Library.Utility.Options; namespace Duplicati.Library.Backend; @@ -69,12 +71,17 @@ public class CIFSBackend : IStreamingBackend, IFolderEnabledBackend /// /// Cache of parsed connection parameters /// - private SMBConnectionParameters _connectionParameters; + private readonly SMBConnectionParameters _connectionParameters; + + /// + /// Timeout options to be used in the backend + /// + private readonly TimeoutOptionsHelper.Timeouts _timeouts; /// /// Shared connection between all methods to avoid re-authentication /// - private SMBShareConnection _sharedConnection; + private SMBShareConnection? _sharedConnection; /// /// Read buffer size for SMB operations (will be capped automatically by SMB negotiated values) @@ -96,16 +103,6 @@ public class CIFSBackend : IStreamingBackend, IFolderEnabledBackend /// private const string AUTH_DOMAIN_OPTION = "auth-domain"; - /// - /// Username for authentication - /// - private const string AUTH_USERNAME_OPTION = "auth-username"; - - /// - /// Password for authentication - /// - private const string AUTH_PASSWORD_OPTION = "auth-password"; - /// /// Defines the default transport to be used in CIFS connection /// @@ -125,6 +122,9 @@ public class CIFSBackend : IStreamingBackend, IFolderEnabledBackend /// public CIFSBackend() { + _DnsName = null!; + _connectionParameters = null!; + _timeouts = null!; } /// @@ -132,7 +132,7 @@ public class CIFSBackend : IStreamingBackend, IFolderEnabledBackend /// /// URL in Duplicati Uri format /// options to be used in the backend - public CIFSBackend(string url, Dictionary options) + public CIFSBackend(string url, Dictionary options) { if (string.IsNullOrEmpty(url)) throw new ArgumentNullException(nameof(url)); @@ -146,37 +146,39 @@ public class CIFSBackend : IStreamingBackend, IFolderEnabledBackend var input = uri.Path.TrimEnd('/'); var slashIndex = input.IndexOf('/'); // Find first slash to separate server and share if present. - options.TryGetValue(AUTH_USERNAME_OPTION, out string authUsername); - options.TryGetValue(AUTH_PASSWORD_OPTION, out string authPassword); - options.TryGetValue(AUTH_DOMAIN_OPTION, out string authDomain); - options.TryGetValue(TRANSPORT_OPTION, out string transport); + var auth = AuthOptionsHelper.Parse(options, uri); + var authDomain = options.GetValueOrDefault(AUTH_DOMAIN_OPTION); + var transport = options.GetValueOrDefault(TRANSPORT_OPTION); int? readBufferSize = null, writeBufferSize = null; - options.TryGetValue(READ_BUFFER_SIZE_OPTION, out string readBufferSizeConfig); - if (!string.IsNullOrWhiteSpace(readBufferSizeConfig)) readBufferSize = Int32.TryParse(readBufferSizeConfig, out int value) ? value : null; + var readBufferSizeConfig = options.GetValueOrDefault(READ_BUFFER_SIZE_OPTION); + if (!string.IsNullOrWhiteSpace(readBufferSizeConfig)) + readBufferSize = int.TryParse(readBufferSizeConfig, out int value) ? value : null; - options.TryGetValue(WRITE_BUFFER_SIZE_OPTION, out string writeBufferSizeConfig); - if (!string.IsNullOrWhiteSpace(writeBufferSizeConfig)) writeBufferSize = Int32.TryParse(readBufferSizeConfig, out int value) ? value : null; + var writeBufferSizeConfig = options.GetValueOrDefault(WRITE_BUFFER_SIZE_OPTION); + if (!string.IsNullOrWhiteSpace(writeBufferSizeConfig)) + writeBufferSize = int.TryParse(readBufferSizeConfig, out int value) ? value : null; // Normalize to 10KB minimum buffers size readBufferSize = readBufferSize < 1024 * 10 ? null : readBufferSize; writeBufferSize = writeBufferSize < 1024 * 10 ? null : writeBufferSize; - SMBTransportType transportType = _transportMap.TryGetValue( + var transportType = _transportMap.TryGetValue( string.IsNullOrEmpty(transport) ? DEFAULT_TRANSPORT : transport.ToLower(), out SMBTransportType type) ? type : throw new UserInformationException($"Transport must be one of: {string.Join(", ", _transportMap.Keys)}", "CIFSConfig"); + _timeouts = TimeoutOptionsHelper.Parse(options); _connectionParameters = new SMBConnectionParameters( uri.Host, transportType, slashIndex >= 0 ? input[..slashIndex] : input, slashIndex >= 0 ? input[(slashIndex + 1)..] : "", authDomain, - authUsername, - authPassword, + auth.Username, + auth.Password, readBufferSize, writeBufferSize ); @@ -186,14 +188,14 @@ public class CIFSBackend : IStreamingBackend, IFolderEnabledBackend /// Implementation of interface property to return supported command parameters /// public IList SupportedCommands => - new List([ - new CommandLineArgument(AUTH_PASSWORD_OPTION, CommandLineArgument.ArgumentType.Password, Strings.CIFSBackend.DescriptionAuthPasswordShort, Strings.CIFSBackend.DescriptionAuthPasswordLong), - new CommandLineArgument(AUTH_USERNAME_OPTION, CommandLineArgument.ArgumentType.String, Strings.CIFSBackend.DescriptionAuthUsernameShort, Strings.CIFSBackend.DescriptionAuthUsernameLong), + [ + .. AuthOptionsHelper.GetOptions(), new CommandLineArgument(AUTH_DOMAIN_OPTION, CommandLineArgument.ArgumentType.String, Strings.CIFSBackend.DescriptionAuthDomainShort, Strings.CIFSBackend.DescriptionAuthDomainLong), new CommandLineArgument(TRANSPORT_OPTION, CommandLineArgument.ArgumentType.Enumeration, Strings.Options.TransportShort, Strings.Options.TransportLong, DEFAULT_TRANSPORT, null, _transportMap.Keys.ToArray()), new CommandLineArgument(READ_BUFFER_SIZE_OPTION, CommandLineArgument.ArgumentType.String, Strings.Options.DescriptionReadBufferSizeShort, Strings.Options.DescriptionReadBufferSizeLong), - new CommandLineArgument(WRITE_BUFFER_SIZE_OPTION, CommandLineArgument.ArgumentType.String, Strings.Options.DescriptionWriteBufferSizeShort, Strings.Options.DescriptionWriteBufferSizeLong) - ]); + new CommandLineArgument(WRITE_BUFFER_SIZE_OPTION, CommandLineArgument.ArgumentType.String, Strings.Options.DescriptionWriteBufferSizeShort, Strings.Options.DescriptionWriteBufferSizeLong), + .. TimeoutOptionsHelper.GetOptions() + ]; /// /// Implementation of interface method for listing remote folder contents @@ -201,7 +203,8 @@ public class CIFSBackend : IStreamingBackend, IFolderEnabledBackend /// List of IFileEntry with directory listing result public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancellationToken) { - foreach (var v in await GetConnection().ListAsync(_connectionParameters.Path, cancellationToken).ConfigureAwait(false)) + var con = await GetConnectionAsync(cancellationToken).ConfigureAwait(false); + foreach (var v in await con.ListAsync(_connectionParameters.Path, cancellationToken).ConfigureAwait(false)) yield return v; } @@ -230,7 +233,8 @@ public class CIFSBackend : IStreamingBackend, IFolderEnabledBackend /// Exceptions arising from either code execution public async Task PutAsync(string remotename, Stream input, CancellationToken cancellationToken) { - await GetConnection().PutAsync(remotename, input, cancellationToken).ConfigureAwait(false); + var con = await GetConnectionAsync(cancellationToken).ConfigureAwait(false); + await con.PutAsync(remotename, input, cancellationToken).ConfigureAwait(false); } /// @@ -259,7 +263,8 @@ public class CIFSBackend : IStreamingBackend, IFolderEnabledBackend /// Exceptions arising from either code execution or FileMissingException public async Task GetAsync(string remotename, Stream output, CancellationToken cancellationToken) { - await GetConnection().GetAsync(remotename, output, cancellationToken).ConfigureAwait(false); + var con = await GetConnectionAsync(cancellationToken).ConfigureAwait(false); + await con.GetAsync(remotename, output, cancellationToken).ConfigureAwait(false); } /// @@ -272,7 +277,8 @@ public class CIFSBackend : IStreamingBackend, IFolderEnabledBackend /// Exceptions arising from either code execution or business logic errors public async Task DeleteAsync(string remotename, CancellationToken cancellationToken) { - await GetConnection().DeleteAsync(remotename, cancellationToken).ConfigureAwait(false); + var con = await GetConnectionAsync(cancellationToken).ConfigureAwait(false); + await con.DeleteAsync(remotename, cancellationToken).ConfigureAwait(false); } /// @@ -291,7 +297,8 @@ public class CIFSBackend : IStreamingBackend, IFolderEnabledBackend public async Task TestAsync(CancellationToken cancellationToken) { // This will throw an exception if the folder is missing - await GetConnection().ListAsync(_connectionParameters.Path, cancellationToken).ConfigureAwait(false); + var con = await GetConnectionAsync(cancellationToken).ConfigureAwait(false); + await con.ListAsync(_connectionParameters.Path, cancellationToken).ConfigureAwait(false); } /// @@ -308,7 +315,8 @@ public class CIFSBackend : IStreamingBackend, IFolderEnabledBackend if (pathParts == null || pathParts.Length == 0) return; - await GetConnection().CreateFolderAsync(_connectionParameters.Path, cancellationToken) + var con = await GetConnectionAsync(cancellationToken).ConfigureAwait(false); + await con.CreateFolderAsync(_connectionParameters.Path ?? "", cancellationToken) .ConfigureAwait(false); } @@ -316,7 +324,8 @@ public class CIFSBackend : IStreamingBackend, IFolderEnabledBackend /// Gets or creates a shared SMB connection /// /// An SMB connection that can be used for file operations - private SMBShareConnection GetConnection() => _sharedConnection ??= new SMBShareConnection(_connectionParameters); + private async Task GetConnectionAsync(CancellationToken cancellationToken) + => _sharedConnection ??= await SMBShareConnection.CreateAsync(_connectionParameters, _timeouts, cancellationToken).ConfigureAwait(false); @@ -337,18 +346,19 @@ public class CIFSBackend : IStreamingBackend, IFolderEnabledBackend } /// - public async IAsyncEnumerable ListAsync(string path, [EnumeratorCancellation] CancellationToken cancellationToken) + public async IAsyncEnumerable ListAsync(string? path, [EnumeratorCancellation] CancellationToken cancellationToken) { var sourcePath = _connectionParameters.Path; if (!string.IsNullOrWhiteSpace(sourcePath)) sourcePath = Util.AppendDirSeparator(sourcePath, "/"); - foreach (var v in await GetConnection().ListAsync(sourcePath + BackendSourceFileEntry.NormalizePathTo(path, '/'), cancellationToken).ConfigureAwait(false)) + var con = await GetConnectionAsync(cancellationToken).ConfigureAwait(false); + foreach (var v in await con.ListAsync(sourcePath + BackendSourceFileEntry.NormalizePathTo(path, '/'), cancellationToken).ConfigureAwait(false)) if (v.Name != "." && v.Name != "..") yield return v; } /// - public Task GetEntryAsync(string path, CancellationToken cancellationToken) - => Task.FromResult(null); + public Task GetEntryAsync(string path, CancellationToken cancellationToken) + => Task.FromResult(null); } \ No newline at end of file diff --git a/Duplicati/Library/Backend/CIFS/Model/SMBConnectionParameters.cs b/Duplicati/Library/Backend/CIFS/Model/SMBConnectionParameters.cs index e2f687625..8fe1770b9 100644 --- a/Duplicati/Library/Backend/CIFS/Model/SMBConnectionParameters.cs +++ b/Duplicati/Library/Backend/CIFS/Model/SMBConnectionParameters.cs @@ -26,84 +26,22 @@ namespace Duplicati.Library.Backend.CIFS.Model; /// /// Connection parameters for establishing an SMB connection. /// -public record SMBConnectionParameters -{ - /// - /// The name or IP address of the SMB server - /// - public string ServerName { get; init; } - - /// - /// The transport protocol type used for SMB communication - /// - public SMBTransportType TransportType { get; init; } - - /// - /// The name of the network share to connect to - /// - public string ShareName { get; init; } - - /// - /// The path within the share to access - /// - public string Path { get; init; } - - /// - /// The authentication domain name - /// - public string AuthDomain { get; init; } - - /// - /// The username for authentication - /// - public string AuthUser { get; init; } - - /// - /// The password for authentication - /// - public string AuthPassword { get; init; } - - /// - /// Write buffer size for SMB operations (will be capped automatically by SMB negotiated values) - /// - public int? WriteBufferSize { get; init; } - - /// - /// Read buffer size for SMB operations (will be capped automatically by SMB negotiated values) - /// - public int? ReadBufferSize { get; init; } - - /// - /// Creates a new instance of SMB connection parameters - /// - /// The name or IP address of the SMB server - /// The transport protocol type used for SMB communication - /// The name of the network share to connect to - /// The path within the share to access - /// The authentication domain name - /// The username for authentication - /// The password for authentication - /// Read buffer size for SMB operations (will be capped automatically by SMB negotiated values) - /// Write buffer size for SMB operations (will be capped automatically by SMB negotiated values) - public SMBConnectionParameters( - string serverName, - SMBTransportType transportType, - string shareName, - string path, - string authDomain, - string authUser, - string authPassword, - int? readBufferSize, - int? writeBufferSize) - { - ServerName = serverName; - TransportType = transportType; - ShareName = shareName; - Path = path; - AuthDomain = authDomain; - AuthUser = authUser; - AuthPassword = authPassword; - ReadBufferSize = readBufferSize; - WriteBufferSize = writeBufferSize; - } -} \ No newline at end of file +/// The name or IP address of the SMB server +/// The transport protocol type used for SMB communication +/// The name of the network share to connect to +/// The path within the share to access +/// The authentication domain name +/// The username for authentication +/// The password for authentication +/// Read buffer size for SMB operations (will be capped automatically by SMB negotiated values) +/// Write buffer size for SMB operations (will be capped automatically by SMB negotiated values) +public sealed record SMBConnectionParameters( + string ServerName, + SMBTransportType TransportType, + string ShareName, + string Path, + string AuthDomain, + string AuthUser, + string AuthPassword, + int? ReadBufferSize, + int? WriteBufferSize); \ No newline at end of file diff --git a/Duplicati/Library/Backend/CIFS/SMBShareConnection.cs b/Duplicati/Library/Backend/CIFS/SMBShareConnection.cs index 47e23b742..0226811cd 100644 --- a/Duplicati/Library/Backend/CIFS/SMBShareConnection.cs +++ b/Duplicati/Library/Backend/CIFS/SMBShareConnection.cs @@ -31,6 +31,8 @@ using Duplicati.Library.Backend.CIFS.Model; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using Duplicati.Library.Localization.Short; +using Duplicati.Library.Utility; +using Duplicati.Library.Utility.Options; using SMBLibrary; using SMBLibrary.Client; using FileAttributes = SMBLibrary.FileAttributes; @@ -46,7 +48,7 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable /// /// SMBConnection client /// - private readonly SMB2Client _smb2Client = new(); + private readonly SMB2Client _smb2Client; /// /// Shared fileStore object. @@ -58,11 +60,19 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable /// private readonly SMBConnectionParameters _connectionParameters; + /// + /// The timeouts to use for operations. + /// + private readonly TimeoutOptionsHelper.Timeouts _timeouts; + /// /// The semaphore to ensure that only one operation is performed at a time. /// private readonly SemaphoreSlim _semaphore = new(1, 1); + /// + /// Flag to indicate if the object has been disposed. + /// private bool _disposed; /// @@ -71,23 +81,51 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable /// It throws specific exceptions for connection and authentication failures. /// /// Connection Parameters - /// Exception to be displayed to user - public SMBShareConnection(SMBConnectionParameters connectionParameters) + /// Timeouts to use for operations + private SMBShareConnection(SMB2Client client, ISMBFileStore fileStore, SMBConnectionParameters connectionParameters, TimeoutOptionsHelper.Timeouts timeouts) { _connectionParameters = connectionParameters; + _timeouts = timeouts; + _smbFileStore = fileStore; + _smb2Client = client; + } - if (!_smb2Client.Connect(connectionParameters.ServerName, connectionParameters.TransportType)) + /// + /// Creates a new SMBShareConnection object asynchronously. + /// + /// The connection parameters + /// The timeouts to use for operations + /// The cancellation token + /// The SMBShareConnection object + /// Exception to be displayed to user + public static async Task CreateAsync(SMBConnectionParameters connectionParameters, TimeoutOptionsHelper.Timeouts timeouts, CancellationToken cancellationToken) + { + var client = new SMB2Client(); + var connected = await Utility.Utility.WithTimeout(timeouts.ShortTimeout, cancellationToken, _ => + client.Connect(connectionParameters.ServerName, connectionParameters.TransportType) + ).ConfigureAwait(false); + + if (!connected) throw new UserInformationException($"{LC.L("Failed to connect to server")} {connectionParameters.ServerName}", "ConnectionError"); - var status = _smb2Client.Login(connectionParameters.AuthDomain ?? "", connectionParameters.AuthUser ?? "", connectionParameters.AuthPassword ?? ""); + var status = await Utility.Utility.WithTimeout(timeouts.ShortTimeout, cancellationToken, _ => + client.Login(connectionParameters.AuthDomain ?? "", connectionParameters.AuthUser ?? "", connectionParameters.AuthPassword ?? "") + ).ConfigureAwait(false); if (status != NTStatus.STATUS_SUCCESS) throw new UserInformationException($"{LC.L("Failed to authenticate to server")} {connectionParameters.ServerName} with status {status}", "ConnectionError"); - _smbFileStore = _smb2Client.TreeConnect(connectionParameters.ShareName, out status); + (var res, status) = await Utility.Utility.WithTimeout(timeouts.ShortTimeout, cancellationToken, _ => + { + var res = client.TreeConnect(connectionParameters.ShareName, out status); + return (res, status); + } + ).ConfigureAwait(false); - if (status != NTStatus.STATUS_SUCCESS) + if (status != NTStatus.STATUS_SUCCESS || res == null) throw new UserInformationException($"{LC.L("Failed to connect to share")} {connectionParameters.ShareName} with status {status}", "ConnectionError"); + + return new SMBShareConnection(client, res, connectionParameters, timeouts); } /// @@ -101,33 +139,33 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable await _semaphore.WaitAsync(cancellationToken); try { - NTStatus status; - object fileHandle; - FileStatus fileStatus; - status = _smbFileStore.CreateFile(out fileHandle, out fileStatus, NormalizeSlashes(Path.Combine(_connectionParameters.Path, fileName)), - AccessMask.GENERIC_WRITE | AccessMask.DELETE | AccessMask.SYNCHRONIZE, - FileAttributes.Normal, - ShareAccess.None, - CreateDisposition.FILE_OPEN, - CreateOptions.FILE_NON_DIRECTORY_FILE | CreateOptions.FILE_SYNCHRONOUS_IO_ALERT, - null); - - if (status == NTStatus.STATUS_OBJECT_NAME_NOT_FOUND) - throw new FileMissingException(); - - if (status == NTStatus.STATUS_SUCCESS) + await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, _ => { - var fileDispositionInformation = new FileDispositionInformation + var status = _smbFileStore.CreateFile(out var fileHandle, out var fileStatus, NormalizeSlashes(Path.Combine(_connectionParameters.Path, fileName)), + AccessMask.GENERIC_WRITE | AccessMask.DELETE | AccessMask.SYNCHRONIZE, + FileAttributes.Normal, + ShareAccess.None, + CreateDisposition.FILE_OPEN, + CreateOptions.FILE_NON_DIRECTORY_FILE | CreateOptions.FILE_SYNCHRONOUS_IO_ALERT, + null); + + if (status == NTStatus.STATUS_OBJECT_NAME_NOT_FOUND) + throw new FileMissingException(); + + if (status == NTStatus.STATUS_SUCCESS) { - DeletePending = true - }; - status = _smbFileStore.SetFileInformation(fileHandle, fileDispositionInformation); - if (status != NTStatus.STATUS_SUCCESS) - throw new UserInformationException($"{LC.L("Failed to delete file on DeleteAsync")} with status {status}", "DeleteFileError"); - status = _smbFileStore.CloseFile(fileHandle); - if (status != NTStatus.STATUS_SUCCESS) - throw new UserInformationException($"{LC.L("Failed to close file on DeleteAsync")} with status {status}", "CloseFileError"); - } + var fileDispositionInformation = new FileDispositionInformation + { + DeletePending = true + }; + status = _smbFileStore.SetFileInformation(fileHandle, fileDispositionInformation); + if (status != NTStatus.STATUS_SUCCESS) + throw new UserInformationException($"{LC.L("Failed to delete file on DeleteAsync")} with status {status}", "DeleteFileError"); + status = _smbFileStore.CloseFile(fileHandle); + if (status != NTStatus.STATUS_SUCCESS) + throw new UserInformationException($"{LC.L("Failed to close file on DeleteAsync")} with status {status}", "CloseFileError"); + } + }).ConfigureAwait(false); } finally { @@ -147,10 +185,10 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable try { // Normalize path separators to forward slashes and trim any trailing separators - string linuxNormalizedPath = path.Replace('/', '\\').TrimEnd('\\'); - string currentPath = ""; + var linuxNormalizedPath = path.Replace('/', '\\').TrimEnd('\\'); + var currentPath = ""; - foreach (string part in linuxNormalizedPath.Split('\\', StringSplitOptions.RemoveEmptyEntries)) + foreach (var part in linuxNormalizedPath.Split('\\', StringSplitOptions.RemoveEmptyEntries)) { if (string.IsNullOrWhiteSpace(part) || part == ".") continue; @@ -159,20 +197,23 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable object? fileHandle = null; try { - NTStatus status = _smbFileStore.CreateFile( - out fileHandle, - out FileStatus fileStatus, - currentPath, - AccessMask.GENERIC_WRITE | AccessMask.SYNCHRONIZE, - FileAttributes.Normal, - ShareAccess.None, - CreateDisposition.FILE_CREATE, - CreateOptions.FILE_DIRECTORY_FILE | CreateOptions.FILE_SYNCHRONOUS_IO_ALERT, - null - ); - if (status != NTStatus.STATUS_SUCCESS && - status != NTStatus.STATUS_OBJECT_NAME_COLLISION) // Ignore if directory already exists - throw new UserInformationException($"{LC.L("Failed to create directory")} {currentPath} with status{status}", "CreateDirectoryError"); + await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, _ => + { + var status = _smbFileStore.CreateFile( + out fileHandle, + out var fileStatus, + currentPath, + AccessMask.GENERIC_WRITE | AccessMask.SYNCHRONIZE, + FileAttributes.Normal, + ShareAccess.None, + CreateDisposition.FILE_CREATE, + CreateOptions.FILE_DIRECTORY_FILE | CreateOptions.FILE_SYNCHRONOUS_IO_ALERT, + null + ); + if (status != NTStatus.STATUS_SUCCESS && + status != NTStatus.STATUS_OBJECT_NAME_COLLISION) // Ignore if directory already exists + throw new UserInformationException($"{LC.L("Failed to create directory")} {currentPath} with status{status}", "CreateDirectoryError"); + }).ConfigureAwait(false); } finally { @@ -207,32 +248,37 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable { try { - var status = _smbFileStore.CreateFile( - out directoryHandle, - out fileStatus, - NormalizeSlashes(path), - AccessMask.GENERIC_READ, - FileAttributes.Directory, - ShareAccess.Read | ShareAccess.Write, - CreateDisposition.FILE_OPEN, - CreateOptions.FILE_DIRECTORY_FILE, - null); + var fileList = await Utility.Utility.WithTimeout(_timeouts.ListTimeout, cancellationToken, _ => + { + var status = _smbFileStore.CreateFile( + out directoryHandle, + out fileStatus, + NormalizeSlashes(path), + AccessMask.GENERIC_READ, + FileAttributes.Directory, + ShareAccess.Read | ShareAccess.Write, + CreateDisposition.FILE_OPEN, + CreateOptions.FILE_DIRECTORY_FILE, + null); - if (status != NTStatus.STATUS_SUCCESS && fileStatus != FileStatus.FILE_OPENED) - if (status == NTStatus.STATUS_OBJECT_PATH_NOT_FOUND || status == NTStatus.STATUS_OBJECT_NAME_NOT_FOUND) - throw new FolderMissingException(); - else - throw new UserInformationException($"{LC.L("Failed to open directory")} {NormalizeSlashes(path)} with status {status}", "DirectoryOpenError"); + if (status != NTStatus.STATUS_SUCCESS && fileStatus != FileStatus.FILE_OPENED) + if (status == NTStatus.STATUS_OBJECT_PATH_NOT_FOUND || status == NTStatus.STATUS_OBJECT_NAME_NOT_FOUND) + throw new FolderMissingException(); + else + throw new UserInformationException($"{LC.L("Failed to open directory")} {NormalizeSlashes(path)} with status {status}", "DirectoryOpenError"); - List fileList; - status = _smbFileStore.QueryDirectory( - out fileList, - directoryHandle, - "*", - FileInformationClass.FileDirectoryInformation); + List fileList; + status = _smbFileStore.QueryDirectory( + out fileList, + directoryHandle, + "*", + FileInformationClass.FileDirectoryInformation); - if (status != NTStatus.STATUS_NO_MORE_FILES) - throw new UserInformationException($"{LC.L("Failed to query directory contents")} with status {status}", "DirectoryQueryError"); + if (status != NTStatus.STATUS_NO_MORE_FILES) + throw new UserInformationException($"{LC.L("Failed to query directory contents")} with status {status}", "DirectoryQueryError"); + + return fileList; + }).ConfigureAwait(false); return [ @@ -279,18 +325,22 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable try { - object? fileHandle; - FileStatus fileStatus; - NTStatus status = _smbFileStore.CreateFile(out fileHandle, out fileStatus, + (var status, var fileStatus, var fileHandle) = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, _ => + { + var status = _smbFileStore.CreateFile(out var fileHandle, out var fileStatus, NormalizeSlashes(Path.Combine(_connectionParameters.Path, filename)), AccessMask.GENERIC_READ | AccessMask.SYNCHRONIZE, FileAttributes.Normal, ShareAccess.Read, CreateDisposition.FILE_OPEN, CreateOptions.FILE_NON_DIRECTORY_FILE | CreateOptions.FILE_SYNCHRONOUS_IO_ALERT, null); + return (status, fileStatus, fileHandle); + } + ).ConfigureAwait(false); if (status == NTStatus.STATUS_SUCCESS || fileStatus != FileStatus.FILE_DOES_NOT_EXIST) { byte[] data; long bytesRead = 0; + using var timeoutStream = destinationStream.ObserveWriteTimeout(_timeouts.ReadWriteTimeout, false); while (!cancellationToken.IsCancellationRequested) { // Use the provided read buffer size if set, otherwise use the protocol negotiated maximum. Never exceed the negotiated maximum. @@ -303,10 +353,10 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable break; bytesRead += data.Length; - await destinationStream.WriteAsync(data, 0, data.Length, cancellationToken); + await timeoutStream.WriteAsync(data, 0, data.Length, cancellationToken); } - await destinationStream.FlushAsync(cancellationToken); + await timeoutStream.FlushAsync(cancellationToken); if (fileHandle != null) { @@ -321,7 +371,7 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable throw new FolderMissingException(); else throw new UserInformationException( - $"{LC.L("Failed to open file with error")} {filename} with status {status.ToString()}", + $"{LC.L("Failed to open file with error")} {filename} with status {status}", "FileOpenError"); } finally @@ -343,25 +393,29 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable await _semaphore.WaitAsync(cancellationToken); try { - object fileHandle; - FileStatus fileStatus; - NTStatus status = _smbFileStore.CreateFile(out fileHandle, out fileStatus, - NormalizeSlashes(Path.Combine(_connectionParameters.Path, filename)), - AccessMask.GENERIC_WRITE | AccessMask.SYNCHRONIZE, - FileAttributes.Normal, ShareAccess.None, - CreateDisposition.FILE_SUPERSEDE, - CreateOptions.FILE_NON_DIRECTORY_FILE | CreateOptions.FILE_SYNCHRONOUS_IO_ALERT, - null); + (var status, var fileStatus, var fileHandle) = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, _ => + { + var status = _smbFileStore.CreateFile(out var fileHandle, out var fileStatus, + NormalizeSlashes(Path.Combine(_connectionParameters.Path, filename)), + AccessMask.GENERIC_WRITE | AccessMask.SYNCHRONIZE, + FileAttributes.Normal, ShareAccess.None, + CreateDisposition.FILE_SUPERSEDE, + CreateOptions.FILE_NON_DIRECTORY_FILE | CreateOptions.FILE_SYNCHRONOUS_IO_ALERT, + null); + return (status, fileStatus, fileHandle); + }).ConfigureAwait(false); + if (status == NTStatus.STATUS_SUCCESS) { // Use the provided write buffer size if set, otherwise use the protocol negotiated maximum. Never exceed the negotiated maximum. - byte[] buffer = new byte[Math.Min(_connectionParameters.WriteBufferSize ?? (int)_smb2Client.MaxWriteSize, _smb2Client.MaxWriteSize)]; + var buffer = new byte[Math.Min(_connectionParameters.WriteBufferSize ?? (int)_smb2Client.MaxWriteSize, _smb2Client.MaxWriteSize)]; int bytesRead; int numberOfBytesWritten; int offset = 0; - while (!cancellationToken.IsCancellationRequested && sourceStream.Position < sourceStream.Length) + using var timeoutStream = sourceStream.ObserveReadTimeout(_timeouts.ReadWriteTimeout, false); + while (!cancellationToken.IsCancellationRequested && timeoutStream.Position < timeoutStream.Length) { - bytesRead = await sourceStream.ReadAsync(buffer, cancellationToken); + bytesRead = await timeoutStream.ReadAsync(buffer, cancellationToken); if (bytesRead == 0) break; status = _smbFileStore.WriteFile(out numberOfBytesWritten, fileHandle, offset, buffer.Take(bytesRead).ToArray()); diff --git a/Duplicati/Library/Backend/CIFS/Strings.cs b/Duplicati/Library/Backend/CIFS/Strings.cs index 53699aa9a..2457dcf9d 100644 --- a/Duplicati/Library/Backend/CIFS/Strings.cs +++ b/Duplicati/Library/Backend/CIFS/Strings.cs @@ -26,10 +26,6 @@ namespace Duplicati.Library.Backend.Strings internal static class CIFSBackend { - public static string DescriptionAuthPasswordLong => LC.L(@"The password used to connect to the server. This may also be supplied as the environment variable ""AUTH_PASSWORD""."); - public static string DescriptionAuthPasswordShort => LC.L(@"Supply the password used to connect to the server"); - public static string DescriptionAuthUsernameLong => LC.L(@"The username used to connect to the server. This may also be supplied as the environment variable ""AUTH_USERNAME""."); - public static string DescriptionAuthUsernameShort => LC.L(@"Supply the username used to connect to the server"); public static string DescriptionAuthDomainLong => LC.L(@"The domain used to connect to the server. This may also be supplied as the environment variable ""AUTH_DOMAIN""."); public static string DescriptionAuthDomainShort => LC.L(@"Supply the domain used to connect to the server"); public static string Description => LC.L(@"This backend can read and write data to CIFS/SMB destinations. Allowed format is ""cifs://server/share""."); diff --git a/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs b/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs index b9e134014..da1a31ef0 100644 --- a/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs +++ b/Duplicati/Library/SourceProvider/Builtin/BackendSourceFileEntry.cs @@ -19,6 +19,7 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +using System.Diagnostics.CodeAnalysis; using System.IO.Pipelines; using System.Runtime.CompilerServices; using Duplicati.Library.Common.IO; @@ -322,7 +323,8 @@ public class BackendSourceFileEntry(BackendSourceProvider parent, string path, b /// /// The path to normalize /// The normalized path - public static string NormalizePathTo(string path, char separator) + [return: NotNullIfNotNull("path")] + public static string? NormalizePathTo(string? path, char separator) { if (string.IsNullOrEmpty(path)) return path; From 3fd663e923153bee42bde45053cc6a4ccdfe7437 Mon Sep 17 00:00:00 2001 From: "Marcelo C." Date: Thu, 20 Mar 2025 15:23:31 -0300 Subject: [PATCH 10/48] Terminal Box.com replacement of deprecated WebClient/OAuth Creating of the new OAuthHelper for HttpClient. Removal of manual assembly of the multipart post. --- Duplicati/Library/Backend/Box/BoxBackend.cs | 403 +++++------------- .../Backend/Box/Model/CreateItemRequest.cs | 9 + .../Backend/Box/Model/ErrorResponse.cs | 13 + .../Library/Backend/Box/Model/FileEntity.cs | 20 + .../Library/Backend/Box/Model/FileList.cs | 11 + .../Library/Backend/Box/Model/FolderList.cs | 9 + .../Library/Backend/Box/Model/IDReference.cs | 8 + .../Backend/Box/Model/ListFolderResponse.cs | 21 + .../Library/Backend/Box/Model/MiniFolder.cs | 11 + .../Library/Backend/Box/Model/MiniUser.cs | 10 + .../Library/Backend/Box/Model/OrderEntry.cs | 9 + .../Backend/Box/Model/ShortListResponse.cs | 8 + .../Library/Backend/Box/Model/UploadEmail.cs | 9 + .../OAuthHelper/JSONWebHelperHttpClient.cs | 74 +++- .../OAuthHelper/OAuthHelperHttpClient.cs | 192 +++++++++ .../OAuthHelper/OAuthServiceResponse.cs | 33 ++ 16 files changed, 534 insertions(+), 306 deletions(-) create mode 100644 Duplicati/Library/Backend/Box/Model/CreateItemRequest.cs create mode 100644 Duplicati/Library/Backend/Box/Model/ErrorResponse.cs create mode 100644 Duplicati/Library/Backend/Box/Model/FileEntity.cs create mode 100644 Duplicati/Library/Backend/Box/Model/FileList.cs create mode 100644 Duplicati/Library/Backend/Box/Model/FolderList.cs create mode 100644 Duplicati/Library/Backend/Box/Model/IDReference.cs create mode 100644 Duplicati/Library/Backend/Box/Model/ListFolderResponse.cs create mode 100644 Duplicati/Library/Backend/Box/Model/MiniFolder.cs create mode 100644 Duplicati/Library/Backend/Box/Model/MiniUser.cs create mode 100644 Duplicati/Library/Backend/Box/Model/OrderEntry.cs create mode 100644 Duplicati/Library/Backend/Box/Model/ShortListResponse.cs create mode 100644 Duplicati/Library/Backend/Box/Model/UploadEmail.cs create mode 100644 Duplicati/Library/Backend/OAuthHelper/OAuthHelperHttpClient.cs create mode 100644 Duplicati/Library/Backend/OAuthHelper/OAuthServiceResponse.cs diff --git a/Duplicati/Library/Backend/Box/BoxBackend.cs b/Duplicati/Library/Backend/Box/BoxBackend.cs index d256255d9..4e49abcc7 100644 --- a/Duplicati/Library/Backend/Box/BoxBackend.cs +++ b/Duplicati/Library/Backend/Box/BoxBackend.cs @@ -19,24 +19,27 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using Duplicati.Library.Common.IO; -using Duplicati.Library.Interface; -using Newtonsoft.Json; using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Net; +using System.Net.Http; +using System.Net.Http.Json; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; +using Duplicati.Library.Common.IO; +using Duplicati.Library.Interface; +using Duplicati.Library.Logging; +using Newtonsoft.Json; +using Uri = Duplicati.Library.Utility.Uri; namespace Duplicati.Library.Backend.Box { - // ReSharper disable once ClassNeverInstantiated.Global - // This class is instantiated dynamically in the BackendLoader. - public class BoxBackend : IBackend, IStreamingBackend + public class BoxBackend : IStreamingBackend { - private static readonly string LOGTAG = Logging.Log.LogTagFromType(); + private static readonly string LOGTAG = Log.LogTagFromType(); private const string AUTHID_OPTION = "authid"; private const string REALLY_DELETE_OPTION = "box-delete-from-trash"; @@ -46,14 +49,14 @@ namespace Duplicati.Library.Backend.Box private const int PAGE_SIZE = 200; - private readonly BoxHelper m_oauth; - private readonly string m_path; - private readonly bool m_deleteFromTrash; + private readonly BoxHelper _oAuthHelper; + private readonly string _path; + private readonly bool _deleteFromTrash; - private string m_currentfolder; - private readonly Dictionary m_filecache = new Dictionary(); + private string _currentFolder; + private readonly Dictionary _fileCache = new(); - private class BoxHelper : OAuthHelper + private class BoxHelper : OAuthHelperHttpClient { public BoxHelper(string authid) : base(authid, "box.com") @@ -61,108 +64,88 @@ namespace Duplicati.Library.Backend.Box AutoAuthHeader = true; } - protected override void ParseException(Exception ex) + public override void AttemptParseAndThrowException(Exception ex, HttpResponseMessage responseContext = null) { - Exception newex = null; - try - { - if (ex is WebException exception && exception.Response is HttpWebResponse hs) - { - string rawdata = null; - using (var rs = Library.Utility.AsyncHttpRequest.TrySetTimeout(hs.GetResponseStream())) - using (var sr = new System.IO.StreamReader(rs)) - rawdata = sr.ReadToEnd(); + if (ex is not HttpRequestException || responseContext == null) + return; + + if (ex is HttpRequestException && responseContext is { StatusCode: HttpStatusCode.TooManyRequests }) + throw new TooManyRequestException(responseContext.Headers.RetryAfter); - if (string.IsNullOrWhiteSpace(rawdata)) - return; - - newex = new Exception("Raw message: " + rawdata); - - var msg = JsonConvert.DeserializeObject(rawdata); - newex = new Exception(string.Format("{0} - {1}: {2}", msg.Status, msg.Code, msg.Message)); - - /*if (msg.ContextInfo != null && msg.ContextInfo.Length > 0) - newex = new Exception(string.Format("{0} - {1}: {2}{3}{4}", msg.Status, msg.Code, msg.Message, Environment.NewLine, string.Join("; ", from n in msg.ContextInfo select n.Message))); - */ - } - } - catch (Exception ex2) - { - Library.Logging.Log.WriteWarningMessage(LOGTAG, "BoxErrorParser", ex2, "Failed to parse error from Box"); - } - - if (newex != null) - throw newex; + using var stream = responseContext.Content.ReadAsStream(); + using var reader = new StreamReader(stream); + var rawData = reader.ReadToEnd(); + var errorResponse = JsonConvert.DeserializeObject(rawData); + throw new UserInformationException($"Box.com ErrorResponse: {errorResponse.Status} - {errorResponse.Code}: {errorResponse.Message}", "box.com"); } } - // ReSharper disable once UnusedMember.Global - // This constructor is needed by the BackendLoader. public BoxBackend() { } - - // ReSharper disable once UnusedMember.Global - // This constructor is needed by the BackendLoader. + public BoxBackend(string url, Dictionary options) { - var uri = new Utility.Uri(url); + var uri = new Uri(url); - m_path = Util.AppendDirSeparator(uri.HostAndPath, "/"); + _path = Util.AppendDirSeparator(uri.HostAndPath, "/"); string authid = null; - if (options.ContainsKey(AUTHID_OPTION)) - authid = options[AUTHID_OPTION]; + if (options.TryGetValue(AUTHID_OPTION, out var option)) + authid = option; - m_deleteFromTrash = Library.Utility.Utility.ParseBoolOption(options, REALLY_DELETE_OPTION); + _deleteFromTrash = Utility.Utility.ParseBoolOption(options, REALLY_DELETE_OPTION); - m_oauth = new BoxHelper(authid); + _oAuthHelper = new BoxHelper(authid); } private async Task GetCurrentFolderWithCacheAsync(CancellationToken cancelToken) { - if (m_currentfolder == null) + if (_currentFolder == null) await GetCurrentFolderAsync(false, cancelToken).ConfigureAwait(false); - return m_currentfolder; + return _currentFolder; } private async Task GetCurrentFolderAsync(bool create, CancellationToken cancelToken) { var parentid = "0"; - foreach (var p in m_path.Split(new string[] { "/" }, StringSplitOptions.RemoveEmptyEntries)) + foreach (var p in _path.Split(["/"], StringSplitOptions.RemoveEmptyEntries)) { - var el = (MiniFolder)await PagedFileListResponse(parentid, true, cancelToken).FirstOrDefaultAsync(x => x.Name == p).ConfigureAwait(false); + var el = (MiniFolder)await PagedFileListResponse(parentid, true, cancelToken).FirstOrDefaultAsync(x => x.Name == p, cancellationToken: cancelToken).ConfigureAwait(false); if (el == null) { if (!create) throw new FolderMissingException(); - - el = await m_oauth.PostAndGetJSONDataAsync( - string.Format("{0}/folders", BOX_API_URL), - cancelToken, - new CreateItemRequest() { Name = p, Parent = new IDReference() { ID = parentid } } + + el = await _oAuthHelper.PostAndGetJsonDataAsync( + $"{BOX_API_URL}/folders", + new CreateItemRequest + { + Name = p, Parent = new IDReference { ID = parentid } + }, + cancelToken ).ConfigureAwait(false); } parentid = el.ID; } - m_currentfolder = parentid; + _currentFolder = parentid; } - private async Task GetFileIDAsync(string name, CancellationToken cancelToken) + private async Task GetFileIdAsync(string name, CancellationToken cancelToken) { - if (m_filecache.ContainsKey(name)) - return m_filecache[name]; + if (_fileCache.TryGetValue(name, out var async)) + return async; // Make sure we enumerate this, otherwise the m_filecache is empty. var currentFolder = await GetCurrentFolderWithCacheAsync(cancelToken).ConfigureAwait(false); - await PagedFileListResponse(currentFolder, false, cancelToken).LastOrDefaultAsync().ConfigureAwait(false); + await PagedFileListResponse(currentFolder, false, cancelToken).LastOrDefaultAsync(cancellationToken: cancelToken).ConfigureAwait(false); - if (m_filecache.ContainsKey(name)) - return m_filecache[name]; + if (_fileCache.TryGetValue(name, out var idAsync)) + return idAsync; throw new FileMissingException(); } @@ -173,11 +156,11 @@ namespace Duplicati.Library.Backend.Box var done = false; if (!onlyfolders) - m_filecache.Clear(); + _fileCache.Clear(); do { - var resp = await m_oauth.GetJSONDataAsync($"{BOX_API_URL}/folders/{parentid}/items?limit={PAGE_SIZE}&offset={offset}&fields=name,size,modified_at", cancelToken).ConfigureAwait(false); + var resp = await _oAuthHelper.GetJsonDataAsync($"{BOX_API_URL}/folders/{parentid}/items?limit={PAGE_SIZE}&offset={offset}&fields=name,size,modified_at", cancelToken).ConfigureAwait(false); if (resp.Entries == null || resp.Entries.Length == 0) break; @@ -189,13 +172,11 @@ namespace Duplicati.Library.Backend.Box done = true; break; } - else - { - if (!onlyfolders && f.Type == "file") - m_filecache[f.Name] = f.ID; - yield return f; - } + if (!onlyfolders && f.Type == "file") + _fileCache[f.Name] = f.ID; + + yield return f; } offset = offset + PAGE_SIZE; @@ -206,62 +187,59 @@ namespace Duplicati.Library.Backend.Box } while (!done); } - #region IStreamingBackend implementation - - public async Task PutAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) + public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken) { var currentFolder = await GetCurrentFolderWithCacheAsync(cancelToken).ConfigureAwait(false); - var createreq = new CreateItemRequest() + var createreq = new CreateItemRequest { Name = remotename, - Parent = new IDReference() + Parent = new IDReference { ID = currentFolder } }; - if (m_filecache.Count == 0) - await PagedFileListResponse(currentFolder, false, cancelToken).LastOrDefaultAsync().ConfigureAwait(false); + if (_fileCache.Count == 0) + await PagedFileListResponse(currentFolder, false, cancelToken).LastOrDefaultAsync(cancelToken).ConfigureAwait(false); - var existing = m_filecache.ContainsKey(remotename); + var existing = _fileCache.ContainsKey(remotename); + + var multipartForm = new MultipartFormDataContent("----DuplicatiFormBoundary" + Guid.NewGuid().ToString("N")); try { string url; - var items = new List(2); if (existing) - url = $"{BOX_UPLOAD_URL}/{m_filecache[remotename]}/content"; + url = $"{BOX_UPLOAD_URL}/{_fileCache[remotename]}/content"; else { url = $"{BOX_UPLOAD_URL}/content"; - items.Add(new MultipartItem(createreq, "attributes")); + multipartForm.Add(JsonContent.Create(createreq),"attributes"); } - - items.Add(new MultipartItem(stream, "file", remotename)); - - var res = (await m_oauth.PostMultipartAndGetJSONDataAsync(url, null, cancelToken, items.ToArray())).Entries.First(); - m_filecache[remotename] = res.ID; + + multipartForm.Add(new StreamContent(stream), "file", remotename); + + var res = (await _oAuthHelper.PostMultipartAndGetJsonDataAsync(url, cancelToken, multipartForm)).Entries.First(); + _fileCache[remotename] = res.ID; } catch { - m_filecache.Clear(); + _fileCache.Clear(); throw; } } - public async Task GetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) + public async Task GetAsync(string remotename, Stream stream, CancellationToken cancelToken) { - var fileId = await GetFileIDAsync(remotename, cancelToken).ConfigureAwait(false); - using (var resp = await m_oauth.GetResponseAsync(string.Format("{0}/files/{1}/content", BOX_API_URL, fileId), cancelToken).ConfigureAwait(false)) - using (var rs = Duplicati.Library.Utility.AsyncHttpRequest.TrySetTimeout(resp.GetResponseStream())) - await Library.Utility.Utility.CopyStreamAsync(rs, stream, cancelToken).ConfigureAwait(false); + var fileId = await GetFileIdAsync(remotename, cancelToken).ConfigureAwait(false); + using var request = _oAuthHelper.CreateRequest($"{BOX_API_URL}/files/{fileId}/content"); + using var resp = await _oAuthHelper.GetResponseAsync(request, cancelToken).ConfigureAwait(false); + resp.EnsureSuccessStatusCode(); + await using var responseStream = await resp.Content.ReadAsStreamAsync(cancelToken).ConfigureAwait(false); + await Utility.Utility.CopyStreamAsync(responseStream, stream, cancelToken).ConfigureAwait(false); } - #endregion - - #region IBackend implementation - public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { var currentFolder = await GetCurrentFolderWithCacheAsync(cancelToken).ConfigureAwait(false); @@ -271,33 +249,38 @@ namespace Duplicati.Library.Backend.Box public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) { - using (System.IO.FileStream fs = System.IO.File.OpenRead(filename)) - await PutAsync(remotename, fs, cancelToken); + await using FileStream fs = File.OpenRead(filename); + await PutAsync(remotename, fs, cancelToken).ConfigureAwait(false); } public async Task GetAsync(string remotename, string filename, CancellationToken cancelToken) { - using (System.IO.FileStream fs = System.IO.File.Create(filename)) - await GetAsync(remotename, fs, cancelToken).ConfigureAwait(false); + await using FileStream fs = File.Create(filename); + await GetAsync(remotename, fs, cancelToken).ConfigureAwait(false); } public async Task DeleteAsync(string remotename, CancellationToken cancelToken) { - var fileid = await GetFileIDAsync(remotename, cancelToken).ConfigureAwait(false); + var fileId = await GetFileIdAsync(remotename, cancelToken).ConfigureAwait(false); try { - using (var r = await m_oauth.GetResponseAsync(string.Format("{0}/files/{1}", BOX_API_URL, fileid), cancelToken, null, "DELETE").ConfigureAwait(false)) + using (var request = _oAuthHelper.CreateRequest($"{BOX_API_URL}/files/{fileId}","DELETE")) + using (var r = await _oAuthHelper.GetResponseAsync(request, cancelToken).ConfigureAwait(false)) { } - if (m_deleteFromTrash) - using (var r = await m_oauth.GetResponseAsync(string.Format("{0}/files/{1}/trash", BOX_API_URL, fileid), cancelToken, null, "DELETE").ConfigureAwait(false)) + if (_deleteFromTrash) + { + using var request = _oAuthHelper.CreateRequest($"{BOX_API_URL}/files/{fileId}/trash","DELETE"); + using (var r = await _oAuthHelper + .GetResponseAsync(request, cancelToken).ConfigureAwait(false)) { } + } } catch { - m_filecache.Clear(); + _fileCache.Clear(); throw; } } @@ -310,202 +293,26 @@ namespace Duplicati.Library.Backend.Box return GetCurrentFolderAsync(true, cancellationToken); } - public string DisplayName - { - get - { - return Strings.Box.DisplayName; - } - } + public string DisplayName => Strings.Box.DisplayName; - public string ProtocolKey - { - get - { - return "box"; - } - } + public string ProtocolKey => "box"; - public IList SupportedCommands - { - get - { - return new List(new ICommandLineArgument[] { - new CommandLineArgument(AUTHID_OPTION, CommandLineArgument.ArgumentType.Password, Strings.Box.AuthidShort, Strings.Box.AuthidLong(OAuthHelper.OAUTH_LOGIN_URL("box.com"))), - new CommandLineArgument(REALLY_DELETE_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.Box.ReallydeleteShort, Strings.Box.ReallydeleteLong), - }); - } - } + public IList SupportedCommands => + new List([ + new CommandLineArgument(AUTHID_OPTION, CommandLineArgument.ArgumentType.Password, Strings.Box.AuthidShort, Strings.Box.AuthidLong(OAuthHelper.OAUTH_LOGIN_URL("box.com"))), + new CommandLineArgument(REALLY_DELETE_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.Box.ReallydeleteShort, Strings.Box.ReallydeleteLong) + ]); - public string Description - { - get - { - return Strings.Box.Description; - } - } + public string Description => Strings.Box.Description; - public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new string[] { + public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new[] { new System.Uri(BOX_API_URL).Host, new System.Uri(BOX_UPLOAD_URL).Host }); - #endregion - - #region IDisposable implementation - public void Dispose() { } - - #endregion - - private class MiniUser : IDReference - { - [JsonProperty("type")] - public string Type { get; set; } - [JsonProperty("name")] - public string Name { get; set; } - [JsonProperty("login")] - public string Login { get; set; } - } - - private class MiniFolder : IDReference - { - [JsonProperty("type")] - public string Type { get; set; } - [JsonProperty("name")] - public string Name { get; set; } - [JsonProperty("etag")] - public string ETag { get; set; } - [JsonProperty("sequence_id")] - public string SequenceID { get; set; } - } - - private class FileEntity : MiniFolder - { - public FileEntity() { Size = -1; } - - [JsonProperty("sha1")] - public string SHA1 { get; set; } - - [JsonProperty("size", NullValueHandling = NullValueHandling.Ignore)] - public long Size { get; set; } - [JsonProperty("modified_at", NullValueHandling = NullValueHandling.Ignore)] - public DateTime ModifiedAt { get; set; } - } - - private class FolderList - { - [JsonProperty("total_count")] - public long TotalCount { get; set; } - [JsonProperty("entries")] - public MiniFolder[] Entries { get; set; } - } - - private class FileList - { - [JsonProperty("total_count")] - public long TotalCount { get; set; } - [JsonProperty("entries")] - public FileEntity[] Entries { get; set; } - [JsonProperty("offset")] - public long Offset { get; set; } - [JsonProperty("limit")] - public long Limit { get; set; } - } - - private class UploadEmail - { - [JsonProperty("access")] - public string Access { get; set; } - [JsonProperty("email")] - public string Email { get; set; } - } - - private class ListFolderResponse : MiniFolder - { - [JsonProperty("created_at")] - public DateTime CreatedAt { get; set; } - [JsonProperty("modified_at")] - public DateTime ModifiedAt { get; set; } - [JsonProperty("description")] - public string Description { get; set; } - [JsonProperty("size")] - public long Size { get; set; } - - [JsonProperty("path_collection")] - public FolderList PathCollection { get; set; } - - [JsonProperty("created_by")] - public MiniUser CreatedBy { get; set; } - [JsonProperty("modified_by")] - public MiniUser ModifiedBy { get; set; } - [JsonProperty("owned_by")] - public MiniUser OwnedBy { get; set; } - - [JsonProperty("shared_link")] - public MiniUser SharedLink { get; set; } - - [JsonProperty("folder_upload_email")] - public UploadEmail FolderUploadEmail { get; set; } - - [JsonProperty("parent")] - public MiniFolder Parent { get; set; } - - [JsonProperty("item_status")] - public string ItemStatus { get; set; } - - [JsonProperty("item_collection")] - public FileList ItemCollection { get; set; } - - } - - private class OrderEntry - { - [JsonProperty("by")] - public string By { get; set; } - [JsonProperty("direction")] - public string Direction { get; set; } - } - - private class ShortListResponse : FileList - { - [JsonProperty("order")] - public OrderEntry[] Order { get; set; } - - } - - private class IDReference - { - [JsonProperty("id")] - public string ID { get; set; } - } - - private class CreateItemRequest - { - [JsonProperty("name")] - public string Name { get; set; } - [JsonProperty("parent")] - public IDReference Parent { get; set; } - } - - private class ErrorResponse - { - [JsonProperty("type")] - public string Type { get; set; } - [JsonProperty("status")] - public int Status { get; set; } - [JsonProperty("code")] - public string Code { get; set; } - [JsonProperty("help_url")] - public string HelpUrl { get; set; } - [JsonProperty("message")] - public string Message { get; set; } - [JsonProperty("request_id")] - public string RequestId { get; set; } - - } } } diff --git a/Duplicati/Library/Backend/Box/Model/CreateItemRequest.cs b/Duplicati/Library/Backend/Box/Model/CreateItemRequest.cs new file mode 100644 index 000000000..b84ecd32b --- /dev/null +++ b/Duplicati/Library/Backend/Box/Model/CreateItemRequest.cs @@ -0,0 +1,9 @@ +using Newtonsoft.Json; + +namespace Duplicati.Library.Backend.Box; + +public class CreateItemRequest +{ + [JsonProperty("name")] public string Name { get; set; } + [JsonProperty("parent")] public IDReference Parent { get; set; } +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/Box/Model/ErrorResponse.cs b/Duplicati/Library/Backend/Box/Model/ErrorResponse.cs new file mode 100644 index 000000000..627780c8f --- /dev/null +++ b/Duplicati/Library/Backend/Box/Model/ErrorResponse.cs @@ -0,0 +1,13 @@ +using Newtonsoft.Json; + +namespace Duplicati.Library.Backend.Box; + +public class ErrorResponse +{ + [JsonProperty("type")] public string Type { get; set; } + [JsonProperty("status")] public int Status { get; set; } + [JsonProperty("code")] public string Code { get; set; } + [JsonProperty("help_url")] public string HelpUrl { get; set; } + [JsonProperty("message")] public string Message { get; set; } + [JsonProperty("request_id")] public string RequestId { get; set; } +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/Box/Model/FileEntity.cs b/Duplicati/Library/Backend/Box/Model/FileEntity.cs new file mode 100644 index 000000000..55e222724 --- /dev/null +++ b/Duplicati/Library/Backend/Box/Model/FileEntity.cs @@ -0,0 +1,20 @@ +using System; +using Newtonsoft.Json; + +namespace Duplicati.Library.Backend.Box; + +public class FileEntity : MiniFolder +{ + public FileEntity() + { + Size = -1; + } + + [JsonProperty("sha1")] public string SHA1 { get; set; } + + [JsonProperty("size", NullValueHandling = NullValueHandling.Ignore)] + public long Size { get; set; } + + [JsonProperty("modified_at", NullValueHandling = NullValueHandling.Ignore)] + public DateTime ModifiedAt { get; set; } +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/Box/Model/FileList.cs b/Duplicati/Library/Backend/Box/Model/FileList.cs new file mode 100644 index 000000000..999aab376 --- /dev/null +++ b/Duplicati/Library/Backend/Box/Model/FileList.cs @@ -0,0 +1,11 @@ +using Newtonsoft.Json; + +namespace Duplicati.Library.Backend.Box; + +public class FileList +{ + [JsonProperty("total_count")] public long TotalCount { get; set; } + [JsonProperty("entries")] public FileEntity[] Entries { get; set; } + [JsonProperty("offset")] public long Offset { get; set; } + [JsonProperty("limit")] public long Limit { get; set; } +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/Box/Model/FolderList.cs b/Duplicati/Library/Backend/Box/Model/FolderList.cs new file mode 100644 index 000000000..028188322 --- /dev/null +++ b/Duplicati/Library/Backend/Box/Model/FolderList.cs @@ -0,0 +1,9 @@ +using Newtonsoft.Json; + +namespace Duplicati.Library.Backend.Box; + +public class FolderList +{ + [JsonProperty("total_count")] public long TotalCount { get; set; } + [JsonProperty("entries")] public MiniFolder[] Entries { get; set; } +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/Box/Model/IDReference.cs b/Duplicati/Library/Backend/Box/Model/IDReference.cs new file mode 100644 index 000000000..29ba6cf14 --- /dev/null +++ b/Duplicati/Library/Backend/Box/Model/IDReference.cs @@ -0,0 +1,8 @@ +using Newtonsoft.Json; + +namespace Duplicati.Library.Backend.Box; + +public class IDReference +{ + [JsonProperty("id")] public string ID { get; set; } +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/Box/Model/ListFolderResponse.cs b/Duplicati/Library/Backend/Box/Model/ListFolderResponse.cs new file mode 100644 index 000000000..395bdaaba --- /dev/null +++ b/Duplicati/Library/Backend/Box/Model/ListFolderResponse.cs @@ -0,0 +1,21 @@ +using System; +using Newtonsoft.Json; + +namespace Duplicati.Library.Backend.Box; + +public class ListFolderResponse : MiniFolder +{ + [JsonProperty("created_at")] public DateTime CreatedAt { get; set; } + [JsonProperty("modified_at")] public DateTime ModifiedAt { get; set; } + [JsonProperty("description")] public string Description { get; set; } + [JsonProperty("size")] public long Size { get; set; } + [JsonProperty("path_collection")] public FolderList PathCollection { get; set; } + [JsonProperty("created_by")] public MiniUser CreatedBy { get; set; } + [JsonProperty("modified_by")] public MiniUser ModifiedBy { get; set; } + [JsonProperty("owned_by")] public MiniUser OwnedBy { get; set; } + [JsonProperty("shared_link")] public MiniUser SharedLink { get; set; } + [JsonProperty("folder_upload_email")] public UploadEmail FolderUploadEmail { get; set; } + [JsonProperty("parent")] public MiniFolder Parent { get; set; } + [JsonProperty("item_status")] public string ItemStatus { get; set; } + [JsonProperty("item_collection")] public FileList ItemCollection { get; set; } +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/Box/Model/MiniFolder.cs b/Duplicati/Library/Backend/Box/Model/MiniFolder.cs new file mode 100644 index 000000000..26363b542 --- /dev/null +++ b/Duplicati/Library/Backend/Box/Model/MiniFolder.cs @@ -0,0 +1,11 @@ +using Newtonsoft.Json; + +namespace Duplicati.Library.Backend.Box; + +public class MiniFolder : IDReference +{ + [JsonProperty("type")] public string Type { get; set; } + [JsonProperty("name")] public string Name { get; set; } + [JsonProperty("etag")] public string ETag { get; set; } + [JsonProperty("sequence_id")] public string SequenceID { get; set; } +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/Box/Model/MiniUser.cs b/Duplicati/Library/Backend/Box/Model/MiniUser.cs new file mode 100644 index 000000000..cc3af66ef --- /dev/null +++ b/Duplicati/Library/Backend/Box/Model/MiniUser.cs @@ -0,0 +1,10 @@ +using Newtonsoft.Json; + +namespace Duplicati.Library.Backend.Box; + +public class MiniUser : IDReference +{ + [JsonProperty("type")] public string Type { get; set; } + [JsonProperty("name")] public string Name { get; set; } + [JsonProperty("login")] public string Login { get; set; } +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/Box/Model/OrderEntry.cs b/Duplicati/Library/Backend/Box/Model/OrderEntry.cs new file mode 100644 index 000000000..5a573fb9c --- /dev/null +++ b/Duplicati/Library/Backend/Box/Model/OrderEntry.cs @@ -0,0 +1,9 @@ +using Newtonsoft.Json; + +namespace Duplicati.Library.Backend.Box; + +public class OrderEntry +{ + [JsonProperty("by")] public string By { get; set; } + [JsonProperty("direction")] public string Direction { get; set; } +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/Box/Model/ShortListResponse.cs b/Duplicati/Library/Backend/Box/Model/ShortListResponse.cs new file mode 100644 index 000000000..4f8db34fd --- /dev/null +++ b/Duplicati/Library/Backend/Box/Model/ShortListResponse.cs @@ -0,0 +1,8 @@ +using Newtonsoft.Json; + +namespace Duplicati.Library.Backend.Box; + +public class ShortListResponse : FileList +{ + [JsonProperty("order")] public OrderEntry[] Order { get; set; } +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/Box/Model/UploadEmail.cs b/Duplicati/Library/Backend/Box/Model/UploadEmail.cs new file mode 100644 index 000000000..de24408d9 --- /dev/null +++ b/Duplicati/Library/Backend/Box/Model/UploadEmail.cs @@ -0,0 +1,9 @@ +using Newtonsoft.Json; + +namespace Duplicati.Library.Backend.Box; + +public class UploadEmail +{ + [JsonProperty("access")] public string Access { get; set; } + [JsonProperty("email")] public string Email { get; set; } +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/OAuthHelper/JSONWebHelperHttpClient.cs b/Duplicati/Library/Backend/OAuthHelper/JSONWebHelperHttpClient.cs index cfcc46b76..214a6122d 100644 --- a/Duplicati/Library/Backend/OAuthHelper/JSONWebHelperHttpClient.cs +++ b/Duplicati/Library/Backend/OAuthHelper/JSONWebHelperHttpClient.cs @@ -38,7 +38,7 @@ namespace Duplicati.Library; public class JsonWebHelperHttpClient(HttpClient httpClient) { /// - /// HttpClient reference + /// HttpClient reference for inheritors /// protected readonly HttpClient _httpClient = httpClient; @@ -62,7 +62,36 @@ public class JsonWebHelperHttpClient(HttpClient httpClient) return request; } + + /// + /// Performs a multipart post and parses the response as JSON + /// + /// The parsed JSON item. + /// The url to post to. + /// Token to cancel the operation. + /// The multipart items. + /// The return type parameter. + public virtual async Task PostMultipartAndGetJsonDataAsync(string url, CancellationToken cancellationToken, MultipartContent parts) + { + using var response = await PostMultipartAsync(url, cancellationToken, parts).ConfigureAwait(false); + return ReadJsonResponse(response); + } + + /// + /// Performs a multipart post + /// + /// The response. + /// The url to post to. + /// Token to cancel the operation. + /// The multipart items. + protected virtual async Task PostMultipartAsync(string url, CancellationToken cancellationToken, MultipartContent parts) + { + using var req = CreateRequest(url, "POST"); + req.Content = parts; + return await _httpClient.SendAsync(req, cancellationToken).ConfigureAwait(false); + } + /// /// Execute Get request and return response and deserializes JSON response into the given type /// @@ -72,7 +101,7 @@ public class JsonWebHelperHttpClient(HttpClient httpClient) /// Destination Type protected virtual async Task GetJsonDataAsync(string url, CancellationToken cancellationToken, Action setup = null) { - var req = CreateRequest(url); + using var req = CreateRequest(url); if (setup != null) setup(req); @@ -103,6 +132,26 @@ public class JsonWebHelperHttpClient(HttpClient httpClient) } ).ConfigureAwait(false); } + + /// + /// Executes a web request and json-deserializes the results as the specified type + /// + /// The deserialized JSON data. + /// The remote URL + /// Token to cancel the operation. + /// The type of data to return. + public virtual async Task GetJsonDataAsync(string url, CancellationToken cancellationToken) + { + + return await GetJsonDataAsync( + url, + cancellationToken, + request => + { + request.Method = HttpMethod.Get; + } + ).ConfigureAwait(false); + } /// /// Reads the JSON response from the server and deserializes it into the given type @@ -113,7 +162,7 @@ public class JsonWebHelperHttpClient(HttpClient httpClient) protected virtual async Task ReadJsonResponseAsync(HttpRequestMessage req, CancellationToken cancellationToken) { using var resp = await GetResponseAsync(req, cancellationToken).ConfigureAwait(false); - return await ReadJsonResponsAsync(resp, cancellationToken).ConfigureAwait(false); + return await ReadJsonResponseAsync(resp, cancellationToken).ConfigureAwait(false); } /// @@ -151,7 +200,7 @@ public class JsonWebHelperHttpClient(HttpClient httpClient) /// Type to cast to /// /// Exception when failing to deserialize the JSON to the Type - protected virtual async Task ReadJsonResponsAsync(HttpResponseMessage response, CancellationToken cancellationToken) + protected virtual async Task ReadJsonResponseAsync(HttpResponseMessage response, CancellationToken cancellationToken) { await using var rs = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); await using var ps = new StreamPeekReader(rs); @@ -171,13 +220,22 @@ public class JsonWebHelperHttpClient(HttpClient httpClient) } } + /// + /// Use this method to register an exception handler, + /// which can throw another, more meaningful exception + /// + public virtual async Task AttemptParseAndThrowExceptionAsync(Exception ex, HttpResponseMessage responseContext = null, CancellationToken cancellationToken = default) + { + + } + /// /// Use this method to register an exception handler, /// which can throw another, more meaningful exception /// public virtual void AttemptParseAndThrowException(Exception ex, HttpResponseMessage responseContext = null) { - + } /// @@ -186,13 +244,14 @@ public class JsonWebHelperHttpClient(HttpClient httpClient) /// Request object /// Cancellation Token /// - private async Task GetResponseAsync(HttpRequestMessage req, CancellationToken cancellationToken) + public async Task GetResponseAsync(HttpRequestMessage req, CancellationToken cancellationToken) { HttpResponseMessage response = null; try { response = await _httpClient.SendAsync(req, cancellationToken).ConfigureAwait(false); - + + response.EnsureSuccessStatusCode(); return response; } catch (Exception ex) @@ -277,6 +336,5 @@ public class JsonWebHelperHttpClient(HttpClient httpClient) return source.Read(buffer, offset, count) + br; } - } } \ No newline at end of file diff --git a/Duplicati/Library/Backend/OAuthHelper/OAuthHelperHttpClient.cs b/Duplicati/Library/Backend/OAuthHelper/OAuthHelperHttpClient.cs new file mode 100644 index 000000000..83bf40618 --- /dev/null +++ b/Duplicati/Library/Backend/OAuthHelper/OAuthHelperHttpClient.cs @@ -0,0 +1,192 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using System; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Threading; +using Duplicati.Library.Utility; + +namespace Duplicati.Library; + +public class OAuthHelperHttpClient : JsonWebHelperHttpClient +{ + private string _Token; + private string _Authid; + private DateTime _mTokenExpires = DateTime.UtcNow; + private string OAuthLoginUrl { get; } + + /// + /// Timeout for authentication requests + /// + private const int AUTHENTICATION_TIMEOUT_SECONDS = 25; + + private const int MAX_AUTHORIZATION_RETRIES = 5; + + private static string OAUTH_LOGIN_URL(string modulename) + { + var u = new Utility.Uri(OAuthContextSettings.ServerURL); + var addr = u.SetPath("") + .SetQuery((u.Query ?? "") + (string.IsNullOrWhiteSpace(u.Query) ? "" : "&") + "type={0}"); + return string.Format(addr.ToString(), modulename); + } + + /// + /// Set to true to automatically add the Authorization header to requests + /// + public bool AutoAuthHeader { get; set; } + + /// + /// Set to true if the provider does not use refresh tokens, but only access tokens + /// + public bool AccessTokenOnly { get; set; } + + /// + /// If true (the default), when a v1 authid is being used it will be swapped + /// with a v2 authid, when the OAuth service returns one (which it dypically + /// does after a provider token refresh has been performed). Some providers + /// are not compatible with v2 authid, tyically because they generate a new + /// refresh token with every access token refresh and invalidates the old. + /// If the oauth service still returns a v2 authid for such a provider, + /// set this property to false to make Duplicati ignore it. + /// + public bool AutoV2 { get; set; } = true; + + public OAuthHelperHttpClient(string authid, string servicename, HttpClient httpClient = null, string useragent = null) + : base(httpClient ?? HttpClientHelper.CreateClient()) + { + _Authid = authid; + OAuthLoginUrl = OAUTH_LOGIN_URL(servicename); + + if (string.IsNullOrEmpty(authid)) + throw new Interface.UserInformationException( + Strings.OAuthHelper.MissingAuthID(OAuthLoginUrl), "MissingAuthID"); + } + + private HttpRequestMessage CreateRequest(string url, string method, bool noAuthorization) + { + var request = base.CreateRequest(url, method); + if (!noAuthorization && AutoAuthHeader && !string.Equals(OAuthContextSettings.ServerURL, url)) request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", AccessToken); + return request; + } + + public override HttpRequestMessage CreateRequest(string url, string method = null) + { + return CreateRequest(url, method, false); + } + + private string AccessToken + { + get + { + if (AccessTokenOnly) + return _Authid; + + if (_Token == null || _mTokenExpires < DateTime.UtcNow) + { + var retries = 0; + + while (true) + { + HttpResponseMessage response = null; + try + { + using var request = base.CreateRequest(OAuthContextSettings.ServerURL); + + request.Headers.TryAddWithoutValidation("X-AuthID", _Authid); + + using var timeoutToken = new CancellationTokenSource(); + timeoutToken.CancelAfter(TimeSpan.FromSeconds(AUTHENTICATION_TIMEOUT_SECONDS)); + + response = _httpClient.Send(request, timeoutToken.Token); + response.EnsureSuccessStatusCode(); + + var res = ReadJsonResponse(response); + + _mTokenExpires = DateTime.UtcNow.AddSeconds(res.expires - 30); + if (AutoV2 && !string.IsNullOrWhiteSpace(res.v2_authid)) + _Authid = res.v2_authid; + return _Token = res.access_token; + } + catch (Exception ex) + { + var clientError = false; + + try + { + // Only retry once on client errors + if (ex is HttpRequestException { StatusCode: not null } exception) + { + var sc = (int)exception.StatusCode; + clientError = sc is >= 400 and <= 499; + } + } + catch + { + // ignored + } + + if (response != null && response.Headers.Contains("X-Reason")) + { + var msg = response.Headers.GetValues("X-Reason").FirstOrDefault(); + + if (string.IsNullOrWhiteSpace(msg)) + msg = response.StatusCode.ToString(); + + //TODO: Test; this was ported from old code but looks fragile. + if (response.StatusCode == HttpStatusCode.ServiceUnavailable) + { + string errorKey = msg == response.StatusCode.ToString() + ? "OAuthOverQuotaError" + : "OAuthLoginError"; + + string errorMessage = errorKey == "OAuthOverQuotaError" + ? Strings.OAuthHelper.OverQuotaError + : Strings.OAuthHelper.AuthorizationFailure(msg, OAuthLoginUrl); + + throw new Interface.UserInformationException(errorMessage, errorKey, + errorKey == "OAuthLoginError" ? ex : null); + } + } + + if (retries >= (clientError ? 1 : MAX_AUTHORIZATION_RETRIES)) + { + AttemptParseAndThrowException(ex, response); + throw; + } + + Thread.Sleep(TimeSpan.FromSeconds(Math.Pow(2, retries))); + retries++; + } + finally + { + response?.Dispose(); + } + + } + } + + return _Token; + } + } +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/OAuthHelper/OAuthServiceResponse.cs b/Duplicati/Library/Backend/OAuthHelper/OAuthServiceResponse.cs new file mode 100644 index 000000000..ee721cac9 --- /dev/null +++ b/Duplicati/Library/Backend/OAuthHelper/OAuthServiceResponse.cs @@ -0,0 +1,33 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +namespace Duplicati.Library; + +public class OAuthServiceResponse +{ + public string access_token { get; set; } + + [Newtonsoft.Json.JsonProperty(NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public int expires { get; set; } + + [Newtonsoft.Json.JsonProperty(NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string v2_authid { get; set; } +} \ No newline at end of file From 597c7e1fa3521430bd63d4753fd6bd0180ff7802 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 21 Mar 2025 08:54:24 +0100 Subject: [PATCH 11/48] Implemented nullable for the backends --- .../Backend/AliyunOSS/AliyunOSSBackend.cs | 34 +++++----- .../Backend/AliyunOSS/AliyunOSSOptions.cs | 10 +-- ...Duplicati.Library.Backend.AliyunOSS.csproj | 3 + .../Backend/AzureBlob/AzureBlobBackend.cs | 5 +- .../Backend/AzureBlob/AzureBlobWrapper.cs | 2 +- ...Duplicati.Library.Backend.AzureBlob.csproj | 2 + Duplicati/Library/Backend/Backblaze/B2.cs | 62 ++++++++++++------- .../Library/Backend/Backblaze/B2AuthHelper.cs | 20 +++--- ...Duplicati.Library.Backend.Backblaze.csproj | 4 +- .../Backend/Backblaze/Model/BucketEntity.cs | 6 +- .../Backblaze/Model/ListFilesRequest.cs | 6 +- .../Library/Backend/Backblaze/Strings.cs | 2 +- Duplicati/Library/Backend/Box/BoxBackend.cs | 7 --- .../Box/Duplicati.Library.Backend.Box.csproj | 3 + Duplicati/Library/Backend/CIFS/CIFS.cs | 8 --- .../Duplicati.Library.Backend.CIFS.csproj | 3 + .../CIFS/Model/SMBConnectionParameters.cs | 6 +- 17 files changed, 95 insertions(+), 88 deletions(-) diff --git a/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs b/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs index a7661a9ac..3c98dcb8e 100644 --- a/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs +++ b/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs @@ -24,13 +24,8 @@ using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using Duplicati.Library.Utility; using Duplicati.Library.Utility.Options; -using System; -using System.Collections.Generic; -using System.IO; using System.Net; using System.Runtime.CompilerServices; -using System.Threading; -using System.Threading.Tasks; namespace Duplicati.Library.Backend.AliyunOSS { @@ -52,30 +47,29 @@ namespace Duplicati.Library.Backend.AliyunOSS private readonly TimeoutOptionsHelper.Timeouts _timeouts; public OSS() - { } - - public OSS(string url, Dictionary options) { - _ossOptions = new AliyunOSSOptions(); + _ossOptions = null!; + _timeouts = null!; + } + + public OSS(string url, Dictionary options) + { _timeouts = TimeoutOptionsHelper.Parse(options); var uri = new Utility.Uri(url?.Trim()); var prefix = uri.HostAndPath?.TrimPath(); - if (!string.IsNullOrEmpty(prefix)) - _ossOptions.Path = prefix; - var auth = AuthOptionsHelper.ParseWithAlias(options, uri, OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY_SECRET) .RequireCredentials(); - _ossOptions.AccessKeyId = auth.Username; - _ossOptions.AccessKeySecret = auth.Password; - - if (options.ContainsKey(OSS_BUCKET_NAME)) - _ossOptions.BucketName = options[OSS_BUCKET_NAME]; - - if (options.ContainsKey(OSS_ENDPOINT)) - _ossOptions.Endpoint = options[OSS_ENDPOINT]; + _ossOptions = new AliyunOSSOptions() + { + BucketName = options.GetValueOrDefault(OSS_BUCKET_NAME), + Endpoint = options.GetValueOrDefault(OSS_ENDPOINT), + AccessKeyId = auth.Username!, + AccessKeySecret = auth.Password!, + Path = prefix ?? string.Empty + }; } private OssClient GetClient(bool isUseNewServiceClient = true) diff --git a/Duplicati/Library/Backend/AliyunOSS/AliyunOSSOptions.cs b/Duplicati/Library/Backend/AliyunOSS/AliyunOSSOptions.cs index a5b58bc66..97490a33c 100644 --- a/Duplicati/Library/Backend/AliyunOSS/AliyunOSSOptions.cs +++ b/Duplicati/Library/Backend/AliyunOSS/AliyunOSSOptions.cs @@ -31,19 +31,19 @@ namespace Duplicati.Library.Backend.AliyunOSS /// A bucket is a container used for storing objects (Object), and all objects must belong to a bucket. /// 存储空间是您用于存储对象（Object）的容器，所有的对象都必须隶属于某个存储空间。 /// - public string BucketName { get; set; } + public required string? BucketName { get; set; } /// /// AccessKeyId is used to identify the user. AccessKeySecret is the key used by the user to encrypt signature strings and by OSS to verify signature strings. The AccessKeySecret must be kept confidential. /// AccessKeyId用于标识用户，AccessKeySecret是用户用于加密签名字符串和OSS用来验证签名字符串的密钥，其中AccessKeySecret 必须保密。 /// - public string AccessKeyId { get; set; } + public required string AccessKeyId { get; set; } /// /// AccessKeyId is used to identify the user. AccessKeySecret is the key used by the user to encrypt signature strings and by OSS to verify signature strings. The AccessKeySecret must be kept confidential. /// AccessKeyId用于标识用户，AccessKeySecret是用户用于加密签名字符串和OSS用来验证签名字符串的密钥，其中AccessKeySecret 必须保密。 /// - public string AccessKeySecret { get; set; } + public required string AccessKeySecret { get; set; } /// /// Endpoint represents the access domain name for OSS external services. @@ -54,13 +54,13 @@ namespace Duplicati.Library.Backend.AliyunOSS /// oss-cn-hongkong.aliyuncs.com /// oss-us-west-1.aliyuncs.com /// - public string Endpoint { get; set; } + public required string? Endpoint { get; set; } /// /// A path or subfolder in a bucket /// 桶中的路径或子文件夹 /// - public string Path { get; set; } + public required string Path { get; set; } } } diff --git a/Duplicati/Library/Backend/AliyunOSS/Duplicati.Library.Backend.AliyunOSS.csproj b/Duplicati/Library/Backend/AliyunOSS/Duplicati.Library.Backend.AliyunOSS.csproj index 60d682d7e..9296dfe7f 100644 --- a/Duplicati/Library/Backend/AliyunOSS/Duplicati.Library.Backend.AliyunOSS.csproj +++ b/Duplicati/Library/Backend/AliyunOSS/Duplicati.Library.Backend.AliyunOSS.csproj @@ -4,6 +4,9 @@ Library false Copyright © 2025 Team Duplicati, MIT license + enable + enable + Duplicati.Library.Backend.AliyunOSS diff --git a/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs b/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs index 129f0c41b..555111110 100644 --- a/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs +++ b/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs @@ -36,11 +36,12 @@ namespace Duplicati.Library.Backend.AzureBlob // This constructor is needed by the BackendLoader. public AzureBlobBackend() { + _azureBlob = null!; } // ReSharper disable once UnusedMember.Global // This constructor is needed by the BackendLoader. - public AzureBlobBackend(string url, Dictionary options) + public AzureBlobBackend(string url, Dictionary options) { var uri = new Utility.Uri(url); uri.RequireHost(); @@ -57,7 +58,7 @@ namespace Duplicati.Library.Backend.AzureBlob if (!auth.HasPassword && string.IsNullOrWhiteSpace(sasToken)) throw new UserInformationException(Strings.AzureBlobBackend.NoAccessKeyOrSasToken, "AzureNoAccessKeyOrSasToken"); - _azureBlob = new AzureBlobWrapper(auth.Username, auth.Password, sasToken, containerName, timeouts); + _azureBlob = new AzureBlobWrapper(auth.Username!, auth.Password, sasToken, containerName, timeouts); } public string DisplayName => Strings.AzureBlobBackend.DisplayName; diff --git a/Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs b/Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs index bf164edd2..1ba6c34a8 100644 --- a/Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs +++ b/Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs @@ -60,7 +60,7 @@ namespace Duplicati.Library.Backend.AzureBlob /// The Shared Access Signature (SAS) token for authentication. /// The name of the blob container. /// The timeout options. - public AzureBlobWrapper(string accountName, string accessKey, string sasToken, string containerName, TimeoutOptionsHelper.Timeouts timeouts) + public AzureBlobWrapper(string accountName, string? accessKey, string? sasToken, string containerName, TimeoutOptionsHelper.Timeouts timeouts) { BlobServiceClient blobServiceClient; if (sasToken != null) diff --git a/Duplicati/Library/Backend/AzureBlob/Duplicati.Library.Backend.AzureBlob.csproj b/Duplicati/Library/Backend/AzureBlob/Duplicati.Library.Backend.AzureBlob.csproj index f3de97e26..44eb59ab0 100644 --- a/Duplicati/Library/Backend/AzureBlob/Duplicati.Library.Backend.AzureBlob.csproj +++ b/Duplicati/Library/Backend/AzureBlob/Duplicati.Library.Backend.AzureBlob.csproj @@ -4,7 +4,9 @@ net8.0 Library Copyright © 2025 Team Duplicati, MIT license + enable enable + Duplicati.Library.Backend.AzureBlob diff --git a/Duplicati/Library/Backend/Backblaze/B2.cs b/Duplicati/Library/Backend/Backblaze/B2.cs index 5a343688f..899e142e7 100644 --- a/Duplicati/Library/Backend/Backblaze/B2.cs +++ b/Duplicati/Library/Backend/Backblaze/B2.cs @@ -100,7 +100,7 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend /// /// The type of bucket (e.g., allPrivate, allPublic) being accessed or created /// - private readonly string _bucketType; + private readonly string? _bucketType; /// /// The number of files to retrieve per API request @@ -110,7 +110,7 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend /// /// Custom download URL for the B2 service, if specified /// - private readonly string _downloadUrl; + private readonly string? _downloadUrl; /// /// Helper class for handling B2 authentication and API requests @@ -125,17 +125,17 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend /// /// Cached upload URL and authorization token for file uploads /// - private UploadUrlResponse _uploadUrl; + private UploadUrlResponse? _uploadUrl; /// /// Cache of file listings, mapping filenames to their versions /// - private Dictionary> _filecache; + private Dictionary>? _filecache; /// /// The current bucket entity containing bucket information and credentials /// - private BucketEntity _bucket; + private BucketEntity? _bucket; /// /// A scoped instance of Http client to be used with the backend, this will use a @@ -154,6 +154,12 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend /// public B2() { + _bucketName = null!; + _prefix = null!; + _urlencodedPrefix = null!; + _b2AuthHelper = null!; + _timeouts = null!; + _httpClient = null!; } /// @@ -161,7 +167,7 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend /// /// URL in Duplicati Uri format /// options to be used in the backend - public B2(string url, Dictionary options) + public B2(string url, Dictionary options) { var uri = new Utility.Uri(url); @@ -201,7 +207,7 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend _httpClient = HttpClientHelper.CreateClient(); _httpClient.Timeout = Timeout.InfiniteTimeSpan; - _b2AuthHelper = new B2AuthHelper(auth.Username, auth.Password, _httpClient); + _b2AuthHelper = new B2AuthHelper(auth.Username!, auth.Password!, _httpClient); _timeouts = TimeoutOptionsHelper.Parse(options); } @@ -253,7 +259,11 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend _uploadUrl = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct => await _b2AuthHelper.PostAndGetJsonDataAsync( $"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_get_upload_url", - new UploadUrlRequest { BucketID = (await GetBucketAsync(cancellationToken)).BucketID }, + new UploadUrlRequest + { + BucketID = (await GetBucketAsync(cancellationToken)).BucketID + ?? throw new Exception("BucketID is null") + }, ct).ConfigureAwait(false) ).ConfigureAwait(false); @@ -300,7 +310,7 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken) { - TempFile tmp = null; + TempFile? tmp = null; var measure = stream; while (measure is OverrideableStream os && @@ -363,7 +373,8 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend UploadFileResponse fileinfo; using (var tr = new StreamReader(rdata)) await using (var jr = new Newtonsoft.Json.JsonTextReader(tr)) - fileinfo = new Newtonsoft.Json.JsonSerializer().Deserialize(jr); + fileinfo = new Newtonsoft.Json.JsonSerializer().Deserialize(jr) + ?? throw new Exception("Failed to parse response"); // Delete old versions if (_filecache!.ContainsKey(remotename)) @@ -417,7 +428,7 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend : _b2AuthHelper.CreateRequest( $"{DownloadUrl}/{_urlencodedPrefix}{Utility.Uri.UrlPathEncode(remotename)}"); - HttpResponseMessage response = null; + HttpResponseMessage? response = null; try { // For GetAsync, no timeout is specified. The only thing that can stop it is the cancellation token @@ -446,13 +457,13 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend /// Implementation of interface method for listing remote folder contents /// /// List of IFileEntry with directory listing result - private async Task RebuildFileCache(CancellationToken cancellationToken) + private async Task>> RebuildFileCache(CancellationToken cancellationToken) { _filecache = null; var cache = new Dictionary>(); - string nextFileId = null; - string nextFileName = null; - string listVersionUrl = $"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_list_file_versions"; + string? nextFileId = null; + string? nextFileName = null; + var listVersionUrl = $"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_list_file_versions"; var listRetryHelper = RetryAfterHelper.CreateOrGetRetryAfterHelper(listVersionUrl); @@ -465,7 +476,8 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend $"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_list_file_versions", new ListFilesRequest { - BucketID = (await GetBucketAsync(cancellationToken)).BucketID, + BucketID = (await GetBucketAsync(cancellationToken)).BucketID + ?? throw new Exception("BucketID is null"), MaxFileCount = _pageSize, Prefix = _prefix, StartFileID = nextFileId, @@ -509,7 +521,7 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend } } while (nextFileId != null); - _filecache = cache; + return _filecache = cache; } /// @@ -518,8 +530,8 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend /// List of IFileEntry with directory listing result public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancellationToken) { - await RebuildFileCache(cancellationToken).ConfigureAwait(false); - foreach (var x in _filecache) + var filecache = await RebuildFileCache(cancellationToken).ConfigureAwait(false); + foreach (var x in filecache) { var newest = x.Value.OrderByDescending(y => y.UploadTimestamp).First(); var ts = Utility.Utility.EPOCH.AddMilliseconds(newest.UploadTimestamp); @@ -560,10 +572,11 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend { try { - if (_filecache == null || !_filecache.ContainsKey(remotename)) - await RebuildFileCache(cancellationToken).ConfigureAwait(false); + var filecache = _filecache; + if (filecache == null || !filecache.ContainsKey(remotename)) + filecache = await RebuildFileCache(cancellationToken).ConfigureAwait(false); - if (!_filecache.TryGetValue(remotename, out var value)) + if (!filecache.TryGetValue(remotename, out var value)) throw new FileMissingException(); foreach (var n in value.OrderBy(x => x.UploadTimestamp)) @@ -579,7 +592,7 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend ).ConfigureAwait(false) ).ConfigureAwait(false); - _filecache[remotename].Clear(); + filecache[remotename].Clear(); } catch { @@ -627,11 +640,12 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend public string Description => Strings.B2.Description; /// - public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new string[] { + public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new string?[] { new System.Uri(B2AuthHelper.AUTH_URL).Host, _b2AuthHelper?.ApiDnsName, _b2AuthHelper?.DownloadDnsName }.Where(x => !string.IsNullOrEmpty(x)) + .Select(x => x!) .ToArray()); /// diff --git a/Duplicati/Library/Backend/Backblaze/B2AuthHelper.cs b/Duplicati/Library/Backend/Backblaze/B2AuthHelper.cs index 18ad6113a..534168d38 100644 --- a/Duplicati/Library/Backend/Backblaze/B2AuthHelper.cs +++ b/Duplicati/Library/Backend/Backblaze/B2AuthHelper.cs @@ -40,7 +40,7 @@ public class B2AuthHelper(string userid, string password, HttpClient httpClient) /// /// Cached authorization response /// - private AuthResponse _CachedAuthResponse; + private AuthResponse? _CachedAuthResponse; /// /// Cached authorization response expiration time @@ -93,7 +93,7 @@ public class B2AuthHelper(string userid, string password, HttpClient httpClient) /// The URL for the request /// HTTP method (defaults to GET if null) /// Configured HttpRequestMessage - public override HttpRequestMessage CreateRequest(string url, string method = null) + public override HttpRequestMessage CreateRequest(string url, string? method = null) { HttpRequestMessage request = base.CreateRequest(url, method); request.Headers.TryAddWithoutValidation("Authorization", AuthorizationToken); @@ -115,16 +115,16 @@ public class B2AuthHelper(string userid, string password, HttpClient httpClient) /// /// API DNS Name /// - public string ApiDnsName => - _CachedAuthResponse == null || string.IsNullOrWhiteSpace(_CachedAuthResponse.APIUrl) + public string? ApiDnsName => + string.IsNullOrWhiteSpace(_CachedAuthResponse?.APIUrl) ? null : new Uri(_CachedAuthResponse.APIUrl).Host; /// /// Download DNS Name /// - public string DownloadDnsName => - _CachedAuthResponse == null || string.IsNullOrWhiteSpace(_CachedAuthResponse.DownloadUrl) + public string? DownloadDnsName => + string.IsNullOrWhiteSpace(_CachedAuthResponse?.DownloadUrl) ? null : new Uri(_CachedAuthResponse.DownloadUrl).Host; @@ -137,7 +137,7 @@ public class B2AuthHelper(string userid, string password, HttpClient httpClient) while (true) { - HttpResponseMessage response = null; + HttpResponseMessage? response = null; try { @@ -197,19 +197,19 @@ public class B2AuthHelper(string userid, string password, HttpClient httpClient) /// Exception to be parsed /// Response context /// New detailed exception - public override void AttemptParseAndThrowException(Exception ex, HttpResponseMessage responseContext = null) + public override void AttemptParseAndThrowException(Exception ex, HttpResponseMessage? responseContext = null) { if (ex is not HttpRequestException || responseContext == null) return; - if (ex is HttpRequestException && responseContext != null && responseContext.StatusCode == HttpStatusCode.TooManyRequests) + if (ex is HttpRequestException && responseContext.StatusCode == HttpStatusCode.TooManyRequests) throw new TooManyRequestException(responseContext.Headers.RetryAfter); using var stream = responseContext.Content.ReadAsStream(); using var reader = new StreamReader(stream); var rawData = reader.ReadToEnd(); var errorResponse = JsonConvert.DeserializeObject(rawData); - throw new UserInformationException($"Backblaze ErrorResponse: {errorResponse.Status} - {errorResponse.Code}: {errorResponse.Message}", "BackblazeErrorResponse"); + throw new UserInformationException($"Backblaze ErrorResponse: {errorResponse?.Status} - {errorResponse?.Code}: {errorResponse?.Message}", "BackblazeErrorResponse"); } /// diff --git a/Duplicati/Library/Backend/Backblaze/Duplicati.Library.Backend.Backblaze.csproj b/Duplicati/Library/Backend/Backblaze/Duplicati.Library.Backend.Backblaze.csproj index b97001bf1..1e4aef315 100644 --- a/Duplicati/Library/Backend/Backblaze/Duplicati.Library.Backend.Backblaze.csproj +++ b/Duplicati/Library/Backend/Backblaze/Duplicati.Library.Backend.Backblaze.csproj @@ -4,7 +4,9 @@ net8.0 Library Copyright © 2025 Team Duplicati, MIT license - true + enable + enable + Duplicati.Library.Backend.Box diff --git a/Duplicati/Library/Backend/Backblaze/Model/BucketEntity.cs b/Duplicati/Library/Backend/Backblaze/Model/BucketEntity.cs index c193d21eb..02b70ec8e 100644 --- a/Duplicati/Library/Backend/Backblaze/Model/BucketEntity.cs +++ b/Duplicati/Library/Backend/Backblaze/Model/BucketEntity.cs @@ -34,17 +34,17 @@ internal class BucketEntity : AccountIDEntity /// This property can be null as specified by the NullValueHandling attribute. /// [JsonProperty("bucketId", NullValueHandling = NullValueHandling.Ignore)] - public string BucketID { get; set; } + public string? BucketID { get; set; } /// /// Gets or sets the name of the bucket. /// [JsonProperty("bucketName")] - public string BucketName { get; set; } + public required string BucketName { get; set; } /// /// Gets or sets the type of the bucket (e.g., "allPrivate", "allPublic"). /// [JsonProperty("bucketType")] - public string BucketType { get; set; } + public string? BucketType { get; set; } } \ No newline at end of file diff --git a/Duplicati/Library/Backend/Backblaze/Model/ListFilesRequest.cs b/Duplicati/Library/Backend/Backblaze/Model/ListFilesRequest.cs index cf2087afa..3f63e829d 100644 --- a/Duplicati/Library/Backend/Backblaze/Model/ListFilesRequest.cs +++ b/Duplicati/Library/Backend/Backblaze/Model/ListFilesRequest.cs @@ -34,14 +34,14 @@ internal class ListFilesRequest : BucketIDEntity /// Used for pagination to continue listing from a specific point. /// [JsonProperty("startFileName", NullValueHandling = NullValueHandling.Ignore)] - public string StartFileName { get; set; } + public string? StartFileName { get; set; } /// /// Gets or sets the file ID to start listing from. /// Used in conjunction with StartFileName for pagination. /// [JsonProperty("startFileId", NullValueHandling = NullValueHandling.Ignore)] - public string StartFileID { get; set; } + public string? StartFileID { get; set; } /// /// Gets or sets the maximum number of files to return in a single request. @@ -54,5 +54,5 @@ internal class ListFilesRequest : BucketIDEntity /// Only files whose names begin with this prefix will be returned. /// [JsonProperty("prefix")] - public string Prefix { get; set; } + public required string Prefix { get; set; } } \ No newline at end of file diff --git a/Duplicati/Library/Backend/Backblaze/Strings.cs b/Duplicati/Library/Backend/Backblaze/Strings.cs index a10b2881f..78f16924d 100644 --- a/Duplicati/Library/Backend/Backblaze/Strings.cs +++ b/Duplicati/Library/Backend/Backblaze/Strings.cs @@ -38,6 +38,6 @@ namespace Duplicati.Library.Backend.Strings public static string B2pagesizeDescriptionShort { get { return LC.L(@"The size of file-listing pages"); } } public static string B2downloadurlDescriptionLong { get { return LC.L(@"Change this if you want to use your custom domain to download files, and uploading will not be affected. The default download URL depends on your account and looks like ""https://f00X.backblazeb2.com""."); } } public static string B2downloadurlDescriptionShort { get { return LC.L(@"The base URL to use for downloading files"); } } - public static string InvalidPageSizeError(string argname, string value) { return LC.L(@"The setting ""{0}"" is invalid for ""{1}"". It must be an integer larger than zero.", value, argname); } + public static string InvalidPageSizeError(string argname, string? value) { return LC.L(@"The setting ""{0}"" is invalid for ""{1}"". It must be an integer larger than zero.", value, argname); } } } diff --git a/Duplicati/Library/Backend/Box/BoxBackend.cs b/Duplicati/Library/Backend/Box/BoxBackend.cs index 21eb991a1..3b827abed 100644 --- a/Duplicati/Library/Backend/Box/BoxBackend.cs +++ b/Duplicati/Library/Backend/Box/BoxBackend.cs @@ -19,20 +19,13 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -#nullable enable - using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using Duplicati.Library.Utility; using Duplicati.Library.Utility.Options; using Newtonsoft.Json; -using System; -using System.Collections.Generic; -using System.Linq; using System.Net; using System.Runtime.CompilerServices; -using System.Threading; -using System.Threading.Tasks; namespace Duplicati.Library.Backend.Box { diff --git a/Duplicati/Library/Backend/Box/Duplicati.Library.Backend.Box.csproj b/Duplicati/Library/Backend/Box/Duplicati.Library.Backend.Box.csproj index d9cbc0442..983a2765c 100644 --- a/Duplicati/Library/Backend/Box/Duplicati.Library.Backend.Box.csproj +++ b/Duplicati/Library/Backend/Box/Duplicati.Library.Backend.Box.csproj @@ -4,6 +4,9 @@ net8.0 Library Copyright © 2025 Team Duplicati, MIT license + enable + enable + Duplicati.Library.Backend.Box diff --git a/Duplicati/Library/Backend/CIFS/CIFS.cs b/Duplicati/Library/Backend/CIFS/CIFS.cs index 04603ec74..2a98f6521 100644 --- a/Duplicati/Library/Backend/CIFS/CIFS.cs +++ b/Duplicati/Library/Backend/CIFS/CIFS.cs @@ -19,15 +19,7 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -#nullable enable - -using System; using Duplicati.Library.Interface; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; using Duplicati.Library.Backend.CIFS; using Duplicati.Library.Backend.CIFS.Model; using SMBLibrary; diff --git a/Duplicati/Library/Backend/CIFS/Duplicati.Library.Backend.CIFS.csproj b/Duplicati/Library/Backend/CIFS/Duplicati.Library.Backend.CIFS.csproj index 11375a4ab..3dbadb34f 100644 --- a/Duplicati/Library/Backend/CIFS/Duplicati.Library.Backend.CIFS.csproj +++ b/Duplicati/Library/Backend/CIFS/Duplicati.Library.Backend.CIFS.csproj @@ -4,6 +4,9 @@ net8.0 Library Copyright © 2025 Team Duplicati, MIT license + enable + enable + Duplicati.Library.Backend.CIFS diff --git a/Duplicati/Library/Backend/CIFS/Model/SMBConnectionParameters.cs b/Duplicati/Library/Backend/CIFS/Model/SMBConnectionParameters.cs index 8fe1770b9..044b03cdd 100644 --- a/Duplicati/Library/Backend/CIFS/Model/SMBConnectionParameters.cs +++ b/Duplicati/Library/Backend/CIFS/Model/SMBConnectionParameters.cs @@ -40,8 +40,8 @@ public sealed record SMBConnectionParameters( SMBTransportType TransportType, string ShareName, string Path, - string AuthDomain, - string AuthUser, - string AuthPassword, + string? AuthDomain, + string? AuthUser, + string? AuthPassword, int? ReadBufferSize, int? WriteBufferSize); \ No newline at end of file From 41d2b220158ade70d86639ef909ed65ecae40dba Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 21 Mar 2025 09:13:48 +0100 Subject: [PATCH 12/48] Adde option names to helper --- Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs | 4 ++-- Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs | 4 ++-- Duplicati/Library/Backend/Backblaze/B2.cs | 6 ++---- Duplicati/Library/Utility/Options/AuthOptionsHelper.cs | 9 +++++++++ 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs b/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs index 3c98dcb8e..27681285b 100644 --- a/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs +++ b/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs @@ -298,8 +298,8 @@ namespace Duplicati.Library.Backend.AliyunOSS get { return new List([ - new CommandLineArgument(OSS_ACCESS_KEY_ID, CommandLineArgument.ArgumentType.String, Strings.OSSBackend.OSSAccessKeyIdDescriptionShort, Strings.OSSBackend.OSSAccessKeyIdDescriptionLong, null, ["auth-username"]), - new CommandLineArgument(OSS_ACCESS_KEY_SECRET, CommandLineArgument.ArgumentType.Password, Strings.OSSBackend.OSSAccessKeySecretDescriptionShort, Strings.OSSBackend.OSSAccessKeySecretDescriptionLong, null, ["auth-password"]), + new CommandLineArgument(OSS_ACCESS_KEY_ID, CommandLineArgument.ArgumentType.String, Strings.OSSBackend.OSSAccessKeyIdDescriptionShort, Strings.OSSBackend.OSSAccessKeyIdDescriptionLong, null, [AuthOptionsHelper.AuthUsername]), + new CommandLineArgument(OSS_ACCESS_KEY_SECRET, CommandLineArgument.ArgumentType.Password, Strings.OSSBackend.OSSAccessKeySecretDescriptionShort, Strings.OSSBackend.OSSAccessKeySecretDescriptionLong, null, [AuthOptionsHelper.AuthPassword]), new CommandLineArgument(OSS_BUCKET_NAME, CommandLineArgument.ArgumentType.String, Strings.OSSBackend.OSSBucketNameDescriptionShort, Strings.OSSBackend.OSSBucketNameDescriptionLong), new CommandLineArgument(OSS_ENDPOINT, CommandLineArgument.ArgumentType.String, Strings.OSSBackend.OSSEndpointDescriptionShort, Strings.OSSBackend.OSSEndpointDescriptionLong), .. TimeoutOptionsHelper.GetOptions() diff --git a/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs b/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs index 555111110..ca8543911 100644 --- a/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs +++ b/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs @@ -108,13 +108,13 @@ namespace Duplicati.Library.Backend.AzureBlob Strings.AzureBlobBackend.StorageAccountNameDescriptionShort, Strings.AzureBlobBackend.StorageAccountNameDescriptionLong, null, - ["auth-username"]), + [AuthOptionsHelper.AuthUsername]), new CommandLineArgument("azure-access-key", CommandLineArgument.ArgumentType.Password, Strings.AzureBlobBackend.AccessKeyDescriptionShort, Strings.AzureBlobBackend.AccessKeyDescriptionLong, null, - ["auth-password"]), + [AuthOptionsHelper.AuthPassword]), new CommandLineArgument("azure-access-sas-token", CommandLineArgument.ArgumentType.Password, Strings.AzureBlobBackend.SasTokenDescriptionShort, diff --git a/Duplicati/Library/Backend/Backblaze/B2.cs b/Duplicati/Library/Backend/Backblaze/B2.cs index 899e142e7..0bfaf6fc9 100644 --- a/Duplicati/Library/Backend/Backblaze/B2.cs +++ b/Duplicati/Library/Backend/Backblaze/B2.cs @@ -298,10 +298,8 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend /// public IList SupportedCommands => new List([ - new CommandLineArgument(B2_ID_OPTION, CommandLineArgument.ArgumentType.String, Strings.B2.B2accountidDescriptionShort, Strings.B2.B2accountidDescriptionLong, null, - ["auth-password"], null), - new CommandLineArgument(B2_KEY_OPTION, CommandLineArgument.ArgumentType.Password, Strings.B2.B2applicationkeyDescriptionShort, Strings.B2.B2applicationkeyDescriptionLong, null, - ["auth-username"], null), + new CommandLineArgument(B2_ID_OPTION, CommandLineArgument.ArgumentType.String, Strings.B2.B2accountidDescriptionShort, Strings.B2.B2accountidDescriptionLong, null, [AuthOptionsHelper.AuthUsername], null), + new CommandLineArgument(B2_KEY_OPTION, CommandLineArgument.ArgumentType.Password, Strings.B2.B2applicationkeyDescriptionShort, Strings.B2.B2applicationkeyDescriptionLong, null, [AuthOptionsHelper.AuthPassword], null), new CommandLineArgument(B2_CREATE_BUCKET_TYPE_OPTION, CommandLineArgument.ArgumentType.String, Strings.B2.B2createbuckettypeDescriptionShort, Strings.B2.B2createbuckettypeDescriptionLong, DEFAULT_BUCKET_TYPE), new CommandLineArgument(B2_PAGESIZE_OPTION, CommandLineArgument.ArgumentType.Integer, Strings.B2.B2pagesizeDescriptionShort, Strings.B2.B2pagesizeDescriptionLong, DEFAULT_PAGE_SIZE.ToString()), new CommandLineArgument(B2_DOWNLOAD_URL_OPTION, CommandLineArgument.ArgumentType.String, Strings.B2.B2downloadurlDescriptionShort, Strings.B2.B2downloadurlDescriptionLong), diff --git a/Duplicati/Library/Utility/Options/AuthOptionsHelper.cs b/Duplicati/Library/Utility/Options/AuthOptionsHelper.cs index ef3c01006..337900f9e 100644 --- a/Duplicati/Library/Utility/Options/AuthOptionsHelper.cs +++ b/Duplicati/Library/Utility/Options/AuthOptionsHelper.cs @@ -32,6 +32,15 @@ namespace Duplicati.Library.Utility.Options; /// public static class AuthOptionsHelper { + /// + /// The authentication username option, without a prefix + /// + public const string AuthUsername = "auth-username"; + /// + /// The authentication password option, without a prefix + /// + public const string AuthPassword = "auth-password"; + /// /// Gets the authentication options /// From 769158c3f05aa24465256b84391f4853915bb553 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 21 Mar 2025 10:05:33 +0100 Subject: [PATCH 13/48] Upgraded CloudFiles --- Duplicati/Library/Backend/Backblaze/B2.cs | 3 +- .../Library/Backend/CloudFiles/CloudFiles.cs | 234 +++++++++--------- ...uplicati.Library.Backend.CloudFiles.csproj | 2 + .../Library/Backend/CloudFiles/Strings.cs | 51 ++-- .../SecretProvider/SecretProviderModules.cs | 4 +- .../Library/Utility/EnumerableExtensions.cs | 51 ++++ .../Utility/Options/AuthOptionsHelper.cs | 10 + .../WebserverCore/Endpoints/V1/Filesystem.cs | 4 +- 8 files changed, 212 insertions(+), 147 deletions(-) create mode 100644 Duplicati/Library/Utility/EnumerableExtensions.cs diff --git a/Duplicati/Library/Backend/Backblaze/B2.cs b/Duplicati/Library/Backend/Backblaze/B2.cs index 0bfaf6fc9..0d7c27477 100644 --- a/Duplicati/Library/Backend/Backblaze/B2.cs +++ b/Duplicati/Library/Backend/Backblaze/B2.cs @@ -642,8 +642,7 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend new System.Uri(B2AuthHelper.AUTH_URL).Host, _b2AuthHelper?.ApiDnsName, _b2AuthHelper?.DownloadDnsName - }.Where(x => !string.IsNullOrEmpty(x)) - .Select(x => x!) + }.WhereNotNullOrWhiteSpace() .ToArray()); /// diff --git a/Duplicati/Library/Backend/CloudFiles/CloudFiles.cs b/Duplicati/Library/Backend/CloudFiles/CloudFiles.cs index 22329ead1..72e627f4a 100644 --- a/Duplicati/Library/Backend/CloudFiles/CloudFiles.cs +++ b/Duplicati/Library/Backend/CloudFiles/CloudFiles.cs @@ -21,14 +21,11 @@ using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; -using System; -using System.Collections.Generic; -using System.Linq; +using Duplicati.Library.Utility; +using Duplicati.Library.Utility.Options; using System.Net; using System.Runtime.CompilerServices; using System.Security.Cryptography; -using System.Threading; -using System.Threading.Tasks; namespace Duplicati.Library.Backend { @@ -36,54 +33,57 @@ namespace Duplicati.Library.Backend // This class is instantiated dynamically in the BackendLoader. public class CloudFiles : IBackend, IStreamingBackend { + /// + /// The log tag for this class + /// + private static readonly string LOGTAG = Logging.Log.LogTagFromType(); + public const string AUTH_URL_US = "https://identity.api.rackspacecloud.com/auth"; public const string AUTH_URL_UK = "https://lon.auth.api.rackspacecloud.com/v1.0"; private const string DUMMY_HOSTNAME = "api.mosso.com"; + private const string AUTH_USERNAME_OPTION = "cloudfiles-username"; + private const string AUTH_PASSWORD_OPTION = "cloudfiles-accesskey"; + private const int ITEM_LIST_LIMIT = 1000; private readonly string m_username; private readonly string m_password; private readonly string m_path; - private string m_storageUrl = null; - private string m_authToken = null; + private string? m_storageUrl = null; + private string? m_authToken = null; private readonly string m_authUrl; + private readonly TimeoutOptionsHelper.Timeouts m_timeouts; // ReSharper disable once UnusedMember.Global // This constructor is needed by the BackendLoader. public CloudFiles() { + m_username = null!; + m_password = null!; + m_path = null!; + m_authUrl = null!; + m_timeouts = null!; } // ReSharper disable once UnusedMember.Global // This constructor is needed by the BackendLoader. - public CloudFiles(string url, Dictionary options) + public CloudFiles(string url, Dictionary options) { var uri = new Utility.Uri(url); + var auth = AuthOptionsHelper.ParseWithAlias(options, uri, AUTH_USERNAME_OPTION, AUTH_PASSWORD_OPTION); - if (options.ContainsKey("auth-username")) - m_username = options["auth-username"]; - if (options.ContainsKey("auth-password")) - m_password = options["auth-password"]; - - if (options.ContainsKey("cloudfiles-username")) - m_username = options["cloudfiles-username"]; - if (options.ContainsKey("cloudfiles-accesskey")) - m_password = options["cloudfiles-accesskey"]; - - if (!string.IsNullOrEmpty(uri.Username)) - m_username = uri.Username; - if (!string.IsNullOrEmpty(uri.Password)) - m_password = uri.Password; - - if (string.IsNullOrEmpty(m_username)) + if (!auth.HasUsername) throw new UserInformationException(Strings.CloudFiles.NoUserIDError, "CloudFilesNoUserID"); - if (string.IsNullOrEmpty(m_password)) + if (auth.HasPassword) throw new UserInformationException(Strings.CloudFiles.NoAPIKeyError, "CloudFilesNoApiKey"); + (m_username, m_password) = auth.GetCredentials(); + //Fallback to the previous format if (url.Contains(DUMMY_HOSTNAME)) { + Logging.Log.WriteWarningMessage(LOGTAG, "CloudFilesDeprecatedFormat", null, Strings.CloudFiles.DeprecatedFormatWarning(DUMMY_HOSTNAME)); var u = new System.Uri(url); if (!string.IsNullOrEmpty(u.UserInfo)) @@ -117,8 +117,11 @@ namespace Duplicati.Library.Backend if (!m_path.StartsWith("/", StringComparison.Ordinal)) m_path = "/" + m_path; - if (!options.TryGetValue("cloudfiles-authentication-url", out m_authUrl)) - m_authUrl = Utility.Utility.ParseBoolOption(options, "cloudfiles-uk-account") ? AUTH_URL_UK : AUTH_URL_US; + var authUrl = options.GetValueOrDefault("cloudfiles-authentication-url"); + if (string.IsNullOrEmpty(authUrl)) + authUrl = Utility.Utility.ParseBoolOption(options, "cloudfiles-uk-account") ? AUTH_URL_UK : AUTH_URL_US; + m_authUrl = authUrl; + m_timeouts = TimeoutOptionsHelper.Parse(options); } #region IBackend Members @@ -139,19 +142,20 @@ namespace Duplicati.Library.Backend var markerUrl = ""; bool repeat; - do { var doc = new System.Xml.XmlDocument(); - - var req = CreateRequest("", extraUrl + markerUrl); + var req = await CreateRequest("", extraUrl + markerUrl, cancelToken).ConfigureAwait(false); try { - var areq = new Utility.AsyncHttpRequest(req); - using (var resp = (HttpWebResponse)areq.GetResponse()) - using (var s = areq.GetResponseStream()) - doc.Load(s); + await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, _ => + { + var areq = new Utility.AsyncHttpRequest(req); + using (var resp = (HttpWebResponse)areq.GetResponse()) + using (var s = areq.GetResponseStream()) + doc.Load(s); + }).ConfigureAwait(false); } catch (WebException wex) { @@ -163,11 +167,11 @@ namespace Duplicati.Library.Backend throw; } - System.Xml.XmlNodeList lst = doc.SelectNodes("container/object"); + var lst = doc.SelectNodes("container/object"); //Perhaps the folder does not exist? //The response should be 404 from the server, but it is not :( - if (lst.Count == 0 && markerUrl == "") //Only on first iteration + if (lst == null || lst.Count == 0 && markerUrl == "") //Only on first iteration { try { await CreateFolderAsync(cancelToken).ConfigureAwait(false); } catch { } //Ignore @@ -176,13 +180,10 @@ namespace Duplicati.Library.Backend var lastItemName = ""; foreach (System.Xml.XmlNode n in lst) { - var name = n["name"].InnerText; - long size; - DateTime mod; - - if (!long.TryParse(n["bytes"].InnerText, out size)) + var name = n["name"]?.InnerText; + if (!long.TryParse(n["bytes"]?.InnerText, out var size)) size = -1; - if (!DateTime.TryParse(n["last_modified"].InnerText, out mod)) + if (!DateTime.TryParse(n["last_modified"]?.InnerText, out var mod)) mod = new DateTime(); lastItemName = name; @@ -199,61 +200,55 @@ namespace Duplicati.Library.Backend public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) { - using (System.IO.FileStream fs = System.IO.File.OpenRead(filename)) + using (var fs = File.OpenRead(filename)) await PutAsync(remotename, fs, cancelToken); } public async Task GetAsync(string remotename, string filename, CancellationToken cancelToken) { - using (System.IO.FileStream fs = System.IO.File.Create(filename)) + using (var fs = File.Create(filename)) await GetAsync(remotename, fs, cancelToken).ConfigureAwait(false); } - public Task DeleteAsync(string remotename, CancellationToken cancelToken) + public async Task DeleteAsync(string remotename, CancellationToken cancelToken) { try { - var req = CreateRequest("/" + remotename, ""); - - req.Method = "DELETE"; - var areq = new Utility.AsyncHttpRequest(req); - using (var resp = (HttpWebResponse)areq.GetResponse()) + var req = await CreateRequest("/" + remotename, "", cancelToken).ConfigureAwait(false); + await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => { - if (resp.StatusCode == System.Net.HttpStatusCode.NotFound) - throw new FileMissingException(); + req.Method = "DELETE"; + var areq = new AsyncHttpRequest(req); + using (var resp = (HttpWebResponse)areq.GetResponse()) + { + if (resp.StatusCode == HttpStatusCode.NotFound) + throw new FileMissingException(); - if ((int)resp.StatusCode >= 300) - throw new WebException(Strings.CloudFiles.FileDeleteError, null, WebExceptionStatus.ProtocolError, resp); - else - using (areq.GetResponseStream()) - { } - } + if ((int)resp.StatusCode >= 300) + throw new WebException(Strings.CloudFiles.FileDeleteError, null, WebExceptionStatus.ProtocolError, resp); + else + using (areq.GetResponseStream()) + { } + } + }).ConfigureAwait(false); } - catch (System.Net.WebException wex) + catch (WebException wex) { - if (wex.Response is HttpWebResponse response && response.StatusCode == System.Net.HttpStatusCode.NotFound) + if (wex.Response is HttpWebResponse response && response.StatusCode == HttpStatusCode.NotFound) throw new FileMissingException(wex); else throw; } - - return Task.CompletedTask; } - public IList SupportedCommands - { - get - { - return new List([ - new CommandLineArgument("auth-password", CommandLineArgument.ArgumentType.Password, Strings.CloudFiles.DescriptionAuthPasswordShort, Strings.CloudFiles.DescriptionAuthPasswordLong), - new CommandLineArgument("auth-username", CommandLineArgument.ArgumentType.String, Strings.CloudFiles.DescriptionAuthUsernameShort, Strings.CloudFiles.DescriptionAuthUsernameLong), - new CommandLineArgument("cloudfiles-username", CommandLineArgument.ArgumentType.String, Strings.CloudFiles.DescriptionUsernameShort, Strings.CloudFiles.DescriptionUsernameLong, null, new string[] {"auth-username"} ), - new CommandLineArgument("cloudfiles-accesskey", CommandLineArgument.ArgumentType.Password, Strings.CloudFiles.DescriptionPasswordShort, Strings.CloudFiles.DescriptionPasswordLong, null, new string[] {"auth-password"}), - new CommandLineArgument("cloudfiles-uk-account", CommandLineArgument.ArgumentType.Boolean, Strings.CloudFiles.DescriptionUKAccountShort, Strings.CloudFiles.DescriptionUKAccountLong("cloudfiles-authentication-url", AUTH_URL_UK)), - new CommandLineArgument("cloudfiles-authentication-url", CommandLineArgument.ArgumentType.String, Strings.CloudFiles.DescriptionAuthenticationURLShort, Strings.CloudFiles.DescriptionAuthenticationURLLong_v2("cloudfiles-uk-account"), AUTH_URL_US), - ]); - } - } + public IList SupportedCommands => + [ + new CommandLineArgument(AUTH_USERNAME_OPTION, CommandLineArgument.ArgumentType.String, Strings.CloudFiles.DescriptionUsernameShort, Strings.CloudFiles.DescriptionUsernameLong, null, [AuthOptionsHelper.AuthUsername] ), + new CommandLineArgument(AUTH_PASSWORD_OPTION, CommandLineArgument.ArgumentType.Password, Strings.CloudFiles.DescriptionPasswordShort, Strings.CloudFiles.DescriptionPasswordLong, null, [AuthOptionsHelper.AuthPassword]), + new CommandLineArgument("cloudfiles-uk-account", CommandLineArgument.ArgumentType.Boolean, Strings.CloudFiles.DescriptionUKAccountShort, Strings.CloudFiles.DescriptionUKAccountLong("cloudfiles-authentication-url", AUTH_URL_UK)), + new CommandLineArgument("cloudfiles-authentication-url", CommandLineArgument.ArgumentType.String, Strings.CloudFiles.DescriptionAuthenticationURLShort, Strings.CloudFiles.DescriptionAuthenticationURLLong_v2("cloudfiles-uk-account"), AUTH_URL_US), + .. TimeoutOptionsHelper.GetOptions(), + ]; public string Description { @@ -268,15 +263,16 @@ namespace Duplicati.Library.Backend //The "Folder not found" is not detectable :( this.TestListAsync(cancelToken); - public Task CreateFolderAsync(CancellationToken cancelToken) + public async Task CreateFolderAsync(CancellationToken cancelToken) { - var createReq = CreateRequest("", ""); - createReq.Method = "PUT"; - var areq = new Utility.AsyncHttpRequest(createReq); - using (var resp = (HttpWebResponse)areq.GetResponse()) - { } - - return Task.CompletedTask; + var createReq = await CreateRequest("", "", cancelToken).ConfigureAwait(false); + await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => + { + createReq.Method = "PUT"; + var areq = new AsyncHttpRequest(createReq); + using (var resp = (HttpWebResponse)areq.GetResponse()) + { } + }).ConfigureAwait(false); } #endregion @@ -292,42 +288,46 @@ namespace Duplicati.Library.Backend #region IStreamingBackend Members public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult( - new string[] { + new string?[] { new System.Uri(m_authUrl).Host, string.IsNullOrWhiteSpace(m_storageUrl) ? null : new System.Uri(m_storageUrl).Host } - .Where(x => !string.IsNullOrWhiteSpace(x)) + .WhereNotNullOrWhiteSpace() .ToArray() ); public async Task GetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) { - var req = CreateRequest("/" + remotename, ""); + var req = await CreateRequest("/" + remotename, "", cancelToken).ConfigureAwait(false); req.Method = "GET"; - var areq = new Utility.AsyncHttpRequest(req); + var areq = new AsyncHttpRequest(req); using (var resp = areq.GetResponse()) - using (var s = areq.GetResponseStream()) + using (var s = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => areq.GetResponseStream()).ConfigureAwait(false)) + using (var timeoutStream = s.ObserveReadTimeout(m_timeouts.ReadWriteTimeout)) using (var hasher = MD5.Create()) - using (var mds = new Utility.HashCalculatingStream(s, hasher)) + using (var mds = new HashCalculatingStream(timeoutStream, hasher)) { - string md5Hash = resp.Headers["ETag"]; + var md5Hash = resp.Headers["ETag"]; await Utility.Utility.CopyStreamAsync(mds, stream, true, cancelToken).ConfigureAwait(false); - if (!String.Equals(mds.GetFinalHashString(), md5Hash, StringComparison.OrdinalIgnoreCase)) + if (!string.Equals(mds.GetFinalHashString(), md5Hash, StringComparison.OrdinalIgnoreCase)) throw new Exception(Strings.CloudFiles.ETagVerificationError); } } public async Task PutAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) { - HttpWebRequest req = CreateRequest("/" + remotename, ""); + var req = await CreateRequest("/" + remotename, "", cancelToken).ConfigureAwait(false); req.Method = "PUT"; req.ContentType = "application/octet-stream"; try { req.ContentLength = stream.Length; } catch { } + // TODO: When reviewing this, lets build a common method to unwrap the stream passed from + // BackendManager.PutOperation, so we can use the same logic to compute hashes in all backends + //If we can pre-calculate the MD5 hash before transmission, do so /*if (stream.CanSeek) { @@ -353,22 +353,23 @@ namespace Duplicati.Library.Backend //TODO: We cannot use the local MD5 calculation, because that could involve a throttled read, // and may invoke various events { - string fileHash = null; + string? fileHash = null; long streamLen = -1; try { streamLen = stream.Length; } catch { } - Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req); - using (System.IO.Stream s = areq.GetRequestStream(streamLen)) + var areq = new AsyncHttpRequest(req); + using (var s = areq.GetRequestStream(streamLen)) + using (var timeoutStream = s.ObserveWriteTimeout(m_timeouts.ReadWriteTimeout)) using (var hasher = MD5.Create()) - using (var mds = new Utility.HashCalculatingStream(s, hasher)) + using (var mds = new HashCalculatingStream(timeoutStream, hasher)) { await Utility.Utility.CopyStreamAsync(stream, mds, tryRewindSource: true, cancelToken: cancelToken); fileHash = mds.GetFinalHashString(); } - string md5Hash = null; + string? md5Hash = null; //We need to verify the eTag locally try @@ -390,7 +391,7 @@ namespace Duplicati.Library.Backend } - if (md5Hash == null || !String.Equals(md5Hash, fileHash, StringComparison.OrdinalIgnoreCase)) + if (md5Hash == null || !string.Equals(md5Hash, fileHash, StringComparison.OrdinalIgnoreCase)) { //Remove the broken file try { await DeleteAsync(remotename, cancelToken); } @@ -403,29 +404,32 @@ namespace Duplicati.Library.Backend #endregion - private HttpWebRequest CreateRequest(string remotename, string query) + private async Task CreateRequest(string remotename, string query, CancellationToken cancelToken) { //If this is the first call, get an authentication token if (string.IsNullOrEmpty(m_authToken) || string.IsNullOrEmpty(m_storageUrl)) { - var authReq = (HttpWebRequest)HttpWebRequest.Create(m_authUrl); - authReq.Headers.Add("X-Auth-User", m_username); - authReq.Headers.Add("X-Auth-Key", m_password); - authReq.Method = "GET"; - - var areq = new Utility.AsyncHttpRequest(authReq); - using (var resp = areq.GetResponse()) + await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => { - m_storageUrl = resp.Headers["X-Storage-Url"]; - m_authToken = resp.Headers["X-Auth-Token"]; - } + var authReq = (HttpWebRequest)HttpWebRequest.Create(m_authUrl); + authReq.Headers.Add("X-Auth-User", m_username); + authReq.Headers.Add("X-Auth-Key", m_password); + authReq.Method = "GET"; - if (string.IsNullOrEmpty(m_authToken) || string.IsNullOrEmpty(m_storageUrl)) - throw new Exception(Strings.CloudFiles.UnexpectedResponseError); + var areq = new AsyncHttpRequest(authReq); + using (var resp = areq.GetResponse()) + { + m_storageUrl = resp.Headers["X-Storage-Url"]; + m_authToken = resp.Headers["X-Auth-Token"]; + } + + if (string.IsNullOrEmpty(m_authToken) || string.IsNullOrEmpty(m_storageUrl)) + throw new Exception(Strings.CloudFiles.UnexpectedResponseError); + }).ConfigureAwait(false); } var req = (HttpWebRequest)HttpWebRequest.Create(m_storageUrl + UrlEncode(m_path + remotename) + query); - req.Headers.Add("X-Auth-Token", UrlEncode(m_authToken)); + req.Headers.Add("X-Auth-Token", UrlEncode(m_authToken!)); req.UserAgent = "Duplicati CloudFiles Backend v" + System.Reflection.Assembly.GetExecutingAssembly().GetName().Version; req.KeepAlive = false; @@ -435,9 +439,9 @@ namespace Duplicati.Library.Backend return req; } - private string UrlEncode(string value) + private static string UrlEncode(string value) { - return Library.Utility.Uri.UrlEncode(value).Replace("+", "%20").Replace("%2f", "/"); + return Utility.Uri.UrlEncode(value).Replace("+", "%20").Replace("%2f", "/"); } } } diff --git a/Duplicati/Library/Backend/CloudFiles/Duplicati.Library.Backend.CloudFiles.csproj b/Duplicati/Library/Backend/CloudFiles/Duplicati.Library.Backend.CloudFiles.csproj index 3c138a37d..9f355da46 100644 --- a/Duplicati/Library/Backend/CloudFiles/Duplicati.Library.Backend.CloudFiles.csproj +++ b/Duplicati/Library/Backend/CloudFiles/Duplicati.Library.Backend.CloudFiles.csproj @@ -5,6 +5,8 @@ Library Duplicati.Library.Backend Copyright © 2025 Team Duplicati, MIT license + enable + enable diff --git a/Duplicati/Library/Backend/CloudFiles/Strings.cs b/Duplicati/Library/Backend/CloudFiles/Strings.cs index 7c66be516..acb5dc405 100644 --- a/Duplicati/Library/Backend/CloudFiles/Strings.cs +++ b/Duplicati/Library/Backend/CloudFiles/Strings.cs @@ -1,34 +1,32 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. using Duplicati.Library.Localization.Short; -namespace Duplicati.Library.Backend.Strings { - internal static class CloudFiles { +namespace Duplicati.Library.Backend.Strings +{ + internal static class CloudFiles + { public static string Description_v2 { get { return LC.L(@"This backend can read and write data to CloudFiles. Allowed format is ""cloudfiles://container/folder""."); } } public static string DisplayName { get { return LC.L(@"Rackspace CloudFiles"); } } public static string DescriptionAuthenticationURLLong_v2(string optionname) { return LC.L(@"CloudFiles use different servers for authentication based on where the account resides. Use this option to set an alternate authentication URL. This option overrides --{0}.", optionname); } public static string DescriptionAuthenticationURLShort { get { return LC.L(@"Provide another authentication URL"); } } - public static string DescriptionAuthPasswordLong { get { return LC.L(@"The password used to connect to the server. This may also be supplied as the environment variable ""AUTH_PASSWORD""."); } } - public static string DescriptionAuthPasswordShort { get { return LC.L(@"Supply the password used to connect to the server"); } } - public static string DescriptionAuthUsernameLong { get { return LC.L(@"The username used to connect to the server. This may also be supplied as the environment variable ""AUTH_USERNAME""."); } } - public static string DescriptionAuthUsernameShort { get { return LC.L(@"Supply the username used to connect to the server"); } } public static string DescriptionPasswordLong { get { return LC.L(@"The API Access Key used to authenticate with CloudFiles."); } } public static string DescriptionPasswordShort { get { return LC.L(@"Supply the access key used to connect to the server"); } } public static string DescriptionUKAccountLong(string optionname, string optionvalue) { return LC.L(@"Duplicati will assume that the credentials given are for a US account. Use this option if the account is a UK based account. Note that this is equivalent to setting --{0}={1}.", optionname, optionvalue); } @@ -41,5 +39,6 @@ namespace Duplicati.Library.Backend.Strings { public static string NoAPIKeyError { get { return LC.L(@"No CloudFiles API Access Key given"); } } public static string NoUserIDError { get { return LC.L(@"No CloudFiles userID given"); } } public static string UnexpectedResponseError { get { return LC.L(@"Unexpected CloudFiles response. Perhaps the API has changed?"); } } + public static string DeprecatedFormatWarning(string hostname) { return LC.L(@$"The URL is using the marker hostname {hostname} which is no longer supported. Please use the new format ""cloudfiles://container/folder"" instead."); } } } diff --git a/Duplicati/Library/SecretProvider/SecretProviderModules.cs b/Duplicati/Library/SecretProvider/SecretProviderModules.cs index 5dce81ee4..4b1f18e5e 100644 --- a/Duplicati/Library/SecretProvider/SecretProviderModules.cs +++ b/Duplicati/Library/SecretProvider/SecretProviderModules.cs @@ -20,6 +20,7 @@ // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Interface; +using Duplicati.Library.Utility; namespace Duplicati.Library.SecretProvider; @@ -40,7 +41,6 @@ public static class SecretProviderModules OperatingSystem.IsLinux() ? new UnixPassProvider() : null, OperatingSystem.IsLinux() ? new LibSecretLinuxProvider() : null, } - .Where(x => x != null) - .Select(x => x!) + .WhereNotNull() .ToList(); } diff --git a/Duplicati/Library/Utility/EnumerableExtensions.cs b/Duplicati/Library/Utility/EnumerableExtensions.cs new file mode 100644 index 000000000..97417d278 --- /dev/null +++ b/Duplicati/Library/Utility/EnumerableExtensions.cs @@ -0,0 +1,51 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +using System.Collections.Generic; +using System.Linq; + +namespace Duplicati.Library.Utility; + +/// +/// Contains extension methods for . +/// +public static class EnumerableExtensions +{ + /// + /// Filters a sequence of values to exclude null elements. + /// + /// The sequence to filter. + /// A sequence that contains only the non-null elements from the input sequence. + public static IEnumerable WhereNotNull(this IEnumerable source) + => source.Where(x => x != null) + .Select(x => x!); + + /// + /// Filters a sequence of values to exclude null or whitespace elements. + /// + /// The sequence to filter. + /// A sequence that contains only the non-null and non-whitespace elements from the input sequence. + public static IEnumerable WhereNotNullOrWhiteSpace(this IEnumerable source) + => source.Where(x => !string.IsNullOrWhiteSpace(x)) + .Select(x => x!); +} diff --git a/Duplicati/Library/Utility/Options/AuthOptionsHelper.cs b/Duplicati/Library/Utility/Options/AuthOptionsHelper.cs index 337900f9e..bcd06b2c6 100644 --- a/Duplicati/Library/Utility/Options/AuthOptionsHelper.cs +++ b/Duplicati/Library/Utility/Options/AuthOptionsHelper.cs @@ -192,6 +192,16 @@ public static class AuthOptionsHelper return this; } + + /// + /// Returns the username and password after checking that they are set + /// + /// The username and password + public (string Username, string Password) GetCredentials() + { + RequireCredentials(); + return (Username!, Password!); + } } } diff --git a/Duplicati/WebserverCore/Endpoints/V1/Filesystem.cs b/Duplicati/WebserverCore/Endpoints/V1/Filesystem.cs index bb1e9509d..ca64d1fc2 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/Filesystem.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/Filesystem.cs @@ -20,6 +20,7 @@ // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Common.IO; using Duplicati.Library.Snapshots; +using Duplicati.Library.Utility; using Duplicati.Server; using Duplicati.WebserverCore.Abstractions; using Duplicati.WebserverCore.Exceptions; @@ -110,8 +111,7 @@ public class Filesystem : IEndpointV1 && di.IsReady // Only try to create TreeNode entries for drives who were ready 'now' ) .Select(TryCreateTreeNodeForDrive) // This will try to create a TreeNode for selected drives - .Where(tn => tn != null) // This filters out such entries that could not be created - .Select(x => x!); + .WhereNotNull(); // This filters out such entries that could not be created } else { From 1336cdbbae42e2e3076b17a989ccfb9b720403bf Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 21 Mar 2025 10:26:01 +0100 Subject: [PATCH 14/48] Rename CIFS to SMB with fallback --- Duplicati.sln | 2 +- Duplicati/Library/Backend/SMB/CIFSBackend.cs | 65 +++++++++++++++++++ .../Duplicati.Library.Backend.SMB.csproj} | 2 +- .../Model/SMBConnectionParameters.cs | 2 +- .../{CIFS/CIFS.cs => SMB/SMBBackend.cs} | 18 ++--- .../{CIFS => SMB}/SMBShareConnection.cs | 12 +--- .../Library/Backend/{CIFS => SMB}/Strings.cs | 2 +- Duplicati/Library/Backends/BackendModules.cs | 3 +- .../Duplicati.Library.Backends.csproj | 2 +- .../ngax/scripts/services/EditUriBuiltins.js | 20 ++++-- .../ngax/scripts/services/SystemInfo.js | 2 +- .../backends/{cifs.html => smb.html} | 0 .../Duplicati.Backend.Tests/CIFS/CIFSTests.cs | 14 ++-- 13 files changed, 105 insertions(+), 39 deletions(-) create mode 100644 Duplicati/Library/Backend/SMB/CIFSBackend.cs rename Duplicati/Library/Backend/{CIFS/Duplicati.Library.Backend.CIFS.csproj => SMB/Duplicati.Library.Backend.SMB.csproj} (94%) rename Duplicati/Library/Backend/{CIFS => SMB}/Model/SMBConnectionParameters.cs (97%) rename Duplicati/Library/Backend/{CIFS/CIFS.cs => SMB/SMBBackend.cs} (96%) rename Duplicati/Library/Backend/{CIFS => SMB}/SMBShareConnection.cs (98%) rename Duplicati/Library/Backend/{CIFS => SMB}/Strings.cs (98%) rename Duplicati/Server/webroot/ngax/templates/backends/{cifs.html => smb.html} (100%) diff --git a/Duplicati.sln b/Duplicati.sln index e9f8d2405..729e86905 100644 --- a/Duplicati.sln +++ b/Duplicati.sln @@ -164,7 +164,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.Backend.p EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.Crashlog", "Duplicati\Library\Crashlog\Duplicati.Library.Crashlog.csproj", "{8ACA2736-3C69-4FA5-BCF9-5EDF50CAF332}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.Backend.CIFS", "Duplicati\Library\Backend\CIFS\Duplicati.Library.Backend.CIFS.csproj", "{836E0557-B40C-4DC7-9A2A-5C062F9ACC6B}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.Backend.SMB", "Duplicati\Library\Backend\SMB\Duplicati.Library.Backend.SMB.csproj", "{836E0557-B40C-4DC7-9A2A-5C062F9ACC6B}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.SourceProviders", "Duplicati\Library\SourceProviders\Duplicati.Library.SourceProviders.csproj", "{8A2C7A9F-3EC8-4DE5-A9FA-9E4BCF955EF3}" EndProject diff --git a/Duplicati/Library/Backend/SMB/CIFSBackend.cs b/Duplicati/Library/Backend/SMB/CIFSBackend.cs new file mode 100644 index 000000000..425b35fcf --- /dev/null +++ b/Duplicati/Library/Backend/SMB/CIFSBackend.cs @@ -0,0 +1,65 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +namespace Duplicati.Library.Backend; + +/// +/// Native CIFS/SMB Backend implementation +/// +public class CIFSBackend : SMBBackend +{ + /// + /// Log tag for the backend + /// + public static readonly string LOGTAG = Logging.Log.LogTagFromType(); + /// + /// Gets the protocol key for the backend + /// + public override string ProtocolKey => "cifs"; + + /// + /// Gets the display name for the backend + /// + public override string DisplayName => "CIFS (deprecated)"; + + /// + /// Gets the description for the backend + /// + public override string Description => "Same as SMB backend, but with a different name. Use SMB instead."; + + /// + /// Empty constructor is required for the backend to be loaded by the backend factory + /// + public CIFSBackend() : base() + { + } + + /// + /// Actual constructor for the backend that accepts the url and options + /// + /// URL in Duplicati Uri format + /// options to be used in the backend + public CIFSBackend(string url, Dictionary options) + : base(url, options) + { + Logging.Log.WriteWarningMessage(LOGTAG, "DeprecatedCIFSBackend", null, "The CIFS backend is deprecated, please use the SMB backend instead."); + } +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/CIFS/Duplicati.Library.Backend.CIFS.csproj b/Duplicati/Library/Backend/SMB/Duplicati.Library.Backend.SMB.csproj similarity index 94% rename from Duplicati/Library/Backend/CIFS/Duplicati.Library.Backend.CIFS.csproj rename to Duplicati/Library/Backend/SMB/Duplicati.Library.Backend.SMB.csproj index 3dbadb34f..41f97d11f 100644 --- a/Duplicati/Library/Backend/CIFS/Duplicati.Library.Backend.CIFS.csproj +++ b/Duplicati/Library/Backend/SMB/Duplicati.Library.Backend.SMB.csproj @@ -6,7 +6,7 @@ Copyright © 2025 Team Duplicati, MIT license enable enable - Duplicati.Library.Backend.CIFS + Duplicati.Library.Backend.SMB diff --git a/Duplicati/Library/Backend/CIFS/Model/SMBConnectionParameters.cs b/Duplicati/Library/Backend/SMB/Model/SMBConnectionParameters.cs similarity index 97% rename from Duplicati/Library/Backend/CIFS/Model/SMBConnectionParameters.cs rename to Duplicati/Library/Backend/SMB/Model/SMBConnectionParameters.cs index 044b03cdd..85a66c43a 100644 --- a/Duplicati/Library/Backend/CIFS/Model/SMBConnectionParameters.cs +++ b/Duplicati/Library/Backend/SMB/Model/SMBConnectionParameters.cs @@ -21,7 +21,7 @@ using SMBLibrary; -namespace Duplicati.Library.Backend.CIFS.Model; +namespace Duplicati.Library.Backend.SMB.Model; /// /// Connection parameters for establishing an SMB connection. diff --git a/Duplicati/Library/Backend/CIFS/CIFS.cs b/Duplicati/Library/Backend/SMB/SMBBackend.cs similarity index 96% rename from Duplicati/Library/Backend/CIFS/CIFS.cs rename to Duplicati/Library/Backend/SMB/SMBBackend.cs index 2a98f6521..4c8cfadac 100644 --- a/Duplicati/Library/Backend/CIFS/CIFS.cs +++ b/Duplicati/Library/Backend/SMB/SMBBackend.cs @@ -20,8 +20,8 @@ // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Interface; -using Duplicati.Library.Backend.CIFS; -using Duplicati.Library.Backend.CIFS.Model; +using Duplicati.Library.Backend.SMB; +using Duplicati.Library.Backend.SMB.Model; using SMBLibrary; using Duplicati.Library.SourceProvider; using System.Runtime.CompilerServices; @@ -33,22 +33,22 @@ namespace Duplicati.Library.Backend; /// /// Native CIFS/SMB Backend implementation /// -public class CIFSBackend : IStreamingBackend, IFolderEnabledBackend +public class SMBBackend : IStreamingBackend, IFolderEnabledBackend { /// /// Implementation of interface property for the backend key /// - public string ProtocolKey => "cifs"; + public virtual string ProtocolKey => "smb"; /// /// Implementation of interface property for the backend display name /// - public string DisplayName => Strings.CIFSBackend.DisplayName; + public virtual string DisplayName => Strings.SMBBackend.DisplayName; /// /// Implementation of interface property for the backend description /// - public string Description => Strings.CIFSBackend.Description; + public virtual string Description => Strings.SMBBackend.Description; /// /// Hostname only (no ports or paths) to be used on DNS resolutions. @@ -112,7 +112,7 @@ public class CIFSBackend : IStreamingBackend, IFolderEnabledBackend /// /// Empty constructor is required for the backend to be loaded by the backend factory /// - public CIFSBackend() + public SMBBackend() { _DnsName = null!; _connectionParameters = null!; @@ -124,7 +124,7 @@ public class CIFSBackend : IStreamingBackend, IFolderEnabledBackend /// /// URL in Duplicati Uri format /// options to be used in the backend - public CIFSBackend(string url, Dictionary options) + public SMBBackend(string url, Dictionary options) { if (string.IsNullOrEmpty(url)) throw new ArgumentNullException(nameof(url)); @@ -182,7 +182,7 @@ public class CIFSBackend : IStreamingBackend, IFolderEnabledBackend public IList SupportedCommands => [ .. AuthOptionsHelper.GetOptions(), - new CommandLineArgument(AUTH_DOMAIN_OPTION, CommandLineArgument.ArgumentType.String, Strings.CIFSBackend.DescriptionAuthDomainShort, Strings.CIFSBackend.DescriptionAuthDomainLong), + new CommandLineArgument(AUTH_DOMAIN_OPTION, CommandLineArgument.ArgumentType.String, Strings.SMBBackend.DescriptionAuthDomainShort, Strings.SMBBackend.DescriptionAuthDomainLong), new CommandLineArgument(TRANSPORT_OPTION, CommandLineArgument.ArgumentType.Enumeration, Strings.Options.TransportShort, Strings.Options.TransportLong, DEFAULT_TRANSPORT, null, _transportMap.Keys.ToArray()), new CommandLineArgument(READ_BUFFER_SIZE_OPTION, CommandLineArgument.ArgumentType.String, Strings.Options.DescriptionReadBufferSizeShort, Strings.Options.DescriptionReadBufferSizeLong), new CommandLineArgument(WRITE_BUFFER_SIZE_OPTION, CommandLineArgument.ArgumentType.String, Strings.Options.DescriptionWriteBufferSizeShort, Strings.Options.DescriptionWriteBufferSizeLong), diff --git a/Duplicati/Library/Backend/CIFS/SMBShareConnection.cs b/Duplicati/Library/Backend/SMB/SMBShareConnection.cs similarity index 98% rename from Duplicati/Library/Backend/CIFS/SMBShareConnection.cs rename to Duplicati/Library/Backend/SMB/SMBShareConnection.cs index 0226811cd..a77bafd12 100644 --- a/Duplicati/Library/Backend/CIFS/SMBShareConnection.cs +++ b/Duplicati/Library/Backend/SMB/SMBShareConnection.cs @@ -19,15 +19,7 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -#nullable enable - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Duplicati.Library.Backend.CIFS.Model; +using Duplicati.Library.Backend.SMB.Model; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using Duplicati.Library.Localization.Short; @@ -37,7 +29,7 @@ using SMBLibrary; using SMBLibrary.Client; using FileAttributes = SMBLibrary.FileAttributes; -namespace Duplicati.Library.Backend.CIFS; +namespace Duplicati.Library.Backend.SMB; /// /// Class the wraps the SMB connection and file store objects, handling the connection, diff --git a/Duplicati/Library/Backend/CIFS/Strings.cs b/Duplicati/Library/Backend/SMB/Strings.cs similarity index 98% rename from Duplicati/Library/Backend/CIFS/Strings.cs rename to Duplicati/Library/Backend/SMB/Strings.cs index 2457dcf9d..e0567cd11 100644 --- a/Duplicati/Library/Backend/CIFS/Strings.cs +++ b/Duplicati/Library/Backend/SMB/Strings.cs @@ -23,7 +23,7 @@ using Duplicati.Library.Localization.Short; namespace Duplicati.Library.Backend.Strings { - internal static class CIFSBackend + internal static class SMBBackend { public static string DescriptionAuthDomainLong => LC.L(@"The domain used to connect to the server. This may also be supplied as the environment variable ""AUTH_DOMAIN""."); diff --git a/Duplicati/Library/Backends/BackendModules.cs b/Duplicati/Library/Backends/BackendModules.cs index c5c2cd714..f2cf6fc1d 100644 --- a/Duplicati/Library/Backends/BackendModules.cs +++ b/Duplicati/Library/Backends/BackendModules.cs @@ -67,7 +67,8 @@ public static class BackendModules new Backend.TencentCOS.COS(), new Backend.WEBDAV(), new Backend.pCloudBackend(), - new Backend.CIFSBackend() + new Backend.SMBBackend(), + new Backend.CIFSBackend(), } .Where(x => x != null) .ToList(); diff --git a/Duplicati/Library/Backends/Duplicati.Library.Backends.csproj b/Duplicati/Library/Backends/Duplicati.Library.Backends.csproj index e3d85f6b3..4540465be 100644 --- a/Duplicati/Library/Backends/Duplicati.Library.Backends.csproj +++ b/Duplicati/Library/Backends/Duplicati.Library.Backends.csproj @@ -10,7 +10,7 @@ - + diff --git a/Duplicati/Server/webroot/ngax/scripts/services/EditUriBuiltins.js b/Duplicati/Server/webroot/ngax/scripts/services/EditUriBuiltins.js index 94f8251ac..c6af89a3a 100644 --- a/Duplicati/Server/webroot/ngax/scripts/services/EditUriBuiltins.js +++ b/Duplicati/Server/webroot/ngax/scripts/services/EditUriBuiltins.js @@ -35,8 +35,9 @@ backupApp.service('EditUriBuiltins', function (AppService, AppUtils, SystemInfo, EditUriBackendConfig.templates['aliyunoss'] = 'templates/backends/aliyunoss.html'; EditUriBackendConfig.templates['e2'] = 'templates/backends/e2.html'; EditUriBackendConfig.templates['pcloud'] = 'templates/backends/pcloud.html'; - EditUriBackendConfig.templates['cifs'] = 'templates/backends/cifs.html'; - + EditUriBackendConfig.templates['smb'] = 'templates/backends/smb.html'; + EditUriBackendConfig.templates['cifs'] = 'templates/backends/smb.html'; + EditUriBackendConfig.testers['s3'] = function(scope, callback) { if (scope.s3_server != 's3.amazonaws.com') @@ -296,11 +297,12 @@ backupApp.service('EditUriBuiltins', function (AppService, AppUtils, SystemInfo, } return this['oauth-base'].apply(this, arguments); }; - EditUriBackendConfig.loaders['cifs'] = function(scope) { + EditUriBackendConfig.loaders['smb'] = function(scope) { if (scope.Transport === undefined || scope.Transport === '') { scope.Transport = 'directtcp'; } }; + EditUriBackendConfig.loaders['cifs'] = EditUriBackendConfig.loaders['smb']; EditUriBackendConfig.loaders['openstack'] = function (scope) { if (scope.openstack_providers == null) { @@ -614,7 +616,7 @@ backupApp.service('EditUriBuiltins', function (AppService, AppUtils, SystemInfo, EditUriBackendConfig.mergeServerAndPath(scope); } - EditUriBackendConfig.parsers['cifs'] = function (scope, module, server, path, port, options) { + EditUriBackendConfig.parsers['smb'] = function (scope, module, server, path, port, options) { if (options['--transport']) scope.Transport = options['--transport']; else @@ -633,6 +635,8 @@ backupApp.service('EditUriBuiltins', function (AppService, AppUtils, SystemInfo, }; + EditUriBackendConfig.parsers['cifs'] = EditUriBackendConfig.parsers['smb']; + // Builders take the scope and produce the uri output EditUriBackendConfig.builders['s3'] = function (scope) { @@ -976,7 +980,7 @@ backupApp.service('EditUriBuiltins', function (AppService, AppUtils, SystemInfo, return url; } - EditUriBackendConfig.builders['cifs'] = function (scope) { + EditUriBackendConfig.builders['smb'] = function (scope) { var opts = { 'transport': scope.Transport, 'auth-domain': scope.Domain @@ -1001,12 +1005,16 @@ backupApp.service('EditUriBuiltins', function (AppService, AppUtils, SystemInfo, ); } - EditUriBackendConfig.validaters['cifs'] = function (scope, continuation) { + EditUriBackendConfig.builders['cifs'] = EditUriBackendConfig.builders['smb']; + + EditUriBackendConfig.validaters['smb'] = function (scope, continuation) { if (EditUriBackendConfig.require_server(scope) && EditUriBackendConfig.require_username(scope) && EditUriBackendConfig.require_field(scope, 'ShareName', gettextCatalog.getString('Share name'))) continuation(); }; + + EditUriBackendConfig.validaters['smb'] = EditUriBackendConfig.validaters['cifs']; EditUriBackendConfig.validaters['file'] = function (scope, continuation) { if (EditUriBackendConfig.require_path(scope)) diff --git a/Duplicati/Server/webroot/ngax/scripts/services/SystemInfo.js b/Duplicati/Server/webroot/ngax/scripts/services/SystemInfo.js index 4180ca0e5..1a0c76ee4 100644 --- a/Duplicati/Server/webroot/ngax/scripts/services/SystemInfo.js +++ b/Duplicati/Server/webroot/ngax/scripts/services/SystemInfo.js @@ -50,7 +50,7 @@ backupApp.service('SystemInfo', function($rootScope, $timeout, $cookies, AppServ 'openstack': gettextCatalog.getString('OpenStack Object Storage / Swift'), 's3': gettextCatalog.getString('S3 Compatible'), 'aftp': gettextCatalog.getString('FTP (Alternative)'), - 'cifs': gettextCatalog.getString('CFIS / SMB'), + 'smb': gettextCatalog.getString('SMB / CIFS'), }, local: {'file': null}, prop: { diff --git a/Duplicati/Server/webroot/ngax/templates/backends/cifs.html b/Duplicati/Server/webroot/ngax/templates/backends/smb.html similarity index 100% rename from Duplicati/Server/webroot/ngax/templates/backends/cifs.html rename to Duplicati/Server/webroot/ngax/templates/backends/smb.html diff --git a/LiveTests/Duplicati.Backend.Tests/CIFS/CIFSTests.cs b/LiveTests/Duplicati.Backend.Tests/CIFS/CIFSTests.cs index d7547f52b..375435cad 100644 --- a/LiveTests/Duplicati.Backend.Tests/CIFS/CIFSTests.cs +++ b/LiveTests/Duplicati.Backend.Tests/CIFS/CIFSTests.cs @@ -21,22 +21,22 @@ using DotNet.Testcontainers.Images; -namespace Duplicati.Backend.Tests.CIFS; +namespace Duplicati.Backend.Tests.SMB; /// -/// CIFS Tests +/// SMB Tests /// [TestClass] -public sealed class CIFSTests : BaseSftpgoTest +public sealed class SMBTests : BaseSftpgoTest { /// - /// Test CIFS with TestContainers creating a Samba Server with TestContainers. + /// Test SMB with TestContainers creating a Samba Server with TestContainers. /// /// This test has no requirement of environment variables. /// [TestMethod] - public async Task TestCIFS() + public async Task TestSMB() { var outputConsumer = new OutputConsumer(); var randomPassword = GeneratePassword(); @@ -73,7 +73,7 @@ smbd --foreground --no-process-group --debug-stdout"; var container = new ContainerBuilder() .WithImage("ubuntu:22.04") - .WithImagePullPolicy(PullPolicy.Missing) + .WithImagePullPolicy(PullPolicy.Missing) .WithCommand("/bin/bash", "-c", "apt-get update && " + "DEBIAN_FRONTEND=noninteractive apt-get install -y samba && " + "bash /etc/samba/entrypoint.sh") @@ -92,7 +92,7 @@ smbd --foreground --no-process-group --debug-stdout"; var exitCode = CommandLine.BackendTester.Program.Main( new[] { - $"cifs://localhost/testshare1/new/?transport=directtcp&auth-domain&auth-username=smbuser1&auth-password={randomPassword}", + $"sMB://localhost/testshare1/new/?transport=directtcp&auth-domain&auth-username=smbuser1&auth-password={randomPassword}", }.Concat(Parameters.GlobalTestParameters).ToArray()); Console.WriteLine(await outputConsumer.GetStreamsOutput()); From bf27f3d4d3675482f6dc9f9c6effc13bce25bf46 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 21 Mar 2025 10:49:36 +0100 Subject: [PATCH 15/48] Updated Dropbox backend --- Duplicati/Library/Backend/Dropbox/Dropbox.cs | 64 +++++------ .../Library/Backend/Dropbox/DropboxHelper.cs | 103 +++++++++++------- .../Duplicati.Library.Backend.Dropbox.csproj | 2 + Duplicati/Library/Backend/Dropbox/WebApi.cs | 79 +++++--------- 4 files changed, 119 insertions(+), 129 deletions(-) diff --git a/Duplicati/Library/Backend/Dropbox/Dropbox.cs b/Duplicati/Library/Backend/Dropbox/Dropbox.cs index b68302980..18c044dbc 100644 --- a/Duplicati/Library/Backend/Dropbox/Dropbox.cs +++ b/Duplicati/Library/Backend/Dropbox/Dropbox.cs @@ -20,12 +20,8 @@ // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; -using System; -using System.Collections.Generic; -using System.IO; +using Duplicati.Library.Utility.Options; using System.Runtime.CompilerServices; -using System.Threading; -using System.Threading.Tasks; namespace Duplicati.Library.Backend { @@ -33,9 +29,7 @@ namespace Duplicati.Library.Backend // This class is instantiated dynamically in the BackendLoader. public class Dropbox : IBackend, IStreamingBackend { - private const string AUTHID_OPTION = "authid"; - - private readonly string m_accesToken; + private static readonly string TOKEN_URL = OAuthHelper.OAUTH_LOGIN_URL("dropbox"); private readonly string m_path; private readonly DropboxHelper dbx; @@ -43,25 +37,27 @@ namespace Duplicati.Library.Backend // This constructor is needed by the BackendLoader. public Dropbox() { + m_path = null!; + dbx = null!; } // ReSharper disable once UnusedMember.Global // This constructor is needed by the BackendLoader. - public Dropbox(string url, Dictionary options) + public Dropbox(string url, Dictionary options) { var uri = new Utility.Uri(url); - m_path = Library.Utility.Uri.UrlDecode(uri.HostAndPath); + m_path = Utility.Uri.UrlDecode(uri.HostAndPath); if (m_path.Length != 0 && !m_path.StartsWith("/", StringComparison.Ordinal)) m_path = "/" + m_path; if (m_path.EndsWith("/", StringComparison.Ordinal)) m_path = m_path.Substring(0, m_path.Length - 1); - if (options.ContainsKey(AUTHID_OPTION)) - m_accesToken = options[AUTHID_OPTION]; + var authId = AuthIdOptionsHelper.Parse(options); + authId.RequireCredentials(TOKEN_URL); - dbx = new DropboxHelper(m_accesToken); + dbx = new DropboxHelper(authId.AuthId!, TimeoutOptionsHelper.Parse(options)); } public void Dispose() @@ -69,15 +65,9 @@ namespace Duplicati.Library.Backend // do nothing } - public string DisplayName - { - get { return Strings.Dropbox.DisplayName; } - } + public string DisplayName => Strings.Dropbox.DisplayName; - public string ProtocolKey - { - get { return "dropbox"; } - } + public string ProtocolKey => "dropbox"; private IFileEntry ParseEntry(MetaData md) { @@ -92,7 +82,11 @@ namespace Duplicati.Library.Backend ife.IsFolder = true; } - try { ife.LastModification = ife.LastAccess = DateTime.Parse(md.server_modified).ToUniversalTime(); } + try + { + if (!string.IsNullOrWhiteSpace(md.server_modified)) + ife.LastModification = ife.LastAccess = DateTime.Parse(md.server_modified).ToUniversalTime(); + } catch { } return ife; @@ -106,7 +100,7 @@ namespace Duplicati.Library.Backend } catch (DropboxException de) { - if (de.errorJSON["error"][".tag"].ToString() == "path" && de.errorJSON["error"]["path"][".tag"].ToString() == "not_found") + if (de.errorJSON?["error"]?[".tag"]?.ToString() == "path" && de.errorJSON?["error"]?["path"]?[".tag"]?.ToString() == "not_found") throw new FolderMissingException(); throw; @@ -117,13 +111,13 @@ namespace Duplicati.Library.Backend { var lfr = await HandleListExceptions(() => dbx.ListFiles(m_path, cancelToken)).ConfigureAwait(false); - foreach (var md in lfr.entries) + foreach (var md in lfr.entries ?? []) yield return ParseEntry(md); while (lfr.has_more) { - lfr = await HandleListExceptions(() => dbx.ListFilesContinue(lfr.cursor, cancelToken)).ConfigureAwait(false); - foreach (var md in lfr.entries) + lfr = await HandleListExceptions(() => dbx.ListFilesContinue(lfr.cursor!, cancelToken)).ConfigureAwait(false); + foreach (var md in lfr.entries ?? []) yield return ParseEntry(md); } } @@ -144,7 +138,7 @@ namespace Duplicati.Library.Backend { try { - string path = String.Format("{0}/{1}", m_path, remotename); + var path = $"{m_path}/{remotename}"; await dbx.DeleteAsync(path, cancelToken).ConfigureAwait(false); } catch (DropboxException) @@ -154,15 +148,11 @@ namespace Duplicati.Library.Backend } } - public IList SupportedCommands - { - get - { - return new List([ - new CommandLineArgument(AUTHID_OPTION, CommandLineArgument.ArgumentType.Password, Strings.Dropbox.AuthidShort, Strings.Dropbox.AuthidLong(OAuthHelper.OAUTH_LOGIN_URL("dropbox"))), - ]); - } - } + public IList SupportedCommands => + [ + .. AuthIdOptionsHelper.GetOptions(TOKEN_URL), + .. TimeoutOptionsHelper.GetOptions(), + ]; public string Description { get { return Strings.Dropbox.Description; } } @@ -180,7 +170,7 @@ namespace Duplicati.Library.Backend catch (DropboxException de) { - if (de.errorJSON["error"][".tag"].ToString() == "path" && de.errorJSON["error"]["path"][".tag"].ToString() == "conflict") + if (de.errorJSON?["error"]?[".tag"]?.ToString() == "path" && de.errorJSON["error"]?["path"]?[".tag"]?.ToString() == "conflict") throw new FolderAreadyExistedException(); throw; } diff --git a/Duplicati/Library/Backend/Dropbox/DropboxHelper.cs b/Duplicati/Library/Backend/Dropbox/DropboxHelper.cs index c5e1bfe47..a068df2e5 100644 --- a/Duplicati/Library/Backend/Dropbox/DropboxHelper.cs +++ b/Duplicati/Library/Backend/Dropbox/DropboxHelper.cs @@ -18,14 +18,12 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. + using Duplicati.Library.Utility; +using Duplicati.Library.Utility.Options; using Newtonsoft.Json; using Newtonsoft.Json.Linq; -using System; -using System.IO; using System.Net; -using System.Threading; -using System.Threading.Tasks; namespace Duplicati.Library.Backend { @@ -34,9 +32,12 @@ namespace Duplicati.Library.Backend private const int DROPBOX_MAX_CHUNK_UPLOAD = 10 * 1024 * 1024; // 10 MB max upload private const string API_ARG_HEADER = "DROPBOX-API-arg"; - public DropboxHelper(string accessToken) + private readonly TimeoutOptionsHelper.Timeouts m_timeouts; + + public DropboxHelper(string accessToken, TimeoutOptionsHelper.Timeouts timeouts) : base(accessToken, "dropbox") { + m_timeouts = timeouts; base.AutoAuthHeader = true; // Pre 2022 tokens are direct Dropbox tokens (no ':') // Post 2022-02-21 tokens are regular authid tokens (with a ':') @@ -52,7 +53,9 @@ namespace Duplicati.Library.Backend try { - return await PostAndGetJSONDataAsync(WebApi.Dropbox.ListFilesUrl(), cancelToken, pa).ConfigureAwait(false); + return await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, ct => + PostAndGetJSONDataAsync(WebApi.Dropbox.ListFilesUrl(), ct, pa) + ).ConfigureAwait(false); } catch (Exception ex) { @@ -67,7 +70,9 @@ namespace Duplicati.Library.Backend try { - return await PostAndGetJSONDataAsync(WebApi.Dropbox.ListFilesContinueUrl(), cancelToken, lfca).ConfigureAwait(false); + return await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, ct => + PostAndGetJSONDataAsync(WebApi.Dropbox.ListFilesContinueUrl(), ct, lfca) + ).ConfigureAwait(false); } catch (Exception ex) { @@ -82,7 +87,9 @@ namespace Duplicati.Library.Backend try { - return await PostAndGetJSONDataAsync(WebApi.Dropbox.CreateFolderUrl(), cancellationToken, pa).ConfigureAwait(false); + return await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancellationToken, ct => + PostAndGetJSONDataAsync(WebApi.Dropbox.CreateFolderUrl(), ct, pa) + ).ConfigureAwait(false); } catch (Exception ex) { @@ -111,13 +118,14 @@ namespace Duplicati.Library.Backend ulong globalBytesRead = 0; using (var rs = areq.GetRequestStream()) + using (var ts = rs.ObserveWriteTimeout(m_timeouts.ReadWriteTimeout)) { int bytesRead = 0; do { bytesRead = await stream.ReadAsync(buffer, 0, sizeToRead, cancelToken).ConfigureAwait(false); globalBytesRead += (ulong)bytesRead; - await rs.WriteAsync(buffer, 0, bytesRead, cancelToken).ConfigureAwait(false); + await ts.WriteAsync(buffer, 0, bytesRead, cancelToken).ConfigureAwait(false); } while (bytesRead > 0 && globalBytesRead < (ulong)chunksize); } @@ -149,6 +157,7 @@ namespace Duplicati.Library.Backend int bytesReadInRequest = 0; sizeToRead = Math.Min(chunksize, (int)Utility.Utility.DEFAULT_BUFFER_SIZE); using (var rs = areq.GetRequestStream()) + using (var ts = rs.ObserveWriteTimeout(m_timeouts.ReadWriteTimeout)) { int bytesRead = 0; do @@ -156,15 +165,18 @@ namespace Duplicati.Library.Backend bytesRead = await stream.ReadAsync(buffer, 0, sizeToRead, cancelToken).ConfigureAwait(false); bytesReadInRequest += bytesRead; globalBytesRead += (ulong)bytesRead; - await rs.WriteAsync(buffer, 0, bytesRead, cancelToken).ConfigureAwait(false); + await ts.WriteAsync(buffer, 0, bytesRead, cancelToken).ConfigureAwait(false); } while (bytesRead > 0 && bytesReadInRequest < chunksize); } - using (var response = GetResponse(areq)) - using (var sr = new StreamReader(response.GetResponseStream())) - await sr.ReadToEndAsync().ConfigureAwait(false); + await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, async ct => + { + using (var response = GetResponse(areq)) + using (var sr = new StreamReader(response.GetResponseStream())) + await sr.ReadToEndAsync().ConfigureAwait(false); + }).ConfigureAwait(false); } // finish session and commit @@ -180,7 +192,9 @@ namespace Duplicati.Library.Backend req.ContentType = "application/octet-stream"; req.Timeout = 200000; - return ReadJSONResponse(req); + return await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => + ReadJSONResponseAsync(req, ct) + ).ConfigureAwait(false); } catch (Exception ex) { @@ -198,8 +212,10 @@ namespace Duplicati.Library.Backend var req = CreateRequest(WebApi.Dropbox.DownloadFilesUrl(), "POST"); req.Headers[API_ARG_HEADER] = JsonConvert.SerializeObject(pa); - using (var response = GetResponse(req)) - await Utility.Utility.CopyStreamAsync(response.GetResponseStream(), fs, cancelToken).ConfigureAwait(false); + using (var response = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => GetResponse(req))) + using (var rs = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => response.GetResponseStream())) + using (var ts = rs.ObserveReadTimeout(m_timeouts.ReadWriteTimeout)) + await Utility.Utility.CopyStreamAsync(ts, fs, cancelToken).ConfigureAwait(false); } catch (Exception ex) { @@ -212,10 +228,13 @@ namespace Duplicati.Library.Backend { try { - var pa = new PathArg() { path = path }; - using (var response = await GetResponseAsync(WebApi.Dropbox.DeleteUrl(), cancelToken, pa).ConfigureAwait(false)) - using (var sr = new StreamReader(response.GetResponseStream())) - sr.ReadToEnd(); + await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, async ct => + { + var pa = new PathArg() { path = path }; + using (var response = await GetResponseAsync(WebApi.Dropbox.DeleteUrl(), ct, pa).ConfigureAwait(false)) + using (var sr = new StreamReader(response.GetResponseStream())) + await sr.ReadToEndAsync(ct).ConfigureAwait(false); + }).ConfigureAwait(false); } catch (Exception ex) { @@ -232,8 +251,9 @@ namespace Duplicati.Library.Backend try { - using (var sr = new StreamReader(exception.Response.GetResponseStream())) - json = sr.ReadToEnd(); + if (exception.Response != null) + using (var sr = new StreamReader(exception.Response.GetResponseStream())) + json = sr.ReadToEnd(); } catch { } @@ -245,27 +265,27 @@ namespace Duplicati.Library.Backend if (httpResp.StatusCode == HttpStatusCode.NotFound) { if (filerequest) - throw new Duplicati.Library.Interface.FileMissingException(json); + throw new Interface.FileMissingException(json); else - throw new Duplicati.Library.Interface.FolderMissingException(json); + throw new Interface.FolderMissingException(json); } if (httpResp.StatusCode == HttpStatusCode.Conflict) { //TODO: Should actually parse and see if something else happens if (filerequest) - throw new Duplicati.Library.Interface.FileMissingException(json); + throw new Interface.FileMissingException(json); else - throw new Duplicati.Library.Interface.FolderMissingException(json); + throw new Interface.FolderMissingException(json); } if (httpResp.StatusCode == HttpStatusCode.Unauthorized) ThrowAuthException(json, exception); if ((int)httpResp.StatusCode == 429 || (int)httpResp.StatusCode == 507) - throw new Duplicati.Library.Interface.UserInformationException(Strings.Dropbox.OverQuotaError(string.IsNullOrWhiteSpace(json) ? exception.Message : json), "DropboxOverQuotaError", ex); + throw new Interface.UserInformationException(Strings.Dropbox.OverQuotaError(string.IsNullOrWhiteSpace(json) ? exception.Message : json), "DropboxOverQuotaError", ex); } - JObject errJson = null; + JObject? errJson = null; try { errJson = JObject.Parse(json); @@ -286,12 +306,12 @@ namespace Duplicati.Library.Backend public class DropboxException : Exception { - public JObject errorJSON { get; set; } + public JObject? errorJSON { get; set; } } public class PathArg { - public string path { get; set; } + public string? path { get; set; } } public class FolderMetadata : MetaData @@ -332,7 +352,7 @@ namespace Duplicati.Library.Backend public class UploadSessionCursor { - public string session_id { get; set; } + public string? session_id { get; set; } public ulong offset { get; set; } } @@ -344,7 +364,7 @@ namespace Duplicati.Library.Backend autorename = false; mute = true; } - public string path { get; set; } + public string? path { get; set; } public string mode { get; set; } public bool autorename { get; set; } public bool mute { get; set; } @@ -353,41 +373,40 @@ namespace Duplicati.Library.Backend public class UploadSessionStartResult { - public string session_id { get; set; } + public string? session_id { get; set; } } public class ListFolderResult { + public MetaData[]? entries { get; set; } - public MetaData[] entries { get; set; } - - public string cursor { get; set; } + public string? cursor { get; set; } public bool has_more { get; set; } } public class ListFolderContinueArg { - public string cursor { get; set; } + public string? cursor { get; set; } } public class MetaData { [JsonProperty(".tag")] - public string tag { get; set; } - public string name { get; set; } - public string server_modified { get; set; } + public string? tag { get; set; } + public string? name { get; set; } + public string? server_modified { get; set; } public ulong size { get; set; } public bool IsFile { get { return tag == "file"; } } // While this is unused, the Dropbox API v2 documentation does not // declare this to be optional. // ReSharper disable once UnusedMember.Global - public string id { get; set; } + public string? id { get; set; } // While this is unused, the Dropbox API v2 documentation does not // declare this to be optional. // ReSharper disable once UnusedMember.Global - public string rev { get; set; } + public string? rev { get; set; } } public class FileMetaData : MetaData diff --git a/Duplicati/Library/Backend/Dropbox/Duplicati.Library.Backend.Dropbox.csproj b/Duplicati/Library/Backend/Dropbox/Duplicati.Library.Backend.Dropbox.csproj index 1e3ba7574..84b5f8768 100644 --- a/Duplicati/Library/Backend/Dropbox/Duplicati.Library.Backend.Dropbox.csproj +++ b/Duplicati/Library/Backend/Dropbox/Duplicati.Library.Backend.Dropbox.csproj @@ -4,6 +4,8 @@ net8.0 Dropbox backend for Duplicati Copyright © 2025 Team Duplicati, MIT license + enable + enable diff --git a/Duplicati/Library/Backend/Dropbox/WebApi.cs b/Duplicati/Library/Backend/Dropbox/WebApi.cs index 563b436ef..acd91bcff 100644 --- a/Duplicati/Library/Backend/Dropbox/WebApi.cs +++ b/Duplicati/Library/Backend/Dropbox/WebApi.cs @@ -1,80 +1,59 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using Duplicati.Library.Utility; - namespace Duplicati.Library.Backend.WebApi { public static class Dropbox { public static string CreateFolderUrl() - { - return Uri.UriBuilder(Url.API, Path.CreateFolder); - } + => Utility.Uri.UriBuilder(Url.API, Path.CreateFolder); public static string ListFilesUrl() - { - return Uri.UriBuilder(Url.API, Path.ListFolder); - } + => Utility.Uri.UriBuilder(Url.API, Path.ListFolder); public static string ListFilesContinueUrl() - { - return Uri.UriBuilder(Url.API, Path.ListFolderContinue); - } + => Utility.Uri.UriBuilder(Url.API, Path.ListFolderContinue); public static string DeleteUrl() - { - return Uri.UriBuilder(Url.API, Path.DeleteFolder); - } + => Utility.Uri.UriBuilder(Url.API, Path.DeleteFolder); public static string UploadSessionStartUrl() - { - return Uri.UriBuilder(Url.CONTENT_API_URL, Path.UploadSessionStart); - } + => Utility.Uri.UriBuilder(Url.CONTENT_API_URL, Path.UploadSessionStart); public static string UploadSessionAppendUrl() - { - return Uri.UriBuilder(Url.CONTENT_API_URL, Path.UploadSessionAppend); - } + => Utility.Uri.UriBuilder(Url.CONTENT_API_URL, Path.UploadSessionAppend); public static string UploadSessionFinishUrl() - { - return Uri.UriBuilder(Url.CONTENT_API_URL, Path.UploadSessionFinish); - } + => Utility.Uri.UriBuilder(Url.CONTENT_API_URL, Path.UploadSessionFinish); public static string DownloadFilesUrl() - { - return Uri.UriBuilder(Url.CONTENT_API_URL, Path.DownloadFiles); - } + => Utility.Uri.UriBuilder(Url.CONTENT_API_URL, Path.DownloadFiles); public static string[] Hosts() - { - return new[] { new System.Uri(Url.API).Host, new System.Uri(Url.CONTENT_API_URL).Host }; - - } + => [new Uri(Url.API).Host, new Uri(Url.CONTENT_API_URL).Host]; private static class Url { public const string API = "https://api.dropboxapi.com/2"; - public const string CONTENT_API_URL = "https://content.dropboxapi.com/2"; + public const string CONTENT_API_URL = "https://content.dropboxapi.com/2"; } private static class Path From dbe3a4eeacd1ab2ecc27245a9d00fdacceea12bd Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 21 Mar 2025 11:09:23 +0100 Subject: [PATCH 16/48] Updated file backend --- .../Duplicati.Library.Backend.File.csproj | 2 + Duplicati/Library/Backend/File/FileBackend.cs | 123 +++++++++--------- Duplicati/Library/Backend/File/Strings.cs | 44 +++---- 3 files changed, 84 insertions(+), 85 deletions(-) diff --git a/Duplicati/Library/Backend/File/Duplicati.Library.Backend.File.csproj b/Duplicati/Library/Backend/File/Duplicati.Library.Backend.File.csproj index baa6fa128..e976a48ba 100644 --- a/Duplicati/Library/Backend/File/Duplicati.Library.Backend.File.csproj +++ b/Duplicati/Library/Backend/File/Duplicati.Library.Backend.File.csproj @@ -5,6 +5,8 @@ Library Duplicati.Library.Backend Copyright © 2025 Team Duplicati, MIT license + enable + enable diff --git a/Duplicati/Library/Backend/File/FileBackend.cs b/Duplicati/Library/Backend/File/FileBackend.cs index 3441940a6..2d3471d1f 100644 --- a/Duplicati/Library/Backend/File/FileBackend.cs +++ b/Duplicati/Library/Backend/File/FileBackend.cs @@ -21,14 +21,11 @@ using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; +using Duplicati.Library.Utility; +using Duplicati.Library.Utility.Options; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Versioning; -using System.Threading; -using System.Threading.Tasks; namespace Duplicati.Library.Backend { @@ -43,71 +40,65 @@ namespace Duplicati.Library.Backend private const string OPTION_DISABLE_LENGTH_VERIFICATION = "disable-length-verification"; private readonly string m_path; - private string m_username; - private string m_password; + private string? m_username; + private string? m_password; private readonly bool m_moveFile; private bool m_hasAutenticated; private readonly bool m_forceReauth; private readonly bool m_verifyDestinationLength; + private readonly TimeoutOptionsHelper.Timeouts m_timeouts; private static readonly ISystemIO systemIO = SystemIO.IO_OS; public File() { + m_path = null!; + m_timeouts = null!; } - public File(string url, Dictionary options) + public File(string url, Dictionary options) { var uri = new Utility.Uri(url); - m_path = uri.HostAndPath; + var path = uri.HostAndPath; + var auth = AuthOptionsHelper.Parse(options, uri); + m_timeouts = TimeoutOptionsHelper.Parse(options); + m_username = auth.Username; + m_password = auth.Password; - if (options.ContainsKey("auth-username")) - m_username = options["auth-username"]; - if (options.ContainsKey("auth-password")) - m_password = options["auth-password"]; - if (!string.IsNullOrEmpty(uri.Username)) - m_username = uri.Username; - if (!string.IsNullOrEmpty(uri.Password)) - m_password = uri.Password; + if (!Path.IsPathRooted(path)) + path = systemIO.PathGetFullPath(path); - if (!System.IO.Path.IsPathRooted(m_path)) - m_path = systemIO.PathGetFullPath(m_path); - - if (options.ContainsKey(OPTION_ALTERNATE_PATHS)) + var altPaths = options.GetValueOrDefault(OPTION_ALTERNATE_PATHS); + if (!string.IsNullOrWhiteSpace(altPaths)) { List paths = [ - m_path, - .. options[OPTION_ALTERNATE_PATHS].Split(new string[] { System.IO.Path.PathSeparator.ToString() }, StringSplitOptions.RemoveEmptyEntries), + path, + .. altPaths.Split(new string[] {Path.PathSeparator.ToString() }, StringSplitOptions.RemoveEmptyEntries), ]; //On windows we expand the drive letter * to all drives if (OperatingSystem.IsWindows()) { - System.IO.DriveInfo[] drives = System.IO.DriveInfo.GetDrives(); - - for (int i = 0; i < paths.Count; i++) + var drives = DriveInfo.GetDrives(); + for (var i = 0; i < paths.Count; i++) { if (paths[i].StartsWith("*:", StringComparison.Ordinal)) { - string rpl_path = paths[i].Substring(1); + var rpl_path = paths[i].Substring(1); paths.RemoveAt(i); i--; - foreach (System.IO.DriveInfo di in drives) + foreach (var di in drives) paths.Insert(++i, di.Name[0] + rpl_path); } } } - string markerfile = null; - //If there is a marker file, we do not allow the primary target path // to be accepted, unless it contains the marker file - if (options.ContainsKey(OPTION_DESTINATION_MARKER)) - { - markerfile = options[OPTION_DESTINATION_MARKER]; - m_path = null; - } + var markerfile = options.GetValueOrDefault(OPTION_DESTINATION_MARKER); + if (!string.IsNullOrWhiteSpace(markerfile)) + path = null; foreach (string p in paths) { @@ -115,7 +106,7 @@ namespace Duplicati.Library.Backend { if (systemIO.DirectoryExists(p) && (markerfile == null || systemIO.FileExists(systemIO.PathCombine(p, markerfile)))) { - m_path = p; + path = p; break; } } @@ -124,10 +115,11 @@ namespace Duplicati.Library.Backend } } - if (m_path == null) + if (string.IsNullOrWhiteSpace(path)) throw new UserInformationException(Strings.FileBackend.NoDestinationWithMarkerFileError(markerfile, paths.ToArray()), "NoDestinationWithMarker"); } + m_path = path; m_moveFile = Utility.Utility.ParseBoolOption(options, OPTION_MOVE_FILE); m_forceReauth = Utility.Utility.ParseBoolOption(options, OPTION_FORCE_REAUTH); m_verifyDestinationLength = !Utility.Utility.ParseBoolOption(options, OPTION_DISABLE_LENGTH_VERIFICATION); @@ -174,14 +166,19 @@ namespace Duplicati.Library.Backend } /// - public IAsyncEnumerable ListAsync(CancellationToken cancelToken) + public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { PreAuthenticate(); if (!systemIO.DirectoryExists(m_path)) throw new FolderMissingException(Strings.FileBackend.FolderMissingError(m_path)); - return systemIO.EnumerateFileEntries(m_path).ToAsyncEnumerable(); + var res = await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, _ => + systemIO.EnumerateFileEntries(m_path) + ).ConfigureAwait(false); + + foreach (var entry in res) + yield return entry; } #if DEBUG_RETRY @@ -201,17 +198,19 @@ namespace Duplicati.Library.Backend string targetFilePath = GetRemoteName(targetFilename); long copiedBytes = 0; using (var targetStream = systemIO.FileCreate(targetFilePath)) - copiedBytes = await Utility.Utility.CopyStreamAsync(sourceStream, targetStream, true, cancelToken).ConfigureAwait(false); + using (var timeoutStream = targetStream.ObserveWriteTimeout(m_timeouts.ReadWriteTimeout)) + copiedBytes = await Utility.Utility.CopyStreamAsync(sourceStream, timeoutStream, true, cancelToken).ConfigureAwait(false); VerifyMatchingSize(targetFilePath, sourceStream, copiedBytes); } #endif - public async Task GetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) + public async Task GetAsync(string remotename, Stream stream, CancellationToken cancelToken) { // FileOpenRead has flags System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read using (var readstream = systemIO.FileOpenRead(GetRemoteName(remotename))) - await Utility.Utility.CopyStreamAsync(readstream, stream, true, cancelToken).ConfigureAwait(false); + using (var timeoutStream = readstream.ObserveReadTimeout(m_timeouts.ReadWriteTimeout)) + await Utility.Utility.CopyStreamAsync(timeoutStream, stream, true, cancelToken).ConfigureAwait(false); } public Task PutAsync(string targetFilename, string sourceFilePath, CancellationToken cancelToken) @@ -245,10 +244,11 @@ namespace Duplicati.Library.Backend return Task.CompletedTask; } - public Task DeleteAsync(string remotename, CancellationToken cancelToken) + public async Task DeleteAsync(string remotename, CancellationToken cancelToken) { - systemIO.FileDelete(GetRemoteName(remotename)); - return Task.CompletedTask; + await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ + => systemIO.FileDelete(GetRemoteName(remotename)) + ).ConfigureAwait(false); } public IList SupportedCommands @@ -258,8 +258,7 @@ namespace Duplicati.Library.Backend var lst = new List(); if (OperatingSystem.IsWindows()) lst.AddRange([ - new CommandLineArgument("auth-username", CommandLineArgument.ArgumentType.String, Strings.FileBackend.DescriptionAuthUsernameShort, Strings.FileBackend.DescriptionAuthUsernameLong), - new CommandLineArgument("auth-password", CommandLineArgument.ArgumentType.Password, Strings.FileBackend.DescriptionAuthPasswordShort, Strings.FileBackend.DescriptionAuthPasswordLong), + .. AuthOptionsHelper.GetOptions(), new CommandLineArgument(OPTION_FORCE_REAUTH, CommandLineArgument.ArgumentType.Boolean, Strings.FileBackend.ForceReauthShort, Strings.FileBackend.ForceReauthLong) ]); @@ -268,6 +267,7 @@ namespace Duplicati.Library.Backend new CommandLineArgument(OPTION_ALTERNATE_PATHS, CommandLineArgument.ArgumentType.Path, Strings.FileBackend.AlternateTargetPathsShort, Strings.FileBackend.AlternateTargetPathsLong(OPTION_DESTINATION_MARKER, System.IO.Path.PathSeparator)), new CommandLineArgument(OPTION_MOVE_FILE, CommandLineArgument.ArgumentType.Boolean, Strings.FileBackend.UseMoveForPutShort, Strings.FileBackend.UseMoveForPutLong), new CommandLineArgument(OPTION_DISABLE_LENGTH_VERIFICATION, CommandLineArgument.ArgumentType.Boolean, Strings.FileBackend.DisableLengthVerificationShort, Strings.FileBackend.DisableLengthVerificationLong), + .. TimeoutOptionsHelper.GetOptions() ]); return lst; @@ -285,13 +285,14 @@ namespace Duplicati.Library.Backend public Task TestAsync(CancellationToken cancelToken) => this.TestListAsync(cancelToken); - public Task CreateFolderAsync(CancellationToken cancelToken) + public async Task CreateFolderAsync(CancellationToken cancelToken) { if (systemIO.DirectoryExists(m_path)) throw new FolderAreadyExistedException(); - systemIO.DirectoryCreate(m_path); - return Task.CompletedTask; + await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ + => systemIO.DirectoryCreate(m_path) + ).ConfigureAwait(false); } #endregion @@ -306,7 +307,7 @@ namespace Duplicati.Library.Backend #endregion - private System.IO.DriveInfo GetDrive() + private DriveInfo? GetDrive() { string root; if (!OperatingSystem.IsWindows()) @@ -314,7 +315,7 @@ namespace Duplicati.Library.Backend string path = Util.AppendDirSeparator(systemIO.PathGetFullPath(m_path)); // If the built-in .NET DriveInfo works, use it - try { return new System.IO.DriveInfo(path); } + try { return new DriveInfo(path); } catch { } root = "/"; @@ -322,7 +323,7 @@ namespace Duplicati.Library.Backend //Find longest common prefix from mounted devices //TODO: Can trick this with symlinks, where the symlink is on one mounted volume, // and the actual storage is on another - foreach (System.IO.DriveInfo di in System.IO.DriveInfo.GetDrives()) + foreach (var di in DriveInfo.GetDrives()) if (path.StartsWith(Util.AppendDirSeparator(di.Name), StringComparison.Ordinal) && di.Name.Length > root.Length) root = di.Name; } @@ -337,7 +338,7 @@ namespace Duplicati.Library.Backend { try { - return new System.IO.DriveInfo(root); + return new DriveInfo(root); } catch (ArgumentException) { @@ -348,7 +349,7 @@ namespace Duplicati.Library.Backend return null; } - public Task GetQuotaInfoAsync(CancellationToken cancelToken) + public Task GetQuotaInfoAsync(CancellationToken cancelToken) { var driveInfo = this.GetDrive(); if (driveInfo != null) @@ -357,7 +358,7 @@ namespace Duplicati.Library.Backend // If the drive actually has a total size of 0, this should be obvious immediately due to write errors if (driveInfo.TotalSize > 0) { - return Task.FromResult(new QuotaInfo(driveInfo.TotalSize, driveInfo.AvailableFreeSpace)); + return Task.FromResult(new QuotaInfo(driveInfo.TotalSize, driveInfo.AvailableFreeSpace)); } } @@ -365,10 +366,10 @@ namespace Duplicati.Library.Backend { // If we can't get the DriveInfo on Windows, fallback to GetFreeDiskSpaceEx // https://stackoverflow.com/questions/2050343/programmatically-determining-space-available-from-unc-path - return Task.FromResult(GetDiskFreeSpace(m_path)); + return Task.FromResult(GetDiskFreeSpace(m_path)); } - return null; + return Task.FromResult(null); } public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(Array.Empty()); @@ -391,7 +392,7 @@ namespace Duplicati.Library.Backend /// Quota info [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] [SupportedOSPlatform("windows")] - public static QuotaInfo GetDiskFreeSpace(string directory) + public static QuotaInfo? GetDiskFreeSpace(string directory) { ulong available; ulong total; @@ -439,7 +440,7 @@ namespace Duplicati.Library.Backend } } - private static void VerifyMatchingSize(string targetFilePath, Stream sourceStream, long? expectedLength) + private static void VerifyMatchingSize(string targetFilePath, Stream? sourceStream, long? expectedLength) { try { diff --git a/Duplicati/Library/Backend/File/Strings.cs b/Duplicati/Library/Backend/File/Strings.cs index 7963a9c69..47f367db2 100644 --- a/Duplicati/Library/Backend/File/Strings.cs +++ b/Duplicati/Library/Backend/File/Strings.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Localization.Short; namespace Duplicati.Library.Backend.Strings @@ -29,12 +29,8 @@ namespace Duplicati.Library.Backend.Strings public static string AlternateDestinationMarkerShort { get { return LC.L(@"Look for a file in the destination folder"); } } public static string AlternateTargetPathsLong(string optionname, char pathseparator) { return LC.L(@"This option allows multiple targets to be specified. The primary target path is placed before the list of paths supplied with this option. Before starting the backup, each folder in the list is checked for existence and optionally the presence of the marker file supplied by --{0}. The first existing path that optionally contains the marker file is then used as the destination. Multiple destinations are separated with a ""{1}"". On Windows, the path may be a UNC path, and the drive letter may be substituted with an asterisk (*), e.g: ""*:\backup"", which will examine all drive letters. If a username and password is supplied, the same credentials are used for all destinations.", optionname, pathseparator); } public static string AlternateTargetPathsShort { get { return LC.L(@"A list of secondary target paths"); } } - public static string DescriptionAuthPasswordLong { get { return LC.L(@"The password used to connect to the server. This may also be supplied as the environment variable ""AUTH_PASSWORD""."); } } - public static string DescriptionAuthPasswordShort { get { return LC.L(@"Supply the password used to connect to the server"); } } - public static string DescriptionAuthUsernameLong { get { return LC.L(@"The username used to connect to the server. This may also be supplied as the environment variable ""AUTH_USERNAME""."); } } - public static string DescriptionAuthUsernameShort { get { return LC.L(@"Supply the username used to connect to the server"); } } public static string FolderMissingError(string foldername) { return LC.L(@"The folder {0} does not exist", foldername); } - public static string NoDestinationWithMarkerFileError(string markername, string[] folders) { return LC.L(@"The marker file ""{0}"" was not found in any of the examined destinations: {1}", markername, string.Join(", ", folders)); } + public static string NoDestinationWithMarkerFileError(string? markername, string[] folders) { return LC.L(@"The marker file ""{0}"" was not found in any of the examined destinations: {1}", markername, string.Join(", ", folders)); } public static string UseMoveForPutLong { get { return LC.L(@"When storing the file, the standard operation is to copy the file and delete the original. This sequence ensures that the operation can be retried if something goes wrong. Activating this option may cause the retry operation to fail. This option has no effect unless the option --{0} is activated.", "disable-streaming-transfers"); } } public static string UseMoveForPutShort { get { return LC.L(@"Move the file instead of copying it"); } } public static string ForceReauthLong { get { return LC.L(@"If this option is set, any existing authentication against the remote share is dropped before attempting to authenticate."); } } From 57c9aab3e42bf7d7d73ddb7f1567e824c282d977 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 21 Mar 2025 11:10:40 +0100 Subject: [PATCH 17/48] Updated FTP backend --- Duplicati/Library/Backend/FTP/AlternateFTPBackend.cs | 4 +--- .../Backend/FTP/Duplicati.Library.Backend.FTP.csproj | 2 ++ Duplicati/Library/Backend/FTP/FTPBackend.cs | 8 -------- Duplicati/Library/Backend/FTP/Strings.cs | 2 +- 4 files changed, 4 insertions(+), 12 deletions(-) diff --git a/Duplicati/Library/Backend/FTP/AlternateFTPBackend.cs b/Duplicati/Library/Backend/FTP/AlternateFTPBackend.cs index ce3e1ffe8..85c56d74b 100644 --- a/Duplicati/Library/Backend/FTP/AlternateFTPBackend.cs +++ b/Duplicati/Library/Backend/FTP/AlternateFTPBackend.cs @@ -18,8 +18,6 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System; -using System.Collections.Generic; using System.Security.Authentication; using Duplicati.Library.Interface; using Duplicati.Library.Utility.Options; @@ -61,7 +59,7 @@ public class AlternateFTPBackend : FTP { } - public AlternateFTPBackend(string url, Dictionary options) : base(url, options) + public AlternateFTPBackend(string url, Dictionary options) : base(url, options) { } diff --git a/Duplicati/Library/Backend/FTP/Duplicati.Library.Backend.FTP.csproj b/Duplicati/Library/Backend/FTP/Duplicati.Library.Backend.FTP.csproj index 3f6aade5c..f526a1c71 100644 --- a/Duplicati/Library/Backend/FTP/Duplicati.Library.Backend.FTP.csproj +++ b/Duplicati/Library/Backend/FTP/Duplicati.Library.Backend.FTP.csproj @@ -4,6 +4,8 @@ net8.0 Library Copyright © 2025 Team Duplicati, MIT license + enable + enable diff --git a/Duplicati/Library/Backend/FTP/FTPBackend.cs b/Duplicati/Library/Backend/FTP/FTPBackend.cs index b9609801a..ecb87bae4 100644 --- a/Duplicati/Library/Backend/FTP/FTPBackend.cs +++ b/Duplicati/Library/Backend/FTP/FTPBackend.cs @@ -19,20 +19,12 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -#nullable enable - -using System; -using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; -using System.IO; -using System.Linq; using System.Net; using System.Net.Security; using System.Runtime.CompilerServices; using System.Security.Authentication; using System.Text; -using System.Threading; -using System.Threading.Tasks; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using Duplicati.Library.Utility; diff --git a/Duplicati/Library/Backend/FTP/Strings.cs b/Duplicati/Library/Backend/FTP/Strings.cs index 73eb2af72..f9b4ddc67 100644 --- a/Duplicati/Library/Backend/FTP/Strings.cs +++ b/Duplicati/Library/Backend/FTP/Strings.cs @@ -69,6 +69,6 @@ namespace Duplicati.Library.Backend public static string DescriptionRelativePathLong { get { return LC.L(@"Use this option to interpret the url path as a path that is relative to the initial login folder. This option only has an effect if the initial starting folder in the FTP server is not the (virtual) root folder. If not set, the path in the url is treated as absolute, ignoring the initial login folder."); } } public static string DescriptionUseCwdNamesShort { get { return LC.L(@"Use CWD instead of absolute paths"); } } public static string DescriptionUseCwdNamesLong { get { return LC.L(@"Use this option to start the connection with a CWD command instead of an absolute path. This can be useful if the FTP server does not support absolute paths."); } } - public static string ErrorCreateFolder(string targetFolderName, string resultingFolder) { return LC.L(@"Error creating folder {0}, gave folder: {1}", targetFolderName, resultingFolder); } + public static string ErrorCreateFolder(string targetFolderName, string? resultingFolder) { return LC.L(@"Error creating folder {0}, gave folder: {1}", targetFolderName, resultingFolder); } } } From 73bb3d811bc85acc1125d4e71d982160d963b334 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 21 Mar 2025 12:34:06 +0100 Subject: [PATCH 18/48] Updated Google services --- ...cati.Library.Backend.GoogleServices.csproj | 3 + .../Backend/GoogleServices/GCSConfig.cs | 72 +++--- .../GoogleServices/GoogleCloudStorage.cs | 134 +++++------ .../Backend/GoogleServices/GoogleCommon.cs | 107 +++++---- .../Backend/GoogleServices/GoogleDrive.cs | 216 +++++++++--------- .../Library/Backend/GoogleServices/Strings.cs | 58 ++--- .../Library/Backend/GoogleServices/WebApi.cs | 196 +++++++--------- Duplicati/Library/Interface/IWebModule.cs | 39 ++-- 8 files changed, 403 insertions(+), 422 deletions(-) diff --git a/Duplicati/Library/Backend/GoogleServices/Duplicati.Library.Backend.GoogleServices.csproj b/Duplicati/Library/Backend/GoogleServices/Duplicati.Library.Backend.GoogleServices.csproj index 198df6dd6..be6989d62 100644 --- a/Duplicati/Library/Backend/GoogleServices/Duplicati.Library.Backend.GoogleServices.csproj +++ b/Duplicati/Library/Backend/GoogleServices/Duplicati.Library.Backend.GoogleServices.csproj @@ -4,7 +4,10 @@ net8.0 Library Copyright © 2025 Team Duplicati, MIT license + enable + enable + diff --git a/Duplicati/Library/Backend/GoogleServices/GCSConfig.cs b/Duplicati/Library/Backend/GoogleServices/GCSConfig.cs index 2166afad0..4effd5c35 100644 --- a/Duplicati/Library/Backend/GoogleServices/GCSConfig.cs +++ b/Duplicati/Library/Backend/GoogleServices/GCSConfig.cs @@ -1,29 +1,25 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System; using Duplicati.Library.Interface; -using System.Collections.Generic; -using System.Linq; -using Duplicati.Library.Localization.Short; namespace Duplicati.Library.Backend.GoogleServices { @@ -32,7 +28,7 @@ namespace Duplicati.Library.Backend.GoogleServices public class GCSConfig : IWebModule { private const ConfigType DEFAULT_CONFIG_TYPE = ConfigType.Locations; - private static readonly string DEFAULT_CONFIG_TYPE_STR = Enum.GetName(typeof(ConfigType), DEFAULT_CONFIG_TYPE); + private static readonly string DEFAULT_CONFIG_TYPE_STR = DEFAULT_CONFIG_TYPE.ToString(); private const string KEY_CONFIGTYPE = "gcs-config"; public enum ConfigType @@ -46,15 +42,13 @@ namespace Duplicati.Library.Backend.GoogleServices #region IWebModule implementation - public System.Collections.Generic.IDictionary Execute(System.Collections.Generic.IDictionary options) + public IDictionary Execute(IDictionary options) { - string k; - options.TryGetValue(KEY_CONFIGTYPE, out k); + options.TryGetValue(KEY_CONFIGTYPE, out var k); if (string.IsNullOrWhiteSpace(k)) k = DEFAULT_CONFIG_TYPE_STR; - ConfigType ct; - if (!Enum.TryParse(k, true, out ct)) + if (!Enum.TryParse(k, true, out var ct)) ct = DEFAULT_CONFIG_TYPE; switch (ct) @@ -66,23 +60,17 @@ namespace Duplicati.Library.Backend.GoogleServices } } - public string Key { get { return "gcs-getconfig"; } } + public string Key => "gcs-getconfig"; - public string DisplayName { get { return LC.L("Google Cloud Storage configuration module"); } } + public string DisplayName => Strings.GCSConfig.DisplayName; - public string Description { get { return LC.L("Expose Google Cloud Storage configuration as a web module"); } } + public string Description => Strings.GCSConfig.Description; - public System.Collections.Generic.IList SupportedCommands - { - get - { - return new List([ - new CommandLineArgument(KEY_CONFIGTYPE, CommandLineArgument.ArgumentType.Enumeration, LC.L("The config to get"), LC.L("Provide different config values"), DEFAULT_CONFIG_TYPE_STR, Enum.GetNames(typeof(ConfigType))) - - ]); - } - } + public IList SupportedCommands => + [ + new CommandLineArgument(KEY_CONFIGTYPE, CommandLineArgument.ArgumentType.Enumeration, Strings.GCSConfig.ConfigTypeShort, Strings.GCSConfig.ConfigTypeLong, DEFAULT_CONFIG_TYPE_STR, Enum.GetNames(typeof(ConfigType))) + ]; #endregion } diff --git a/Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs b/Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs index 30e9db6de..c566115d2 100644 --- a/Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs +++ b/Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs @@ -23,14 +23,11 @@ using Duplicati.Library.Backend.GoogleServices; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using Duplicati.Library.Utility; +using Duplicati.Library.Utility.Options; using Newtonsoft.Json; -using System; -using System.Collections.Generic; using System.Net; using System.Runtime.CompilerServices; using System.Text; -using System.Threading; -using System.Threading.Tasks; namespace Duplicati.Library.Backend.GoogleCloudStorage { @@ -38,7 +35,7 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage // This class is instantiated dynamically in the BackendLoader. public class GoogleCloudStorage : IBackend, IStreamingBackend, IRenameEnabledBackend, ITimeoutExemptBackend { - private const string AUTHID_OPTION = "authid"; + private static readonly string TOKEN_URL = OAuthHelper.OAUTH_LOGIN_URL("gcs"); private const string PROJECT_OPTION = "gcs-project"; private const string LOCATION_OPTION = "gcs-location"; @@ -46,17 +43,22 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage private readonly string m_bucket; private readonly string m_prefix; - private readonly string m_project; + private readonly string? m_project; private readonly OAuthHelper m_oauth; - private readonly string m_location; - private readonly string m_storage_class; + private readonly string? m_location; + private readonly string? m_storage_class; + private readonly TimeoutOptionsHelper.Timeouts m_timeouts; public GoogleCloudStorage() { + m_bucket = null!; + m_prefix = null!; + m_oauth = null!; + m_timeouts = null!; } - public GoogleCloudStorage(string url, Dictionary options) + public GoogleCloudStorage(string url, Dictionary options) { var uri = new Utility.Uri(url); @@ -67,38 +69,37 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage if (m_prefix.StartsWith("/", StringComparison.Ordinal)) m_prefix = m_prefix.Substring(1); - string authid; - options.TryGetValue(AUTHID_OPTION, out authid); - options.TryGetValue(PROJECT_OPTION, out m_project); - options.TryGetValue(LOCATION_OPTION, out m_location); - options.TryGetValue(STORAGECLASS_OPTION, out m_storage_class); + var authId = AuthIdOptionsHelper.Parse(options) + .RequireCredentials(TOKEN_URL); - if (string.IsNullOrEmpty(authid)) - throw new UserInformationException(Strings.GoogleCloudStorage.MissingAuthID(AUTHID_OPTION), "GoogleCloudStorageMissingAuthID"); + m_timeouts = TimeoutOptionsHelper.Parse(options); + m_project = options.GetValueOrDefault(PROJECT_OPTION); + m_location = options.GetValueOrDefault(LOCATION_OPTION); + m_storage_class = options.GetValueOrDefault(STORAGECLASS_OPTION); - m_oauth = new OAuthHelper(authid, this.ProtocolKey); + m_oauth = new OAuthHelper(authId.AuthId!, this.ProtocolKey); m_oauth.AutoAuthHeader = true; } private class ListBucketResponse { - public string nextPageToken { get; set; } - public BucketResourceItem[] items { get; set; } + public string? nextPageToken { get; set; } + public BucketResourceItem[]? items { get; set; } } private class BucketResourceItem { - public string name { get; set; } + public string? name { get; set; } public DateTime? updated { get; set; } public long? size { get; set; } } private class CreateBucketRequest { - public string name { get; set; } - public string location { get; set; } - public string storageClass { get; set; } + public string? name { get; set; } + public string? location { get; set; } + public string? storageClass { get; set; } } private async Task HandleListExceptions(Func> func) @@ -123,12 +124,15 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage var url = WebApi.GoogleCloudStorage.ListUrl(m_bucket, Utility.Uri.UrlEncode(m_prefix)); while (true) { - var resp = await HandleListExceptions(() => m_oauth.ReadJSONResponseAsync(url, cancelToken)).ConfigureAwait(false); + var resp = await HandleListExceptions(() => + Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, ct => + m_oauth.ReadJSONResponseAsync(url, ct)) + ).ConfigureAwait(false); if (resp.items != null) foreach (var f in resp.items) { - var name = f.name; + var name = f.name ?? ""; if (name.StartsWith(m_prefix, StringComparison.OrdinalIgnoreCase)) name = name.Substring(m_prefix.Length); if (f.size == null) @@ -148,33 +152,33 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) { - using (System.IO.FileStream fs = System.IO.File.OpenRead(filename)) + using (var fs = File.OpenRead(filename)) await PutAsync(remotename, fs, cancelToken).ConfigureAwait(false); } public async Task GetAsync(string remotename, string filename, CancellationToken cancelToken) { - using (System.IO.FileStream fs = System.IO.File.Create(filename)) + using (var fs = File.Create(filename)) await GetAsync(remotename, fs, cancelToken).ConfigureAwait(false); } public Task DeleteAsync(string remotename, CancellationToken cancelToken) { - var req = m_oauth.CreateRequest(WebApi.GoogleCloudStorage.DeleteUrl(m_bucket, Library.Utility.Uri.UrlPathEncode(m_prefix + remotename))); req.Method = "DELETE"; - return m_oauth.ReadJSONResponseAsync(req, cancelToken); + return Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct + => m_oauth.ReadJSONResponseAsync(req, ct)); } public Task TestAsync(CancellationToken cancelToken) => this.TestListAsync(cancelToken); - public async Task CreateFolderAsync(CancellationToken cancelToken) + public Task CreateFolderAsync(CancellationToken cancelToken) { if (string.IsNullOrEmpty(m_project)) throw new UserInformationException(Strings.GoogleCloudStorage.ProjectIDMissingError(PROJECT_OPTION), "GoogleCloudStorageMissingProjectID"); - var data = System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new CreateBucketRequest + var data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new CreateBucketRequest { name = m_bucket, location = m_location, @@ -186,23 +190,19 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage req.ContentLength = data.Length; req.ContentType = "application/json; charset=UTF-8"; - var areq = new AsyncHttpRequest(req); + return Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, async ct => + { + var areq = new AsyncHttpRequest(req); + using (var rs = areq.GetRequestStream()) + await rs.WriteAsync(data, 0, data.Length, ct); - using (var rs = areq.GetRequestStream()) - await rs.WriteAsync(data, 0, data.Length, cancelToken); - - await m_oauth.ReadJSONResponseAsync(areq, cancelToken).ConfigureAwait(false); + await m_oauth.ReadJSONResponseAsync(areq, ct).ConfigureAwait(false); + }); } - public string DisplayName - { - get { return Strings.GoogleCloudStorage.DisplayName; } - } + public string DisplayName => Strings.GoogleCloudStorage.DisplayName; - public string ProtocolKey - { - get { return "gcs"; } - } + public string ProtocolKey => "gcs"; public IList SupportedCommands { @@ -211,50 +211,49 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage StringBuilder locations = new StringBuilder(); StringBuilder storageClasses = new StringBuilder(); - foreach (KeyValuePair s in WebApi.GoogleCloudStorage.KNOWN_GCS_LOCATIONS) + foreach (var s in WebApi.GoogleCloudStorage.KNOWN_GCS_LOCATIONS) locations.AppendLine(string.Format("{0}: {1}", s.Key, s.Value)); - foreach (KeyValuePair s in WebApi.GoogleCloudStorage.KNOWN_GCS_STORAGE_CLASSES) + foreach (var s in WebApi.GoogleCloudStorage.KNOWN_GCS_STORAGE_CLASSES) storageClasses.AppendLine(string.Format("{0}: {1}", s.Key, s.Value)); - return new List(new ICommandLineArgument[] { + return [ new CommandLineArgument(LOCATION_OPTION, CommandLineArgument.ArgumentType.String, Strings.GoogleCloudStorage.LocationDescriptionShort, Strings.GoogleCloudStorage.LocationDescriptionLong(locations.ToString())), new CommandLineArgument(STORAGECLASS_OPTION, CommandLineArgument.ArgumentType.String, Strings.GoogleCloudStorage.StorageclassDescriptionShort, Strings.GoogleCloudStorage.StorageclassDescriptionLong(storageClasses.ToString())), - new CommandLineArgument(AUTHID_OPTION, CommandLineArgument.ArgumentType.Password, Strings.GoogleCloudStorage.AuthidShort, Strings.GoogleCloudStorage.AuthidLong(OAuthHelper.OAUTH_LOGIN_URL("gcs"))), + .. AuthIdOptionsHelper.GetOptions(TOKEN_URL), new CommandLineArgument(PROJECT_OPTION, CommandLineArgument.ArgumentType.String, Strings.GoogleCloudStorage.ProjectDescriptionShort, Strings.GoogleCloudStorage.ProjectDescriptionLong), - }); + .. TimeoutOptionsHelper.GetOptions(), + ]; } } - public string Description - { - get { return Strings.GoogleCloudStorage.Description; } - } + public string Description => Strings.GoogleCloudStorage.Description; public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(WebApi.GoogleCloudStorage.Hosts()); #endregion - public async Task PutAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) + public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken) { var item = new BucketResourceItem { name = m_prefix + remotename }; var url = WebApi.GoogleCloudStorage.PutUrl(m_bucket); - var res = await GoogleCommon.ChunkedUploadWithResumeAsync(m_oauth, item, url, stream, cancelToken).ConfigureAwait(false); + var res = await GoogleCommon.ChunkedUploadWithResumeAsync(m_oauth, item, url, stream, m_timeouts.ShortTimeout, m_timeouts.ReadWriteTimeout, cancelToken).ConfigureAwait(false); if (res == null) throw new Exception("Upload succeeded, but no data was returned"); } - public async Task GetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) + public async Task GetAsync(string remotename, Stream stream, CancellationToken cancelToken) { try { - var url = WebApi.GoogleCloudStorage.GetUrl(m_bucket, Library.Utility.Uri.UrlPathEncode(m_prefix + remotename)); + var url = WebApi.GoogleCloudStorage.GetUrl(m_bucket, Utility.Uri.UrlPathEncode(m_prefix + remotename)); var req = m_oauth.CreateRequest(url); var areq = new AsyncHttpRequest(req); - using (var resp = areq.GetResponse()) - using (var rs = areq.GetResponseStream()) - await Library.Utility.Utility.CopyStreamAsync(rs, stream, cancelToken).ConfigureAwait(false); + using (var resp = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => areq.GetResponse()).ConfigureAwait(false)) + using (var rs = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => areq.GetResponseStream()).ConfigureAwait(false)) + using (var ts = rs.ObserveReadTimeout(m_timeouts.ReadWriteTimeout)) + await Utility.Utility.CopyStreamAsync(ts, stream, cancelToken).ConfigureAwait(false); } catch (WebException wex) { @@ -265,9 +264,9 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage } } - public async Task RenameAsync(string oldname, string newname, CancellationToken cancelToken) + public Task RenameAsync(string oldname, string newname, CancellationToken cancelToken) { - var data = System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new BucketResourceItem + var data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new BucketResourceItem { name = m_prefix + newname, })); @@ -277,11 +276,14 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage req.ContentLength = data.Length; req.ContentType = "application/json; charset=UTF-8"; - var areq = new AsyncHttpRequest(req); - using (var rs = areq.GetRequestStream()) - await rs.WriteAsync(data, 0, data.Length, cancelToken).ConfigureAwait(false); + return Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, async ct => + { + var areq = new AsyncHttpRequest(req); + using (var rs = areq.GetRequestStream()) + await rs.WriteAsync(data, 0, data.Length, ct).ConfigureAwait(false); - await m_oauth.ReadJSONResponseAsync(req, cancelToken).ConfigureAwait(false); + await m_oauth.ReadJSONResponseAsync(req, ct).ConfigureAwait(false); + }); } #region IDisposable implementation diff --git a/Duplicati/Library/Backend/GoogleServices/GoogleCommon.cs b/Duplicati/Library/Backend/GoogleServices/GoogleCommon.cs index bce49da94..05dd67b5f 100644 --- a/Duplicati/Library/Backend/GoogleServices/GoogleCommon.cs +++ b/Duplicati/Library/Backend/GoogleServices/GoogleCommon.cs @@ -1,30 +1,27 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Utility; using Newtonsoft.Json; -using System; using System.Net; -using System.Threading; -using System.Threading.Tasks; namespace Duplicati.Library.Backend.GoogleServices { @@ -42,7 +39,7 @@ namespace Duplicati.Library.Backend.GoogleServices /// The Oauth instance /// The resumeable uploaduri /// The length of the entire stream - private static long QueryUploadRange(OAuthHelper oauth, string uploaduri, long streamlength, out T response) + private static long QueryUploadRange(OAuthHelper oauth, string uploaduri, long streamlength, out T? response) where T : class { response = null; @@ -51,8 +48,9 @@ namespace Duplicati.Library.Backend.GoogleServices req.ContentLength = 0; req.Headers["Content-Range"] = string.Format("bytes */{0}", streamlength); + // TODO: Apply timeout when this is upgraded to HttpClient var areq = new AsyncHttpRequest(req); - using(var resp = oauth.GetResponseWithoutException(areq)) + using (var resp = oauth.GetResponseWithoutException(areq)) { var code = (int)resp.StatusCode; @@ -62,7 +60,7 @@ namespace Duplicati.Library.Backend.GoogleServices response = oauth.ReadJSONResponse(resp); if (response == null) throw new Exception(string.Format("Upload succeeded, but no data was returned, status code: {0}", code)); - + return streamlength; } @@ -73,9 +71,9 @@ namespace Duplicati.Library.Backend.GoogleServices // https://code.google.com/a/google.com/p/apps-api-issues/issues/detail?id=3884 if (resp.Headers["Range"] == null) - return 0; + return 0; else - return long.Parse(resp.Headers["Range"].Split(new char[] { '-' })[1]) + 1; + return long.Parse(resp.Headers["Range"]!.Split(new char[] { '-' })[1]) + 1; } else throw new WebException(string.Format("Unexpected status code: {0}", code), null, WebExceptionStatus.ServerProtocolViolation, resp); @@ -90,10 +88,12 @@ namespace Duplicati.Library.Backend.GoogleServices /// The data to submit as JSON metadata. /// The URL to register the upload session against. /// The stream with content data to upload. + /// The short request timeout. + /// The read write timeout. /// The HTTP Method. /// The type of data to upload as metadata. /// The type of data returned from the upload. - public static async Task ChunkedUploadWithResumeAsync(OAuthHelper oauth, TRequest requestdata, string url, System.IO.Stream stream, CancellationToken cancelToken, string method = "POST") + public static async Task ChunkedUploadWithResumeAsync(OAuthHelper oauth, TRequest requestdata, string url, Stream stream, TimeSpan shortTimeout, TimeSpan readWriteTimeout, CancellationToken cancelToken, string method = "POST") where TRequest : class where TResponse : class { @@ -105,25 +105,30 @@ namespace Duplicati.Library.Backend.GoogleServices if (data != null) req.ContentType = "application/json; charset=UTF-8"; - + req.Headers["X-Upload-Content-Type"] = "application/octet-stream"; req.Headers["X-Upload-Content-Length"] = stream.Length.ToString(); - var areq = new AsyncHttpRequest(req); - if (data != null) - using(var rs = areq.GetRequestStream()) - await rs.WriteAsync(data, 0, data.Length, cancelToken).ConfigureAwait(false); - - string uploaduri; - using(var resp = (HttpWebResponse)areq.GetResponse()) + var uploaduri = await Utility.Utility.WithTimeout(shortTimeout, cancelToken, async ct => { - if (resp.StatusCode != HttpStatusCode.OK || string.IsNullOrWhiteSpace(resp.Headers["Location"])) - throw new WebException("Failed to start upload session", null, WebExceptionStatus.UnknownError, resp); + var areq = new AsyncHttpRequest(req); + if (data != null) + using (var rs = areq.GetRequestStream()) + await rs.WriteAsync(data, 0, data.Length, cancelToken).ConfigureAwait(false); - uploaduri = resp.Headers["Location"]; - } + using (var resp = (HttpWebResponse)areq.GetResponse()) + { + if (resp.StatusCode != HttpStatusCode.OK || string.IsNullOrWhiteSpace(resp.Headers["Location"])) + throw new WebException("Failed to start upload session", null, WebExceptionStatus.UnknownError, resp); - return await ChunkedUploadAsync(oauth, uploaduri, stream, cancelToken).ConfigureAwait(false); + return resp.Headers["Location"]; + } + }).ConfigureAwait(false); + + if (string.IsNullOrWhiteSpace(uploaduri)) + throw new Exception("Failed to start upload session"); + + return await ChunkedUploadAsync(oauth, uploaduri, stream, shortTimeout, readWriteTimeout, cancelToken).ConfigureAwait(false); } /// @@ -133,8 +138,10 @@ namespace Duplicati.Library.Backend.GoogleServices /// The Oauth instance /// The resumeable uploaduri /// The stream with data to upload. + /// The short request timeout. + /// The read write timeout. /// The type of data in the response. - private static async Task ChunkedUploadAsync(OAuthHelper oauth, string uploaduri, System.IO.Stream stream, CancellationToken cancelToken) + private static async Task ChunkedUploadAsync(OAuthHelper oauth, string uploaduri, Stream stream, TimeSpan shortTimeout, TimeSpan readWriteTimeout, CancellationToken cancelToken) where T : class { var queryRange = false; @@ -143,14 +150,13 @@ namespace Duplicati.Library.Backend.GoogleServices var buffer = new byte[Library.Utility.Utility.DEFAULT_BUFFER_SIZE]; // Repeatedly try uploading until all retries are done - while(true) + while (true) { try { if (queryRange) { - T re; - offset = GoogleCommon.QueryUploadRange(oauth, uploaduri, stream.Length, out re); + offset = QueryUploadRange(oauth, uploaduri, stream.Length, out var re); queryRange = false; if (re != null) @@ -172,25 +178,26 @@ namespace Duplicati.Library.Backend.GoogleServices // Upload the remaining data var areq = new AsyncHttpRequest(req); - using(var rs = areq.GetRequestStream()) + using (var rs = areq.GetRequestStream()) + using (var ts = rs.ObserveWriteTimeout(readWriteTimeout)) { var remaining = chunkSize; - while(remaining > 0) + while (remaining > 0) { var n = stream.Read(buffer, 0, (int)Math.Min(remaining, Library.Utility.Utility.DEFAULT_BUFFER_SIZE)); - await rs.WriteAsync(buffer, 0, n, cancelToken); + await ts.WriteAsync(buffer, 0, n, cancelToken); remaining -= n; } } // Check the response - using(var resp = oauth.GetResponseWithoutException(areq)) + using (var resp = await Utility.Utility.WithTimeout(shortTimeout, cancelToken, _ => oauth.GetResponseWithoutException(areq)).ConfigureAwait(false)) { var code = (int)resp.StatusCode; if (code == 308 && resp.Headers["Range"] != null) { - offset = long.Parse(resp.Headers["Range"].Split(new char[] {'-'})[1]) + 1; + offset = long.Parse(resp.Headers["Range"]!.Split(new char[] { '-' })[1]) + 1; retries = 0; } else if (code >= 200 && code <= 299) diff --git a/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs b/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs index 96088c5b7..6c210aead 100644 --- a/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs +++ b/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs @@ -23,16 +23,11 @@ using Duplicati.Library.Backend.GoogleServices; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using Duplicati.Library.Utility; +using Duplicati.Library.Utility.Options; using Newtonsoft.Json; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; using System.Net; using System.Runtime.CompilerServices; using System.Text; -using System.Threading; -using System.Threading.Tasks; namespace Duplicati.Library.Backend.GoogleDrive { @@ -40,41 +35,51 @@ namespace Duplicati.Library.Backend.GoogleDrive // This class is instantiated dynamically in the BackendLoader. public class GoogleDrive : IBackend, IStreamingBackend, IQuotaEnabledBackend, IRenameEnabledBackend, ITimeoutExemptBackend { - private const string AUTHID_OPTION = "authid"; private const string TEAMDRIVE_ID = "googledrive-teamdrive-id"; private const string FOLDER_MIMETYPE = "application/vnd.google-apps.folder"; + private static readonly string TOKEN_URL = OAuthHelper.OAUTH_LOGIN_URL("googledrive"); + private readonly string m_path; - private readonly string m_teamDriveID; + private readonly string? m_teamDriveID; private readonly OAuthHelper m_oauth; private readonly Dictionary m_filecache; + private readonly TimeoutOptionsHelper.Timeouts m_timeouts; - private string m_currentFolderId; + private string? m_currentFolderId; public GoogleDrive() { + m_path = null!; + m_oauth = null!; + m_filecache = null!; + m_timeouts = null!; } - public GoogleDrive(string url, Dictionary options) + public GoogleDrive(string url, Dictionary options) { var uri = new Utility.Uri(url); m_path = Util.AppendDirSeparator(uri.HostAndPath, "/"); - string authid = null; - if (options.ContainsKey(AUTHID_OPTION)) - authid = options[AUTHID_OPTION]; + var auth = AuthIdOptionsHelper.Parse(options) + .RequireCredentials(TOKEN_URL); - if (options.ContainsKey(TEAMDRIVE_ID)) - m_teamDriveID = options[TEAMDRIVE_ID]; - - m_oauth = new OAuthHelper(authid, this.ProtocolKey) { AutoAuthHeader = true }; + m_timeouts = TimeoutOptionsHelper.Parse(options); + m_teamDriveID = options.GetValueOrDefault(TEAMDRIVE_ID); + m_oauth = new OAuthHelper(auth.AuthId!, this.ProtocolKey) { AutoAuthHeader = true }; m_filecache = new Dictionary(); } private async Task GetFolderIdAsync(string path, bool autocreate, CancellationToken cancelToken) { - var curparent = m_teamDriveID ?? (await GetAboutInfoAsync(cancelToken).ConfigureAwait(false)).rootFolderId; + var curparent = string.IsNullOrWhiteSpace(m_teamDriveID) + ? (await GetAboutInfoAsync(cancelToken).ConfigureAwait(false)).rootFolderId + : m_teamDriveID; + + if (string.IsNullOrEmpty(curparent)) + throw new UserInformationException("Unable to get root folder", "GoogleDriveNoRootFolder"); + var curdisplay = new StringBuilder("/"); foreach (var p in path.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries)) @@ -97,6 +102,10 @@ namespace Duplicati.Library.Backend.GoogleDrive } curdisplay.Append(p).Append("/"); + + if (string.IsNullOrWhiteSpace(curparent)) + throw new UserInformationException($"Unable to get folder id for {curdisplay}", "GoogleDriveNoFolderId"); + } return curparent; @@ -110,11 +119,9 @@ namespace Duplicati.Library.Backend.GoogleDrive return m_currentFolderId; } - private async Task GetFileEntriesAsync(string remotename, bool throwMissingException, CancellationToken cancelToken) + private async Task GetFileEntriesAsync(string remotename, bool throwMissingException, CancellationToken cancelToken) { - GoogleDriveFolderItem[] entries; - m_filecache.TryGetValue(remotename, out entries); - + m_filecache.TryGetValue(remotename, out var entries); if (entries != null) return entries; @@ -124,7 +131,7 @@ namespace Duplicati.Library.Backend.GoogleDrive if (entries == null || entries.Length == 0) { if (throwMissingException) - throw new FileMissingException(); + throw new FileMissingException(Strings.GoogleDrive.FilenotFound(remotename)); else return null; } @@ -139,7 +146,7 @@ namespace Duplicati.Library.Backend.GoogleDrive #region IStreamingBackend implementation - public async Task PutAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) + public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken) { try { @@ -147,10 +154,9 @@ namespace Duplicati.Library.Backend.GoogleDrive if (m_filecache.Count == 0) await foreach (var file in ListAsync(cancelToken).ConfigureAwait(false)) { /* Enumerate the full listing */ } - GoogleDriveFolderItem[] files; - m_filecache.TryGetValue(remotename, out files); + m_filecache.TryGetValue(remotename, out var files); - string fileId = null; + string? fileId = null; if (files != null) { if (files.Length == 1) @@ -174,7 +180,7 @@ namespace Duplicati.Library.Backend.GoogleDrive teamDriveId = m_teamDriveID }; - var res = await GoogleCommon.ChunkedUploadWithResumeAsync(m_oauth, item, url, stream, cancelToken, isUpdate ? "PUT" : "POST").ConfigureAwait(false); + var res = await GoogleCommon.ChunkedUploadWithResumeAsync(m_oauth, item, url, stream, m_timeouts.ShortTimeout, m_timeouts.ReadWriteTimeout, cancelToken, isUpdate ? "PUT" : "POST").ConfigureAwait(false); m_filecache[remotename] = [res]; } catch @@ -184,19 +190,22 @@ namespace Duplicati.Library.Backend.GoogleDrive } } - public async Task GetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) + public async Task GetAsync(string remotename, Stream stream, CancellationToken cancelToken) { // Prevent repeated download url lookups if (m_filecache.Count == 0) await foreach (var file in ListAsync(cancelToken).ConfigureAwait(false)) { /* Enumerate the full listing */ } - var fileId = (await GetFileEntriesAsync(remotename, true, cancelToken).ConfigureAwait(false)).OrderByDescending(x => x.createdDate).First().id; + var fileId = (await GetFileEntriesAsync(remotename, true, cancelToken).ConfigureAwait(false))?.OrderByDescending(x => x.createdDate).FirstOrDefault()?.id; + if (string.IsNullOrEmpty(fileId)) + throw new FileMissingException(Strings.GoogleDrive.FilenotFound(remotename)); var req = m_oauth.CreateRequest(WebApi.GoogleDrive.GetUrl(fileId)); var areq = new AsyncHttpRequest(req); - using (var resp = (HttpWebResponse)areq.GetResponse()) - using (var rs = areq.GetResponseStream()) - await Duplicati.Library.Utility.Utility.CopyStreamAsync(rs, stream, cancelToken); + using (var resp = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => (HttpWebResponse)areq.GetResponse()).ConfigureAwait(false)) + using (var rs = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => areq.GetResponseStream()).ConfigureAwait(false)) + using (var ts = rs.ObserveReadTimeout(m_timeouts.ReadWriteTimeout)) + await Utility.Utility.CopyStreamAsync(ts, stream, cancelToken); } #endregion @@ -216,7 +225,7 @@ namespace Duplicati.Library.Backend.GoogleDrive var currentFolderId = await GetCurrentFolderIdAsync(cancelToken).ConfigureAwait(false); await foreach (var n in ListFolder(currentFolderId, null, null, cancelToken).ConfigureAwait(false)) { - FileEntry fe = null; + FileEntry? fe = null; if (n.fileSize == null) fe = new FileEntry(n.title); @@ -231,10 +240,9 @@ namespace Duplicati.Library.Backend.GoogleDrive if (!fe.IsFolder) { - GoogleDriveFolderItem[] lst; - if (!m_filecache.TryGetValue(fe.Name, out lst)) + if (!m_filecache.TryGetValue(fe.Name, out var lst)) { - m_filecache[fe.Name] = new GoogleDriveFolderItem[] { n }; + m_filecache[fe.Name] = [n]; } else { @@ -262,13 +270,13 @@ namespace Duplicati.Library.Backend.GoogleDrive public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) { - using (var fs = System.IO.File.OpenRead(filename)) + using (var fs = File.OpenRead(filename)) await PutAsync(remotename, fs, cancelToken).ConfigureAwait(false); } public async Task GetAsync(string remotename, string filename, CancellationToken cancelToken) { - using (var fs = System.IO.File.Create(filename)) + using (var fs = File.Create(filename)) await GetAsync(remotename, fs, cancelToken); } @@ -276,13 +284,19 @@ namespace Duplicati.Library.Backend.GoogleDrive { try { - foreach (var fileid in from n in await GetFileEntriesAsync(remotename, true, cancelToken).ConfigureAwait(false) select n.id) + var entries = await GetFileEntriesAsync(remotename, true, cancelToken).ConfigureAwait(false); + if (entries == null) + throw new FileMissingException(Strings.GoogleDrive.FilenotFound(remotename)); + + foreach (var fileid in entries.Select(x => x.id).WhereNotNullOrWhiteSpace()) { var url = WebApi.GoogleDrive.DeleteUrl(Library.Utility.Uri.UrlPathEncode(fileid), m_teamDriveID); - await m_oauth.GetJSONDataAsync(url, cancelToken, x => - { - x.Method = "DELETE"; - }); + await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct + => m_oauth.GetJSONDataAsync(url, cancelToken, x => + { + x.Method = "DELETE"; + }) + ).ConfigureAwait(false); } m_filecache.Remove(remotename); @@ -304,32 +318,19 @@ namespace Duplicati.Library.Backend.GoogleDrive m_currentFolderId = await GetFolderIdAsync(m_path, true, cancelToken).ConfigureAwait(false); } - public string DisplayName - { - get - { - return Strings.GoogleDrive.DisplayName; - } - } + public string DisplayName => Strings.GoogleDrive.DisplayName; - public string ProtocolKey - { - get - { - return "googledrive"; - } - } + public string ProtocolKey => "googledrive"; - public System.Collections.Generic.IList SupportedCommands => new List(new ICommandLineArgument[] { - new CommandLineArgument(AUTHID_OPTION, - CommandLineArgument.ArgumentType.Password, - Strings.GoogleDrive.AuthidShort, - Strings.GoogleDrive.AuthidLong(OAuthHelper.OAUTH_LOGIN_URL("googledrive"))), - new CommandLineArgument(TEAMDRIVE_ID, - CommandLineArgument.ArgumentType.String, - Strings.GoogleDrive.TeamDriveIdShort, - Strings.GoogleDrive.TeamDriveIdLong), - }); + public IList SupportedCommands => + [ + .. AuthIdOptionsHelper.GetOptions(TOKEN_URL), + new CommandLineArgument(TEAMDRIVE_ID, + CommandLineArgument.ArgumentType.String, + Strings.GoogleDrive.TeamDriveIdShort, + Strings.GoogleDrive.TeamDriveIdLong), + .. TimeoutOptionsHelper.GetOptions(), + ]; public string Description { @@ -342,7 +343,7 @@ namespace Duplicati.Library.Backend.GoogleDrive #endregion #region IQuotaEnabledBackend implementation - public async Task GetQuotaInfoAsync(CancellationToken cancelToken) + public async Task GetQuotaInfoAsync(CancellationToken cancelToken) { try { @@ -365,9 +366,10 @@ namespace Duplicati.Library.Backend.GoogleDrive try { var files = await GetFileEntriesAsync(oldname, true, cancellationToken).ConfigureAwait(false); + if (files == null) + throw new FileMissingException(Strings.GoogleDrive.FilenotFound(oldname)); if (files.Length > 1) - throw new UserInformationException(string.Format(Strings.GoogleDrive.MultipleEntries(oldname, m_path)), - "GoogleDriveMultipleEntries"); + throw new UserInformationException(Strings.GoogleDrive.MultipleEntries(oldname, m_path), "GoogleDriveMultipleEntries"); using var stream = new MemoryStream(); await GetAsync(oldname, stream, cancellationToken).ConfigureAwait(false); @@ -396,13 +398,13 @@ namespace Duplicati.Library.Backend.GoogleDrive private class GoogleDriveParentReference { - public string id { get; set; } + public string? id { get; set; } } private class GoogleDriveListResponse { - public string nextPageToken { get; set; } - public GoogleDriveFolderItem[] items { get; set; } + public string? nextPageToken { get; set; } + public GoogleDriveFolderItem[]? items { get; set; } } private class GoogleDriveFolderItemLabels @@ -412,41 +414,44 @@ namespace Duplicati.Library.Backend.GoogleDrive private class GoogleDriveFolderItem { - public string id { get; set; } - public string title { get; set; } - public string description { get; set; } - public string mimeType { get; set; } - public GoogleDriveFolderItemLabels labels { get; set; } + public string? id { get; set; } + public string? title { get; set; } + public string? description { get; set; } + public string? mimeType { get; set; } + public GoogleDriveFolderItemLabels? labels { get; set; } public DateTime? createdDate { get; set; } public DateTime? modifiedDate { get; set; } public long? fileSize { get; set; } - public string teamDriveId { get; set; } - public GoogleDriveParentReference[] parents { get; set; } + public string? teamDriveId { get; set; } + public GoogleDriveParentReference[]? parents { get; set; } } private class GoogleDriveAboutResponse { public long? quotaBytesTotal { get; set; } public long? quotaBytesUsed { get; set; } - public string rootFolderId { get; set; } + public string? rootFolderId { get; set; } } - private async IAsyncEnumerable ListFolder(string parentfolder, bool? onlyFolders, string name, [EnumeratorCancellation] CancellationToken cancelToken) + private async IAsyncEnumerable ListFolder(string parentfolder, bool? onlyFolders, string? name, [EnumeratorCancellation] CancellationToken cancelToken) { - var fileQuery = new string[] { + var fileQuery = new string?[] { string.IsNullOrEmpty(name) ? null : string.Format("title = '{0}'", EscapeTitleEntries(name)), onlyFolders == null ? null : string.Format("mimeType {0}= '{1}'", onlyFolders.Value ? "" : "!", FOLDER_MIMETYPE), string.Format("'{0}' in parents", EscapeTitleEntries(parentfolder)), "trashed=false" }; - var encodedFileQuery = Library.Utility.Uri.UrlEncode(string.Join(" and ", fileQuery.Where(x => x != null))); + var encodedFileQuery = Utility.Uri.UrlEncode(string.Join(" and ", fileQuery.Where(x => x != null))); var url = WebApi.GoogleDrive.ListUrl(encodedFileQuery, m_teamDriveID); while (true) { - var res = await m_oauth.GetJSONDataAsync(url, cancelToken).ConfigureAwait(false); - foreach (var n in res.items) + var res = await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, ct + => m_oauth.GetJSONDataAsync(url, ct) + ).ConfigureAwait(false); + + foreach (var n in res.items ?? []) yield return n; var token = res.nextPageToken; @@ -458,9 +463,8 @@ namespace Duplicati.Library.Backend.GoogleDrive } private Task GetAboutInfoAsync(CancellationToken cancelToken) - { - return m_oauth.GetJSONDataAsync(WebApi.GoogleDrive.AboutInfoUrl(), cancelToken); - } + => Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct + => m_oauth.GetJSONDataAsync(WebApi.GoogleDrive.AboutInfoUrl(), ct)); private Task CreateFolderAsync(string name, string parent, CancellationToken cancelToken) { @@ -470,25 +474,27 @@ namespace Duplicati.Library.Backend.GoogleDrive description = name, mimeType = FOLDER_MIMETYPE, labels = new GoogleDriveFolderItemLabels { hidden = true }, - parents = new GoogleDriveParentReference[] { new GoogleDriveParentReference { id = parent } } + parents = [new GoogleDriveParentReference { id = parent }] }; - var data = System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(folder)); + var data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(folder)); - return m_oauth.GetJSONDataAsync( - WebApi.GoogleDrive.CreateFolderUrl(m_teamDriveID), - cancelToken, - x => - { - x.Method = "POST"; - x.ContentType = "application/json; charset=UTF-8"; - x.ContentLength = data.Length; + return Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct + => m_oauth.GetJSONDataAsync( + WebApi.GoogleDrive.CreateFolderUrl(m_teamDriveID), + ct, + x => + { + x.Method = "POST"; + x.ContentType = "application/json; charset=UTF-8"; + x.ContentLength = data.Length; - }, async (req, ct) => - { - using (var rs = req.GetRequestStream()) - await rs.WriteAsync(data, 0, data.Length, ct).ConfigureAwait(false); - }); + }, async (req, ct) => + { + using (var rs = req.GetRequestStream()) + await rs.WriteAsync(data, 0, data.Length, ct).ConfigureAwait(false); + }) + ); } } } diff --git a/Duplicati/Library/Backend/GoogleServices/Strings.cs b/Duplicati/Library/Backend/GoogleServices/Strings.cs index 35e660397..0d4a899f4 100644 --- a/Duplicati/Library/Backend/GoogleServices/Strings.cs +++ b/Duplicati/Library/Backend/GoogleServices/Strings.cs @@ -1,35 +1,33 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Localization.Short; namespace Duplicati.Library.Backend.Strings { - internal static class GoogleCloudStorage { + internal static class GoogleCloudStorage + { public static string Description { get { return LC.L(@"This backend can read and write data to Google Cloud Storage. Allowed format is ""gcs://bucket/folder""."); } } public static string DisplayName { get { return LC.L(@"Google Cloud Storage"); } } - public static string MissingAuthID(string url) { return LC.L(@"You need an AuthID. You can get it from: {0}", url); } public static string ProjectIDMissingError(string projectoption) { return LC.L(@"You must supply a project ID with --{0} for creating a bucket.", projectoption); } - public static string AuthidLong(string url) { return LC.L(@"The authorization token retrieved from {0}", url); } - public static string AuthidShort { get { return LC.L(@"The authorization code"); } } public static string LocationDescriptionLong(string regions) { return LC.L(@"This option is only used when creating new buckets. Use this option to change what region the data is stored in. Charges vary with bucket location. Known bucket locations: {0}", regions); } public static string LocationDescriptionShort { get { return LC.L(@"Specify location option for creating a bucket"); } } @@ -40,13 +38,21 @@ namespace Duplicati.Library.Backend.Strings public static string ProjectDescriptionShort { get { return LC.L(@"Specify project for creating a bucket"); } } } - internal static class GoogleDrive { + internal static class GoogleDrive + { public static string Description { get { return LC.L(@"This backend can read and write data to Google Drive. Allowed format is ""googledrive://folder/subfolder""."); } } public static string DisplayName { get { return LC.L(@"Google Drive"); } } public static string MultipleEntries(string folder, string parent) { return LC.L(@"There is more than one item named ""{0}"" in the folder ""{1}"".", folder, parent); } - public static string AuthidLong(string url) { return LC.L(@"The authorization token retrieved from {0}", url); } - public static string AuthidShort { get { return LC.L(@"The authorization code"); } } + public static string FilenotFound(string name) { return LC.L(@"File not found: {0}", name); } public static string TeamDriveIdLong { get { return LC.L("This option sets the team drive to use. Leaving it empty uses the personal drive."); } } public static string TeamDriveIdShort { get { return LC.L("Team drive ID"); } } } + + internal static class GCSConfig + { + public static string DisplayName => LC.L("Google Cloud Storage configuration module"); + public static string Description => LC.L("Expose Google Cloud Storage configuration as a web module"); + public static string ConfigTypeShort => LC.L("The config to get"); + public static string ConfigTypeLong => LC.L("Provide different config values"); + } } diff --git a/Duplicati/Library/Backend/GoogleServices/WebApi.cs b/Duplicati/Library/Backend/GoogleServices/WebApi.cs index 10d868ac6..5f488a610 100644 --- a/Duplicati/Library/Backend/GoogleServices/WebApi.cs +++ b/Duplicati/Library/Backend/GoogleServices/WebApi.cs @@ -1,25 +1,24 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System.Collections.Generic; using System.Collections.Specialized; using Duplicati.Library.Utility; @@ -28,41 +27,41 @@ namespace Duplicati.Library.Backend.WebApi public static class GoogleCloudStorage { // From: https://cloud.google.com/storage/docs/bucket-locations - public static readonly KeyValuePair[] KNOWN_GCS_LOCATIONS = { - new KeyValuePair("(default)", null), - new KeyValuePair("Europe", "EU"), - new KeyValuePair("United States", "US"), - new KeyValuePair("Asia", "ASIA"), + public static readonly KeyValuePair[] KNOWN_GCS_LOCATIONS = { + new KeyValuePair("(default)", null), + new KeyValuePair("Europe", "EU"), + new KeyValuePair("United States", "US"), + new KeyValuePair("Asia", "ASIA"), //Regional buckets: https://cloud.google.com/storage/docs/regional-buckets - new KeyValuePair("Eastern Asia-Pacific", "ASIA-EAST1"), - new KeyValuePair("Central United States 1", "US-CENTRAL1"), - new KeyValuePair("Central United States 2", "US-CENTRAL2"), - new KeyValuePair("Eastern United States 1", "US-EAST1"), - new KeyValuePair("Eastern United States 2", "US-EAST2"), - new KeyValuePair("Eastern United States 3", "US-EAST3"), - new KeyValuePair("Western United States", "US-WEST1"), + new KeyValuePair("Eastern Asia-Pacific", "ASIA-EAST1"), + new KeyValuePair("Central United States 1", "US-CENTRAL1"), + new KeyValuePair("Central United States 2", "US-CENTRAL2"), + new KeyValuePair("Eastern United States 1", "US-EAST1"), + new KeyValuePair("Eastern United States 2", "US-EAST2"), + new KeyValuePair("Eastern United States 3", "US-EAST3"), + new KeyValuePair("Western United States", "US-WEST1"), }; - public static readonly KeyValuePair[] KNOWN_GCS_STORAGE_CLASSES = { - new KeyValuePair("(default)", null), - new KeyValuePair("Standard", "STANDARD"), - new KeyValuePair("Durable Reduced Availability (DRA)", "DURABLE_REDUCED_AVAILABILITY"), - new KeyValuePair("Nearline", "NEARLINE"), + public static readonly KeyValuePair[] KNOWN_GCS_STORAGE_CLASSES = { + new KeyValuePair("(default)", null), + new KeyValuePair("Standard", "STANDARD"), + new KeyValuePair("Durable Reduced Availability (DRA)", "DURABLE_REDUCED_AVAILABILITY"), + new KeyValuePair("Nearline", "NEARLINE"), }; - public static string[] Hosts() - { - return new [] { new System.Uri(Url.UPLOAD).Host, - new System.Uri(Url.API).Host }; - } + public static string[] Hosts() => + [ + new System.Uri(Url.UPLOAD).Host, + new System.Uri(Url.API).Host + ]; public static string DeleteUrl(string bucketId, string objectId) { var path = BucketObjectPath(bucketId, objectId); - return Uri.UriBuilder(Url.API, path); + return Utility.Uri.UriBuilder(Url.API, path); } public static string CreateFolderUrl(string projectId) @@ -72,20 +71,16 @@ namespace Duplicati.Library.Backend.WebApi { QueryParam.Project, projectId } }; - return Uri.UriBuilder(Url.API, Path.Bucket, queryParams); + return Utility.Uri.UriBuilder(Url.API, Path.Bucket, queryParams); } public static string RenameUrl(string bucketId, string objectId) - { - return Uri.UriBuilder(Url.API, BucketObjectPath(bucketId, objectId)); - } + => Utility.Uri.UriBuilder(Url.API, BucketObjectPath(bucketId, objectId)); public static string ListUrl(string bucketId, string prefix) - { - return ListUrl(bucketId, prefix, null); - } + => ListUrl(bucketId, prefix, null); - public static string ListUrl(string bucketId, string prefix, string token) + public static string ListUrl(string bucketId, string prefix, string? token) { var queryParams = new NameValueCollection { { QueryParam.Prefix, prefix } @@ -96,7 +91,7 @@ namespace Duplicati.Library.Backend.WebApi queryParams.Set(QueryParam.PageToken, token); } - return Uri.UriBuilder(Url.API, BucketObjectPath(bucketId), queryParams); + return Utility.Uri.UriBuilder(Url.API, BucketObjectPath(bucketId), queryParams); } public static string PutUrl(string bucketId) @@ -106,7 +101,7 @@ namespace Duplicati.Library.Backend.WebApi { QueryParam.UploadType, QueryValue.Resumable } }; var path = UrlPath.Create(Path.Bucket).Append(bucketId).Append(Path.Object).ToString(); - return Uri.UriBuilder(Url.UPLOAD, path, queryParams); + return Utility.Uri.UriBuilder(Url.UPLOAD, path, queryParams); } public static string GetUrl(string bucketId, string objectId) @@ -118,7 +113,7 @@ namespace Duplicati.Library.Backend.WebApi }; var path = BucketObjectPath(bucketId, objectId); - return Uri.UriBuilder(Url.API, path, queryParams); + return Utility.Uri.UriBuilder(Url.API, path, queryParams); } private static class Url @@ -148,36 +143,27 @@ namespace Duplicati.Library.Backend.WebApi public const string Object = "o"; } - private static string BucketObjectPath(string bucketId, string objectId = null) - { - return UrlPath.Create(Path.Bucket) - .Append(bucketId) - .Append(Path.Object) - .Append(objectId).ToString(); - } + private static string BucketObjectPath(string bucketId, string? objectId = null) + => UrlPath.Create(Path.Bucket) + .Append(bucketId) + .Append(Path.Object) + .Append(objectId).ToString(); } public static class GoogleDrive { public static string[] Hosts() - { - return new [] { new System.Uri(Url.DRIVE).Host, new System.Uri(Url.UPLOAD).Host }; - - } + => [new System.Uri(Url.DRIVE).Host, new System.Uri(Url.UPLOAD).Host]; public static string GetUrl(string fileId) - { - return FileQueryUrl(fileId, new NameValueCollection{ + => FileQueryUrl(fileId, new NameValueCollection{ { QueryParam.Alt, QueryValue.Media } }); - } - public static string DeleteUrl(string fileId, string teamDriveId) - { - return FileQueryUrl(Uri.UrlPathEncode(fileId), AddTeamDriveParam(teamDriveId)); - } + public static string DeleteUrl(string fileId, string? teamDriveId) + => FileQueryUrl(Utility.Uri.UrlPathEncode(fileId), AddTeamDriveParam(teamDriveId)); - public static string PutUrl(string fileId, bool useTeamDrive) + public static string PutUrl(string? fileId, bool useTeamDrive) { var queryParams = new NameValueCollection { { QueryParam.UploadType, @@ -189,16 +175,14 @@ namespace Duplicati.Library.Backend.WebApi } return !string.IsNullOrWhiteSpace(fileId) ? - FileUploadUrl(Uri.UrlPathEncode(fileId), queryParams) : + FileUploadUrl(Utility.Uri.UrlPathEncode(fileId), queryParams) : FileUploadUrl(queryParams); } - public static string ListUrl(string fileQuery, string teamDriveId) - { - return ListUrl(fileQuery, teamDriveId, null); - } - - public static string ListUrl(string fileQuery, string teamDriveId, string token) + public static string ListUrl(string fileQuery, string? teamDriveId) + => ListUrl(fileQuery, teamDriveId, null); + + public static string ListUrl(string fileQuery, string? teamDriveId, string? token) { var queryParams = new NameValueCollection { @@ -207,7 +191,7 @@ namespace Duplicati.Library.Backend.WebApi }; queryParams.Add(AddTeamDriveParam(teamDriveId)); - + if (token != null) { queryParams.Set(QueryParam.PageToken, token); @@ -216,15 +200,11 @@ namespace Duplicati.Library.Backend.WebApi return FileQueryUrl(queryParams); } - public static string CreateFolderUrl(string teamDriveId) - { - return FileQueryUrl(AddTeamDriveParam(teamDriveId)); - } + public static string CreateFolderUrl(string? teamDriveId) + => FileQueryUrl(AddTeamDriveParam(teamDriveId)); public static string AboutInfoUrl() - { - return Uri.UriBuilder(Url.DRIVE, Path.About); - } + => Utility.Uri.UriBuilder(Url.DRIVE, Path.About); private static class Url { @@ -259,40 +239,30 @@ namespace Duplicati.Library.Backend.WebApi } private static string FileQueryUrl(NameValueCollection values) - { - return Uri.UriBuilder(Url.DRIVE, Path.File, values); - } + => Utility.Uri.UriBuilder(Url.DRIVE, Path.File, values); - private static string FileQueryUrl(string fileId, NameValueCollection values = null) - { - return Uri.UriBuilder(Url.DRIVE, UrlPath.Create(Path.File).Append(fileId).ToString(), - values); - } + private static string FileQueryUrl(string fileId, NameValueCollection? values = null) + => Utility.Uri.UriBuilder(Url.DRIVE, UrlPath.Create(Path.File).Append(fileId).ToString(), values); - private static string FileUploadUrl(string fileId, NameValueCollection values) - { - return Uri.UriBuilder(Url.UPLOAD, UrlPath.Create(Path.File).Append(fileId).ToString(), - values); - } + private static string FileUploadUrl(string fileId, NameValueCollection? values) + => Utility.Uri.UriBuilder(Url.UPLOAD, UrlPath.Create(Path.File).Append(fileId).ToString(), values); private static string FileUploadUrl(NameValueCollection values) - { - return Uri.UriBuilder(Url.UPLOAD, Path.File, values); - } + => Utility.Uri.UriBuilder(Url.UPLOAD, Path.File, values); - private static NameValueCollection AddTeamDriveParam(string teamDriveId) + private static NameValueCollection AddTeamDriveParam(string? teamDriveId) { return teamDriveId != null ? new NameValueCollection { - { WebApi.GoogleDrive.QueryParam.SupportsTeamDrive, - WebApi.GoogleDrive.QueryValue.True }, - { WebApi.GoogleDrive.QueryParam.TeamDriveId, + { QueryParam.SupportsTeamDrive, + QueryValue.True }, + { QueryParam.TeamDriveId, teamDriveId }, - { WebApi.GoogleDrive.QueryParam.IncludeTeamDrive, - WebApi.GoogleDrive.QueryValue.True }, - { WebApi.GoogleDrive.QueryParam.corpora, - WebApi.GoogleDrive.QueryValue.TeamDrive } + { QueryParam.IncludeTeamDrive, + QueryValue.True }, + { QueryParam.corpora, + QueryValue.TeamDrive } - } : new NameValueCollection( ); + } : new NameValueCollection(); } } } \ No newline at end of file diff --git a/Duplicati/Library/Interface/IWebModule.cs b/Duplicati/Library/Interface/IWebModule.cs index c057858aa..a0a316fc2 100644 --- a/Duplicati/Library/Interface/IWebModule.cs +++ b/Duplicati/Library/Interface/IWebModule.cs @@ -1,25 +1,24 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System; using System.Collections.Generic; namespace Duplicati.Library.Interface From 4b2c0be974fe067e4b251d86265ab2f1f5c4fecb Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 21 Mar 2025 13:36:39 +0100 Subject: [PATCH 19/48] Updated S3 and iDrive --- .../Duplicati.Library.Backend.Idrivee2.csproj | 2 + .../Backend/Idrivee2/Idrivee2Backend.cs | 170 +++++++++--------- .../S3/Duplicati.Library.Backend.S3.csproj | 2 + Duplicati/Library/Backend/S3/IS3Client.cs | 10 +- Duplicati/Library/Backend/S3/S3AwsClient.cs | 56 +++--- Duplicati/Library/Backend/S3/S3Backend.cs | 86 +++++---- Duplicati/Library/Backend/S3/S3Config.cs | 69 +++---- Duplicati/Library/Backend/S3/S3IAM.cs | 50 ++---- Duplicati/Library/Backend/S3/S3MinioClient.cs | 68 ++++--- Duplicati/Library/Backend/S3/Strings.cs | 21 +++ 10 files changed, 261 insertions(+), 273 deletions(-) diff --git a/Duplicati/Library/Backend/Idrivee2/Duplicati.Library.Backend.Idrivee2.csproj b/Duplicati/Library/Backend/Idrivee2/Duplicati.Library.Backend.Idrivee2.csproj index 756380cd2..583dd40a3 100644 --- a/Duplicati/Library/Backend/Idrivee2/Duplicati.Library.Backend.Idrivee2.csproj +++ b/Duplicati/Library/Backend/Idrivee2/Duplicati.Library.Backend.Idrivee2.csproj @@ -5,6 +5,8 @@ Library Duplicati.Library.Backend Copyright © 2025 Team Duplicati, MIT license + enable + enable diff --git a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs index 15358a635..5fd7fe442 100644 --- a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs +++ b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs @@ -21,84 +21,71 @@ using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; +using Duplicati.Library.Utility; +using Duplicati.Library.Utility.Options; using System.Runtime.CompilerServices; -using System.Threading; -using System.Threading.Tasks; namespace Duplicati.Library.Backend { public class Idrivee2Backend : IBackend, IStreamingBackend, IFolderEnabledBackend { - private static readonly string LOGTAG = Logging.Log.LogTagFromType(); - - static Idrivee2Backend() - { - - } - + // TODO: Non-standard naming, should be access-key-id and access-key-secret + private const string AUTH_USERNAME_OPTION = "access_key_id"; + private const string AUTH_PASSWORD_OPTION = "access_key_secret"; private readonly string m_prefix; private readonly string m_bucket; - private IS3Client m_s3Client; + private IS3Client? m_s3Client; + private readonly TimeoutOptionsHelper.Timeouts m_timeouts; + private readonly AuthOptionsHelper.AuthOptions m_auth; + private readonly Dictionary m_options; public Idrivee2Backend() { + m_bucket = null!; + m_prefix = null!; + m_timeouts = null!; + m_auth = null!; + m_options = null!; } - public Idrivee2Backend(string url, Dictionary options) + public Idrivee2Backend(string url, Dictionary options) { var uri = new Utility.Uri(url); m_bucket = uri.Host; m_prefix = uri.Path; m_prefix = m_prefix.Trim(); if (m_prefix.Length != 0) - { m_prefix = Util.AppendDirSeparator(m_prefix, "/"); - } - string accessKeyId = null; - string accessKeySecret = null; - if (options.ContainsKey("auth-username")) - accessKeyId = options["auth-username"]; - if (options.ContainsKey("auth-password")) - accessKeySecret = options["auth-password"]; - - if (options.ContainsKey("access_key_id")) - accessKeyId = options["access_key_id"]; - if (options.ContainsKey("secret_access_key")) - accessKeySecret = options["secret_access_key"]; - - if (string.IsNullOrEmpty(accessKeyId)) + m_timeouts = TimeoutOptionsHelper.Parse(options); + m_auth = AuthOptionsHelper.ParseWithAlias(options, uri, AUTH_USERNAME_OPTION, AUTH_PASSWORD_OPTION); + if (!m_auth.HasUsername) throw new UserInformationException(Strings.Idrivee2Backend.NoKeyIdError, "Idrivee2NoKeyId"); - if (string.IsNullOrEmpty(accessKeySecret)) + if (!m_auth.HasPassword) throw new UserInformationException(Strings.Idrivee2Backend.NoKeySecretError, "Idrivee2NoKeySecret"); - string host = GetRegionEndpoint("https://api.idrivee2.com/api/service/get_region_end_point/" + accessKeyId); - - m_s3Client = new S3AwsClient(accessKeyId, accessKeySecret, null, host, null, true, false, options); + m_options = options; } - public string GetRegionEndpoint(string url) + public async Task GetRegionEndpointAsync(string url, CancellationToken cancellationToken) { try { - System.Net.HttpWebRequest req = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(url); + var req = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(url); req.Method = System.Net.WebRequestMethods.Http.Get; - Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req); + var areq = new Utility.AsyncHttpRequest(req); - using (System.Net.HttpWebResponse resp = (System.Net.HttpWebResponse)areq.GetResponse()) + using (var resp = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancellationToken, _ => (System.Net.HttpWebResponse)areq.GetResponse()).ConfigureAwait(false)) { int code = (int)resp.StatusCode; if (code < 200 || code >= 300) //For some reason Mono does not throw this automatically throw new Exception("Failed to fetch region endpoint"); - using (var s = areq.GetResponseStream()) + using (var s = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancellationToken, _ => areq.GetResponseStream())) + using (var t = s.ObserveReadTimeout(m_timeouts.ReadWriteTimeout)) { - using (var reader = new StreamReader(s)) + using (var reader = new StreamReader(t)) { string endpoint = reader.ReadToEnd(); return endpoint; @@ -115,18 +102,14 @@ namespace Duplicati.Library.Backend #region IBackend Members - public string DisplayName - { - get { return Strings.Idrivee2Backend.DisplayName; } - } + public string DisplayName => Strings.Idrivee2Backend.DisplayName; public string ProtocolKey => "e2"; - public bool SupportsStreaming => true; - public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { - await foreach (IFileEntry file in Connection.ListBucketAsync(m_bucket, m_prefix, false, cancelToken).ConfigureAwait(false)) + var con = await GetConnection(cancelToken).ConfigureAwait(false); + await foreach (IFileEntry file in con.ListBucketAsync(m_bucket, m_prefix, false, cancelToken).ConfigureAwait(false)) yield return file; } @@ -138,7 +121,8 @@ namespace Duplicati.Library.Backend public async Task PutAsync(string remotename, Stream input, CancellationToken cancelToken) { - await Connection.AddFileStreamAsync(m_bucket, GetFullKey(remotename), input, cancelToken).ConfigureAwait(false); + var con = await GetConnection(cancelToken).ConfigureAwait(false); + await con.AddFileStreamAsync(m_bucket, GetFullKey(remotename), input, cancelToken).ConfigureAwait(false); } public async Task GetAsync(string remotename, string localname, CancellationToken cancellationToken) @@ -147,55 +131,46 @@ namespace Duplicati.Library.Backend await GetAsync(remotename, fs, cancellationToken).ConfigureAwait(false); } - public Task GetAsync(string remotename, Stream output, CancellationToken cancellationToken) + public async Task GetAsync(string remotename, Stream output, CancellationToken cancellationToken) { - return Connection.GetFileStreamAsync(m_bucket, GetFullKey(remotename), output, cancellationToken); + var con = await GetConnection(cancellationToken).ConfigureAwait(false); + await con.GetFileStreamAsync(m_bucket, GetFullKey(remotename), output, cancellationToken).ConfigureAwait(false); } - public Task DeleteAsync(string remotename, CancellationToken cancellationToken) + public async Task DeleteAsync(string remotename, CancellationToken cancellationToken) { - return Connection.DeleteObjectAsync(m_bucket, GetFullKey(remotename), cancellationToken); + var con = await GetConnection(cancellationToken).ConfigureAwait(false); + await con.DeleteObjectAsync(m_bucket, GetFullKey(remotename), cancellationToken).ConfigureAwait(false); } - public IList SupportedCommands - { - get - { - var exts = S3AwsClient.GetAwsExtendedOptions(); + public IList SupportedCommands => + [ + .. S3AwsClient.GetAwsExtendedOptions(), + new CommandLineArgument(AUTH_USERNAME_OPTION, CommandLineArgument.ArgumentType.String, Strings.Idrivee2Backend.KeyIDDescriptionShort, Strings.Idrivee2Backend.KeyIDDescriptionLong,null, [AuthOptionsHelper.AuthUsername], null), + new CommandLineArgument(AUTH_PASSWORD_OPTION, CommandLineArgument.ArgumentType.Password, Strings.Idrivee2Backend.KeySecretDescriptionShort, Strings.Idrivee2Backend.KeySecretDescriptionLong, null, [AuthOptionsHelper.AuthPassword ], null), + .. TimeoutOptionsHelper.GetOptions(), + ]; - var normal = new ICommandLineArgument[] { - new CommandLineArgument("access_key_secret", CommandLineArgument.ArgumentType.Password, Strings.Idrivee2Backend.KeySecretDescriptionShort, Strings.Idrivee2Backend.KeySecretDescriptionLong, null, new[]{"auth-password"}, null), - new CommandLineArgument("access_key_id", CommandLineArgument.ArgumentType.String, Strings.Idrivee2Backend.KeyIDDescriptionShort, Strings.Idrivee2Backend.KeyIDDescriptionLong,null, new[]{"auth-username"}, null) - }; - - return normal.Union(exts).ToList(); - } - } - - public string Description - { - get - { - return Strings.Idrivee2Backend.Description; - } - } + public string Description => Strings.Idrivee2Backend.Description; public Task TestAsync(CancellationToken cancelToken) => this.TestListAsync(cancelToken); - public Task CreateFolderAsync(CancellationToken cancelToken) + public async Task CreateFolderAsync(CancellationToken cancelToken) { + var con = await GetConnection(cancelToken).ConfigureAwait(false); //S3 does not complain if the bucket already exists - return Connection.AddBucketAsync(m_bucket, cancelToken); + await con.AddBucketAsync(m_bucket, cancelToken).ConfigureAwait(false); } #endregion #region IRenameEnabledBackend Members - public Task Rename(string source, string target, CancellationToken cancelToken) + public async Task Rename(string source, string target, CancellationToken cancelToken) { - return Connection.RenameFileAsync(m_bucket, GetFullKey(source), GetFullKey(target), cancelToken); + var con = await GetConnection(cancelToken).ConfigureAwait(false); + await con.RenameFileAsync(m_bucket, GetFullKey(source), GetFullKey(target), cancelToken); } #endregion @@ -210,29 +185,46 @@ namespace Duplicati.Library.Backend #endregion - private IS3Client Connection => m_s3Client; - - public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new[] { m_s3Client.GetDnsHost() }); - - private string GetFullKey(string name) + private async Task GetConnection(CancellationToken cancellationToken) { - //AWS SDK encodes the filenames correctly - return m_prefix + name; + if (m_s3Client == null) + { + (var accessKeyId, var accessKeySecret) = m_auth.GetCredentials(); + // TODO: Do not make blocking calls in the constructor + var host = await GetRegionEndpointAsync("https://api.idrivee2.com/api/service/get_region_end_point/" + accessKeyId, cancellationToken).ConfigureAwait(false); + m_s3Client = new S3AwsClient(accessKeyId, accessKeySecret, null, host, null, true, false, m_timeouts, m_options); + } + + return m_s3Client; } + public async Task GetDNSNamesAsync(CancellationToken cancelToken) + { + var con = await GetConnection(cancelToken).ConfigureAwait(false); + var dnshost = con.GetDnsHost(); + return string.IsNullOrWhiteSpace(dnshost) + ? [] + : [dnshost]; + } + + private string GetFullKey(string? name) + //AWS SDK encodes the filenames correctly + => $"{m_prefix}{name}"; + /// - public IAsyncEnumerable ListAsync(string path, CancellationToken cancellationToken) + public async IAsyncEnumerable ListAsync(string? path, [EnumeratorCancellation] CancellationToken cancellationToken) { var filterPath = GetFullKey(path); if (!string.IsNullOrWhiteSpace(filterPath)) filterPath = Util.AppendDirSeparator(filterPath, "/"); - return m_s3Client.ListBucketAsync(m_bucket, filterPath, true, cancellationToken); + var con = await GetConnection(cancellationToken).ConfigureAwait(false); + await foreach (var files in con.ListBucketAsync(m_bucket, filterPath, true, cancellationToken).ConfigureAwait(false)) + yield return files; } - /// - public Task GetEntryAsync(string path, CancellationToken cancellationToken) - => Task.FromResult(null); + public Task GetEntryAsync(string path, CancellationToken cancellationToken) + => Task.FromResult(null); } } diff --git a/Duplicati/Library/Backend/S3/Duplicati.Library.Backend.S3.csproj b/Duplicati/Library/Backend/S3/Duplicati.Library.Backend.S3.csproj index 148804d3b..70e6d0648 100644 --- a/Duplicati/Library/Backend/S3/Duplicati.Library.Backend.S3.csproj +++ b/Duplicati/Library/Backend/S3/Duplicati.Library.Backend.S3.csproj @@ -5,6 +5,8 @@ Library Duplicati.Library.Backend Copyright © 2025 Team Duplicati, MIT license + enable + enable diff --git a/Duplicati/Library/Backend/S3/IS3Client.cs b/Duplicati/Library/Backend/S3/IS3Client.cs index 3a0cb65b9..d71c7ccee 100644 --- a/Duplicati/Library/Backend/S3/IS3Client.cs +++ b/Duplicati/Library/Backend/S3/IS3Client.cs @@ -18,10 +18,6 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System; -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; using Duplicati.Library.Interface; namespace Duplicati.Library.Backend @@ -36,10 +32,10 @@ namespace Duplicati.Library.Backend Task RenameFileAsync(string bucketName, string source, string target, CancellationToken cancelToken); - Task GetFileStreamAsync(string bucketName, string keyName, System.IO.Stream target, CancellationToken cancelToken); + Task GetFileStreamAsync(string bucketName, string keyName, Stream target, CancellationToken cancelToken); - string GetDnsHost(); + string? GetDnsHost(); - Task AddFileStreamAsync(string bucketName, string keyName, System.IO.Stream source, CancellationToken cancelToken); + Task AddFileStreamAsync(string bucketName, string keyName, Stream source, CancellationToken cancelToken); } } \ No newline at end of file diff --git a/Duplicati/Library/Backend/S3/S3AwsClient.cs b/Duplicati/Library/Backend/S3/S3AwsClient.cs index 5fd16627b..9b7aa21ff 100644 --- a/Duplicati/Library/Backend/S3/S3AwsClient.cs +++ b/Duplicati/Library/Backend/S3/S3AwsClient.cs @@ -25,12 +25,8 @@ using Amazon.S3.Model; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using Duplicati.Library.Utility; -using System; -using System.Collections.Generic; -using System.Linq; +using Duplicati.Library.Utility.Options; using System.Runtime.CompilerServices; -using System.Threading; -using System.Threading.Tasks; namespace Duplicati.Library.Backend { @@ -39,23 +35,23 @@ namespace Duplicati.Library.Backend /// public class S3AwsClient : IS3Client { - private static readonly string LOGTAG = Logging.Log.LogTagFromType(); private const int ITEM_LIST_LIMIT = 1000; private const string EXT_OPTION_PREFIX = "s3-ext-"; - private readonly string m_locationConstraint; - private readonly string m_storageClass; - private AmazonS3Client m_client; + private readonly string? m_locationConstraint; + private readonly string? m_storageClass; + private readonly AmazonS3Client m_client; private readonly bool m_useChunkEncoding; - private readonly string m_dnsHost; + private readonly string? m_dnsHost; private readonly bool m_useV2ListApi; + private readonly TimeoutOptionsHelper.Timeouts m_timeouts; - public S3AwsClient(string awsID, string awsKey, string locationConstraint, string servername, - string storageClass, bool useSSL, bool disableChunkEncoding, Dictionary options) + public S3AwsClient(string awsID, string awsKey, string? locationConstraint, string servername, + string? storageClass, bool useSSL, bool disableChunkEncoding, TimeoutOptionsHelper.Timeouts timeouts, Dictionary options) { - var cfg = S3AwsClient.GetDefaultAmazonS3Config(); + var cfg = GetDefaultAmazonS3Config(); cfg.UseHttp = !useSSL; cfg.ServiceURL = (useSSL ? "https://" : "http://") + servername; @@ -63,6 +59,7 @@ namespace Duplicati.Library.Backend m_client = new AmazonS3Client(awsID, awsKey, cfg); + m_timeouts = timeouts; m_useV2ListApi = string.Equals(options.GetValueOrDefault("list-api-version", "v1"), "v2", StringComparison.OrdinalIgnoreCase); m_locationConstraint = locationConstraint; m_storageClass = storageClass; @@ -80,7 +77,7 @@ namespace Duplicati.Library.Backend if (!string.IsNullOrEmpty(m_locationConstraint)) request.BucketRegionName = m_locationConstraint; - return m_client.PutBucketAsync(request, cancelToken); + return Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => m_client.PutBucketAsync(request, ct)); } public static AmazonS3Config GetDefaultAmazonS3Config() @@ -123,7 +120,7 @@ namespace Duplicati.Library.Backend return x; }); - public virtual async Task GetFileStreamAsync(string bucketName, string keyName, System.IO.Stream target, CancellationToken cancelToken) + public virtual async Task GetFileStreamAsync(string bucketName, string keyName, Stream target, CancellationToken cancelToken) { try { @@ -133,15 +130,10 @@ namespace Duplicati.Library.Backend Key = keyName }; - using (var objectGetResponse = await m_client.GetObjectAsync(objectGetRequest).ConfigureAwait(false)) + using (var objectGetResponse = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => m_client.GetObjectAsync(objectGetRequest, ct)).ConfigureAwait(false)) using (var s = objectGetResponse.ResponseStream) - { - // TODO: This does not work and throws InvalidOperationException() - try { s.ReadTimeout = (int)TimeSpan.FromMinutes(1).TotalMilliseconds; } - catch { } - - await Utility.Utility.CopyStreamAsync(s, target, cancelToken).ConfigureAwait(false); - } + using (var t = s.ObserveReadTimeout(m_timeouts.ReadWriteTimeout)) + await Utility.Utility.CopyStreamAsync(t, target, cancelToken).ConfigureAwait(false); } catch (AmazonS3Exception s3Ex) { @@ -151,19 +143,20 @@ namespace Duplicati.Library.Backend } - public string GetDnsHost() + public string? GetDnsHost() { return m_dnsHost; } - public virtual async Task AddFileStreamAsync(string bucketName, string keyName, System.IO.Stream source, + public virtual async Task AddFileStreamAsync(string bucketName, string keyName, Stream source, CancellationToken cancelToken) { + using var ts = source.ObserveReadTimeout(m_timeouts.ReadWriteTimeout); var objectAddRequest = new PutObjectRequest { BucketName = bucketName, Key = keyName, - InputStream = source, + InputStream = ts, UseChunkEncoding = m_useChunkEncoding }; if (!string.IsNullOrWhiteSpace(m_storageClass)) @@ -192,7 +185,7 @@ namespace Duplicati.Library.Backend Key = keyName }; - return m_client.DeleteObjectAsync(objectDeleteRequest, cancellationToken); + return Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancellationToken, ct => m_client.DeleteObjectAsync(objectDeleteRequest, ct)); } /// @@ -206,7 +199,7 @@ namespace Duplicati.Library.Backend public async IAsyncEnumerable ListBucketAsync(string bucketName, string prefix, bool recursive, [EnumeratorCancellation] CancellationToken cancellationToken) { var isTruncated = true; - string filename = null; + string? filename = null; var delimiter = recursive ? "" : "/"; if (!string.IsNullOrWhiteSpace(prefix)) prefix = Util.AppendDirSeparator(prefix, "/"); @@ -232,7 +225,7 @@ namespace Duplicati.Library.Backend MaxKeys = ITEM_LIST_LIMIT, Delimiter = delimiter }; - listResponse = await m_client.ListObjectsV2Async(listRequest, cancellationToken).ConfigureAwait(false); + listResponse = await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancellationToken, ct => m_client.ListObjectsV2Async(listRequest, ct)).ConfigureAwait(false); } else { @@ -245,7 +238,7 @@ namespace Duplicati.Library.Backend MaxKeys = ITEM_LIST_LIMIT, Delimiter = delimiter }; - var listResponsev1 = await m_client.ListObjectsAsync(listRequest, cancellationToken).ConfigureAwait(false); + var listResponsev1 = await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancellationToken, ct => m_client.ListObjectsAsync(listRequest, ct)).ConfigureAwait(false); // Map the V1 response to the V2 response listResponse = new ListObjectsV2Response @@ -314,7 +307,7 @@ namespace Duplicati.Library.Backend DestinationKey = target }; - await m_client.CopyObjectAsync(copyObjectRequest, cancelToken).ConfigureAwait(false); + await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, ct => m_client.CopyObjectAsync(copyObjectRequest, ct)).ConfigureAwait(false); await DeleteObjectAsync(bucketName, source, cancelToken).ConfigureAwait(false); } @@ -324,7 +317,6 @@ namespace Duplicati.Library.Backend { if (m_client != null) m_client.Dispose(); - m_client = null; } #endregion diff --git a/Duplicati/Library/Backend/S3/S3Backend.cs b/Duplicati/Library/Backend/S3/S3Backend.cs index e7e310a2a..6db810f3c 100644 --- a/Duplicati/Library/Backend/S3/S3Backend.cs +++ b/Duplicati/Library/Backend/S3/S3Backend.cs @@ -21,15 +21,9 @@ using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Runtime.CompilerServices; +using Duplicati.Library.Utility.Options; using System.Text; using System.Text.RegularExpressions; -using System.Threading; -using System.Threading.Tasks; namespace Duplicati.Library.Backend { @@ -37,6 +31,9 @@ namespace Duplicati.Library.Backend { private static readonly string LOGTAG = Logging.Log.LogTagFromType(); + private const string AUTH_USERNAME_OPTION = "aws-access-key-id"; + private const string AUTH_PASSWORD_OPTION = "aws-secret-access-key"; + private const string STORAGECLASS_OPTION = "s3-storage-class"; private const string SERVER_NAME = "s3-server-name"; private const string LOCATION_OPTION = "s3-location-constraint"; @@ -180,33 +177,29 @@ namespace Duplicati.Library.Backend private readonly bool m_recurseLists; private const string DEFAULT_S3_HOST = "s3.amazonaws.com"; - private IS3Client s3Client; + private readonly IS3Client m_s3Client; public S3() { + m_bucket = null!; + m_prefix = null!; + m_s3Client = null!; } - public S3(string url, Dictionary options) + public S3(string url, Dictionary options) { var uri = new Utility.Uri(url); uri.RequireHost(); m_bucket = uri.Host; m_prefix = uri.Path; + var timeout = TimeoutOptionsHelper.Parse(options); - if (!options.TryGetValue("aws-access-key-id", out var awsID)) - options.TryGetValue("auth-username", out awsID); - if (!options.TryGetValue("aws-secret-access-key", out var awsKey)) - options.TryGetValue("auth-password", out awsKey); + var auth = AuthOptionsHelper.ParseWithAlias(options, uri, AUTH_USERNAME_OPTION, AUTH_PASSWORD_OPTION); - if (!string.IsNullOrEmpty(uri.Username)) - awsID = uri.Username; - if (!string.IsNullOrEmpty(uri.Password)) - awsKey = uri.Password; - - if (string.IsNullOrEmpty(awsID)) + if (!auth.HasUsername) throw new UserInformationException(Strings.S3Backend.NoAMZUserIDError, "S3NoAmzUserID"); - if (string.IsNullOrEmpty(awsKey)) + if (auth.HasPassword) throw new UserInformationException(Strings.S3Backend.NoAMZKeyError, "S3NoAmzKey"); var useSSL = Utility.Utility.ParseBoolOption(options, SSL_OPTION); @@ -240,13 +233,14 @@ namespace Duplicati.Library.Backend var s3ClientOptionValue = options.GetValueOrDefault(S3_CLIENT_OPTION); + (var awsID, var awsKey) = auth.GetCredentials(); if (string.IsNullOrWhiteSpace(s3ClientOptionValue) || string.Equals(s3ClientOptionValue, "aws", StringComparison.OrdinalIgnoreCase)) { - s3Client = new S3AwsClient(awsID, awsKey, locationConstraint, hostname, storageClass, useSSL, disableChunkEncoding, options); + m_s3Client = new S3AwsClient(awsID, awsKey, locationConstraint, hostname, storageClass, useSSL, disableChunkEncoding, timeout, options); } else if (string.Equals(s3ClientOptionValue, "minio", StringComparison.OrdinalIgnoreCase)) { - s3Client = new S3MinioClient(awsID, awsKey, locationConstraint, hostname, storageClass, useSSL, options); + m_s3Client = new S3MinioClient(awsID, awsKey, locationConstraint, hostname, storageClass, useSSL, timeout, options); } else { @@ -310,8 +304,8 @@ namespace Duplicati.Library.Backend { get { - StringBuilder hostnames = new StringBuilder(); - StringBuilder locations = new StringBuilder(); + var hostnames = new StringBuilder(); + var locations = new StringBuilder(); foreach (var s in KNOWN_S3_PROVIDERS) hostnames.AppendLine(string.Format("{0}: {1}", s.Key, s.Value)); @@ -320,9 +314,9 @@ namespace Duplicati.Library.Backend var exts = S3AwsClient.GetAwsExtendedOptions(); - var normal = new ICommandLineArgument[] { - new CommandLineArgument("aws-secret-access-key", CommandLineArgument.ArgumentType.Password, Strings.S3Backend.AMZKeyDescriptionShort, Strings.S3Backend.AMZKeyDescriptionLong,null, new string[] {"auth-password"}, null ), - new CommandLineArgument("aws-access-key-id", CommandLineArgument.ArgumentType.String, Strings.S3Backend.AMZUserIDDescriptionShort, Strings.S3Backend.AMZUserIDDescriptionLong, null, new string[] {"auth-username"}, null), + return [ + new CommandLineArgument("aws-access-key-id", CommandLineArgument.ArgumentType.String, Strings.S3Backend.AMZUserIDDescriptionShort, Strings.S3Backend.AMZUserIDDescriptionLong, null, [AuthOptionsHelper.AuthUsername], null), + new CommandLineArgument("aws-secret-access-key", CommandLineArgument.ArgumentType.Password, Strings.S3Backend.AMZKeyDescriptionShort, Strings.S3Backend.AMZKeyDescriptionLong,null, [AuthOptionsHelper.AuthPassword], null ), new CommandLineArgument(STORAGECLASS_OPTION, CommandLineArgument.ArgumentType.String, Strings.S3Backend.S3StorageclassDescriptionShort, Strings.S3Backend.S3StorageclassDescriptionLong), new CommandLineArgument(SERVER_NAME, CommandLineArgument.ArgumentType.String, Strings.S3Backend.S3ServerNameDescriptionShort, Strings.S3Backend.S3ServerNameDescriptionLong(hostnames.ToString()), DEFAULT_S3_HOST), new CommandLineArgument(LOCATION_OPTION, CommandLineArgument.ArgumentType.String, Strings.S3Backend.S3LocationDescriptionShort, Strings.S3Backend.S3LocationDescriptionLong(locations.ToString())), @@ -331,12 +325,9 @@ namespace Duplicati.Library.Backend new CommandLineArgument(S3_DISABLE_CHUNK_ENCODING_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.S3Backend.DescriptionDisableChunkEncodingShort, Strings.S3Backend.DescriptionDisableChunkEncodingLong, "false"), new CommandLineArgument(S3_LIST_API_VERSION_OPTION, CommandLineArgument.ArgumentType.Enumeration, Strings.S3Backend.DescriptionListApiVersionShort, Strings.S3Backend.DescriptionListApiVersionLong, "v1", null, ["v1", "v2"]), new CommandLineArgument(S3_RECURSIVE_LIST, CommandLineArgument.ArgumentType.Boolean, Strings.S3Backend.DescriptionRecursiveListShort, Strings.S3Backend.DescriptionRecursiveListLong, "false"), - new CommandLineArgument("auth-password", CommandLineArgument.ArgumentType.Password, Strings.S3Backend.AuthPasswordDescriptionShort, Strings.S3Backend.AuthPasswordDescriptionLong), - new CommandLineArgument("auth-username", CommandLineArgument.ArgumentType.String, Strings.S3Backend.AuthUsernameDescriptionShort, Strings.S3Backend.AuthUsernameDescriptionLong), - }; - - return normal.Union(exts).ToList(); - + .. TimeoutOptionsHelper.GetOptions(), + .. exts + ]; } } @@ -372,37 +363,42 @@ namespace Duplicati.Library.Backend public void Dispose() { - s3Client?.Dispose(); - s3Client = null; + m_s3Client?.Dispose(); } #endregion private IS3Client Connection { - get { return s3Client; } + get { return m_s3Client; } } - public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new[] { s3Client.GetDnsHost() }); - - private string GetFullKey(string name) + public Task GetDNSNamesAsync(CancellationToken cancelToken) { - //AWS SDK encodes the filenames correctly - return m_prefix + name; + var host = m_s3Client.GetDnsHost(); + return Task.FromResult( + string.IsNullOrWhiteSpace(host) + ? [] + : [host] + ); } + private string GetFullKey(string? name) + //AWS SDK encodes the filenames correctly + => $"{m_prefix}{name}"; + /// - public IAsyncEnumerable ListAsync(string path, CancellationToken cancellationToken) + public IAsyncEnumerable ListAsync(string? path, CancellationToken cancellationToken) { var filterPath = GetFullKey(path); if (!string.IsNullOrWhiteSpace(filterPath)) filterPath = Util.AppendDirSeparator(filterPath, "/"); - return s3Client.ListBucketAsync(m_bucket, filterPath, m_recurseLists, cancellationToken); + return m_s3Client.ListBucketAsync(m_bucket, filterPath, m_recurseLists, cancellationToken); } /// - public Task GetEntryAsync(string path, CancellationToken cancellationToken) - => Task.FromResult(null); + public Task GetEntryAsync(string path, CancellationToken cancellationToken) + => Task.FromResult(null); } } diff --git a/Duplicati/Library/Backend/S3/S3Config.cs b/Duplicati/Library/Backend/S3/S3Config.cs index 4558cad18..03777b5df 100644 --- a/Duplicati/Library/Backend/S3/S3Config.cs +++ b/Duplicati/Library/Backend/S3/S3Config.cs @@ -1,35 +1,32 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System; -using System.Collections.Generic; using Duplicati.Library.Interface; -using Duplicati.Library.Localization.Short; namespace Duplicati.Library.Backend { public class S3Config : IWebModule { private const ConfigType DEFAULT_CONFIG_TYPE = ConfigType.Providers; - private static readonly string DEFAULT_CONFIG_TYPE_STR = Enum.GetName(typeof(ConfigType), DEFAULT_CONFIG_TYPE); + private static readonly string DEFAULT_CONFIG_TYPE_STR = DEFAULT_CONFIG_TYPE.ToString(); private const string KEY_CONFIGTYPE = "s3-config"; public enum ConfigType @@ -48,13 +45,11 @@ namespace Duplicati.Library.Backend public IDictionary Execute(IDictionary options) { - string k; - options.TryGetValue(KEY_CONFIGTYPE, out k); + options.TryGetValue(KEY_CONFIGTYPE, out var k); if (string.IsNullOrWhiteSpace(k)) k = DEFAULT_CONFIG_TYPE_STR; - ConfigType ct; - if (!Enum.TryParse(k, true, out ct)) + if (!Enum.TryParse(k, true, out var ct)) ct = DEFAULT_CONFIG_TYPE; switch (ct) @@ -70,22 +65,16 @@ namespace Duplicati.Library.Backend } } - public string Key { get { return "s3-getconfig"; } } + public string Key => "s3-getconfig"; - public string DisplayName { get { return LC.L("S3 configuration module"); } } + public string DisplayName => Strings.S3Config.DisplayName; - public string Description { get { return LC.L("Expose S3 configuration as a web module"); } } + public string Description => Strings.S3Config.Description; - public IList SupportedCommands - { - get - { - return new List([ - new CommandLineArgument(KEY_CONFIGTYPE, CommandLineArgument.ArgumentType.Enumeration, LC.L("The config to get"), LC.L("Provide different config values"), DEFAULT_CONFIG_TYPE_STR, Enum.GetNames(typeof(ConfigType))) - - ]); - } - } + public IList SupportedCommands => + [ + new CommandLineArgument(KEY_CONFIGTYPE, CommandLineArgument.ArgumentType.Enumeration, Strings.S3Config.ConfigTypeShort, Strings.S3Config.ConfigTypeLong, DEFAULT_CONFIG_TYPE_STR, Enum.GetNames(typeof(ConfigType))) + ]; #endregion } diff --git a/Duplicati/Library/Backend/S3/S3IAM.cs b/Duplicati/Library/Backend/S3/S3IAM.cs index d4aadd439..5c6b6d09e 100644 --- a/Duplicati/Library/Backend/S3/S3IAM.cs +++ b/Duplicati/Library/Backend/S3/S3IAM.cs @@ -19,13 +19,9 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System; -using System.Collections.Generic; -using System.Linq; using Duplicati.Library.Interface; using Amazon.IdentityManagement; using Amazon.IdentityManagement.Model; -using Duplicati.Library.Localization.Short; using Duplicati.Library.Utility; namespace Duplicati.Library.Backend @@ -67,71 +63,61 @@ namespace Duplicati.Library.Backend } "; - public string Key { get { return "s3-iamconfig"; } } + public string Key => "s3-iamconfig"; - public string DisplayName { get { return LC.L("S3 IAM support module"); } } + public string DisplayName => Strings.S3IAM.DisplayName; - public string Description { get { return LC.L("Expose S3 IAM manipulation as a web module"); } } + public string Description => Strings.S3IAM.Description; - public IList SupportedCommands - { - get - { - return new List([ - new CommandLineArgument(KEY_OPERATION, CommandLineArgument.ArgumentType.Enumeration, LC.L("The operation to perform"), LC.L("Select the operation to perform"), null, Enum.GetNames(typeof(Operation))), - new CommandLineArgument(KEY_USERNAME, CommandLineArgument.ArgumentType.String, LC.L("The username"), LC.L("The Amazon Access Key ID")), - new CommandLineArgument(KEY_PASSWORD, CommandLineArgument.ArgumentType.String, LC.L("The password"), LC.L("The Amazon Secret Key")), + public IList SupportedCommands => new List([ + new CommandLineArgument(KEY_OPERATION, CommandLineArgument.ArgumentType.Enumeration, Strings.S3IAM.OperationShort, Strings.S3IAM.OperationLong, null, Enum.GetNames(typeof(Operation))), + new CommandLineArgument(KEY_USERNAME, CommandLineArgument.ArgumentType.String, Strings.S3IAM.UsernameShort, Strings.S3IAM.UsernameLong), + new CommandLineArgument(KEY_PASSWORD, CommandLineArgument.ArgumentType.String, Strings.S3IAM.PasswordShort, Strings.S3IAM.PasswordLong) ]); - } - } public IDictionary Execute(IDictionary options) { - options.TryGetValue(KEY_OPERATION, out string operationstring); - options.TryGetValue(KEY_USERNAME, out string username); - options.TryGetValue(KEY_PASSWORD, out string password); - options.TryGetValue(KEY_PATH, out string path); + options.TryGetValue(KEY_OPERATION, out var operationstring); + options.TryGetValue(KEY_USERNAME, out var username); + options.TryGetValue(KEY_PASSWORD, out var password); + options.TryGetValue(KEY_PATH, out var path); ValidateArgument(operationstring, KEY_OPERATION); - if (!Enum.TryParse(operationstring, true, out Operation operation)) + if (!Enum.TryParse(operationstring, true, out var operation)) throw new ArgumentException(string.Format("Unable to parse {0} as an operation", operationstring)); switch (operation) { case Operation.GetPolicyDoc: ValidateArgument(path, KEY_PATH); - return GetPolicyDoc(path); + return GetPolicyDoc(path!); case Operation.CreateIAMUser: ValidateArgument(username, KEY_USERNAME); ValidateArgument(password, KEY_PASSWORD); ValidateArgument(path, KEY_PATH); - return CreateUnprivilegedUser(username, password, path); + return CreateUnprivilegedUser(username!, password!, path!); default: ValidateArgument(username, KEY_USERNAME); ValidateArgument(password, KEY_PASSWORD); - return CanCreateUser(username, password); + return CanCreateUser(username!, password!); } } - private static void ValidateArgument(string arg, string type) + private static void ValidateArgument(string? arg, string type) { if (string.IsNullOrWhiteSpace(arg)) - { throw new ArgumentNullException(type); - } } private static IDictionary GetPolicyDoc(string path) - { - return new Dictionary + => new Dictionary { ["doc"] = GeneratePolicyDoc(path) }; - } private static string GeneratePolicyDoc(string path) { @@ -149,7 +135,7 @@ namespace Duplicati.Library.Backend } - private static Boolean DetermineIfCreateUserIsAllowed(User user, AmazonIdentityManagementServiceClient cl) + private static bool DetermineIfCreateUserIsAllowed(User user, AmazonIdentityManagementServiceClient cl) { var simulatePrincipalPolicy = new SimulatePrincipalPolicyRequest { diff --git a/Duplicati/Library/Backend/S3/S3MinioClient.cs b/Duplicati/Library/Backend/S3/S3MinioClient.cs index 3f7d48f1c..3c1fbb209 100644 --- a/Duplicati/Library/Backend/S3/S3MinioClient.cs +++ b/Duplicati/Library/Backend/S3/S3MinioClient.cs @@ -18,17 +18,13 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System; -using System.Collections.Generic; -using System.IO; using System.Reactive.Linq; using System.Runtime.CompilerServices; -using System.Threading; using System.Threading.Channels; -using System.Threading.Tasks; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using Duplicati.Library.Utility; +using Duplicati.Library.Utility.Options; using Minio; using Minio.Exceptions; @@ -38,13 +34,15 @@ namespace Duplicati.Library.Backend { private static readonly string Logtag = Logging.Log.LogTagFromType(); - private MinioClient m_client; - private readonly string m_locationConstraint; + private readonly MinioClient m_client; + private readonly string? m_locationConstraint; private readonly string m_dnsHost; + private readonly TimeoutOptionsHelper.Timeouts m_timeouts; - public S3MinioClient(string awsID, string awsKey, string locationConstraint, - string servername, string storageClass, bool useSSL, Dictionary options) + public S3MinioClient(string awsID, string awsKey, string? locationConstraint, + string servername, string? storageClass, bool useSSL, TimeoutOptionsHelper.Timeouts timeouts, Dictionary options) { + m_timeouts = timeouts; m_locationConstraint = locationConstraint; m_client = new MinioClient( servername, @@ -84,12 +82,15 @@ namespace Duplicati.Library.Backend public async IAsyncEnumerable ListBucketAsync(string bucketName, string prefix, bool recursive, [EnumeratorCancellation] CancellationToken cancellationToken) { - ThrowExceptionIfBucketDoesNotExist(bucketName); + await ThrowExceptionIfBucketDoesNotExist(bucketName, cancellationToken).ConfigureAwait(false); if (!string.IsNullOrWhiteSpace(prefix)) prefix = Util.AppendDirSeparator(prefix, "/"); - var observable = m_client.ListObjectsAsync(bucketName, prefix, recursive, cancellationToken); + var observable = await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancellationToken, ct + => m_client.ListObjectsAsync(bucketName, prefix, recursive, ct) + ).ConfigureAwait(false); + await foreach (var obj in ToAsyncEnumerable(observable, cancellationToken).ConfigureAwait(false)) { if (obj.Key == prefix || !obj.Key.StartsWith(prefix)) @@ -109,7 +110,9 @@ namespace Duplicati.Library.Backend { try { - await m_client.MakeBucketAsync(bucketName, m_locationConstraint, cancelToken).ConfigureAwait(false); + await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct + => m_client.MakeBucketAsync(bucketName, m_locationConstraint, cancelToken) + ).ConfigureAwait(false); } catch (MinioException e) { @@ -123,7 +126,9 @@ namespace Duplicati.Library.Backend { try { - await m_client.RemoveObjectAsync(bucketName, keyName, cancelToken).ConfigureAwait(false); + await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct + => m_client.RemoveObjectAsync(bucketName, keyName, ct) + ).ConfigureAwait(false); } catch (MinioException e) { @@ -140,8 +145,9 @@ namespace Duplicati.Library.Backend { try { - await m_client.CopyObjectAsync(bucketName, source, - bucketName, target, cancellationToken: cancelToken).ConfigureAwait(false); + await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct + => m_client.CopyObjectAsync(bucketName, source, bucketName, target, cancellationToken: ct) + ).ConfigureAwait(false); } catch (MinioException e) { @@ -163,11 +169,17 @@ namespace Duplicati.Library.Backend // If the object is not found, statObject() throws an exception, // else it means that the object exists. // Execution is successful. - await m_client.StatObjectAsync(bucketName, keyName, cancellationToken: cancelToken).ConfigureAwait(false); + await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct + => m_client.StatObjectAsync(bucketName, keyName, cancellationToken: cancelToken) + ).ConfigureAwait(false); // Get input stream to have content of 'my-objectname' from 'my-bucketname' await m_client.GetObjectAsync(bucketName, keyName, - (stream) => { Utility.Utility.CopyStream(stream, target); }).ConfigureAwait(false); + (stream) => + { + using var t = stream.ObserveReadTimeout(m_timeouts.ReadWriteTimeout); + Utility.Utility.CopyStream(t, target); + }).ConfigureAwait(false); } catch (MinioException e) { @@ -180,7 +192,7 @@ namespace Duplicati.Library.Backend } } - public string GetDnsHost() + public string? GetDnsHost() { return m_dnsHost; } @@ -188,14 +200,15 @@ namespace Duplicati.Library.Backend public virtual async Task AddFileStreamAsync(string bucketName, string keyName, Stream source, CancellationToken cancelToken) { - ThrowExceptionIfBucketDoesNotExist(bucketName); + await ThrowExceptionIfBucketDoesNotExist(bucketName, cancelToken).ConfigureAwait(false); try { + using var t = source.ObserveReadTimeout(m_timeouts.ReadWriteTimeout); await m_client.PutObjectAsync(bucketName, keyName, - source, - source.Length, + t, + t.Length, "application/octet-stream", cancellationToken: cancelToken); } catch (MinioException e) @@ -212,18 +225,17 @@ namespace Duplicati.Library.Backend throw new FileMissingException($"File {keyName} not found in bucket {bucketName}"); } - private void ThrowExceptionIfBucketDoesNotExist(string bucketName) - { - if (!m_client.BucketExistsAsync(bucketName).Await()) - throw new FolderMissingException($"Bucket {bucketName} does not exist."); - } - + private Task ThrowExceptionIfBucketDoesNotExist(string bucketName, CancellationToken cancelToken) + => Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, async ct => + { + if (!await m_client.BucketExistsAsync(bucketName, ct)) + throw new FolderMissingException($"Bucket {bucketName} does not exist."); + }); #region IDisposable Members public void Dispose() { - m_client = null; } #endregion diff --git a/Duplicati/Library/Backend/S3/Strings.cs b/Duplicati/Library/Backend/S3/Strings.cs index c7721bcdd..0daf410fd 100644 --- a/Duplicati/Library/Backend/S3/Strings.cs +++ b/Duplicati/Library/Backend/S3/Strings.cs @@ -55,4 +55,25 @@ namespace Duplicati.Library.Backend.Strings public static string DescriptionRecursiveListLong { get { return LC.L(@"To reduce the number of objects listed, the default is to only list the first level of objects. Use this option to list all objects in the bucket."); } } public static string UnknownS3ClientError(string client) { return LC.L(@"Unknown S3 client: {0}", client); } } + + internal static class S3Config + { + public static string DisplayName => LC.L("S3 configuration module"); + public static string Description => LC.L("Expose S3 configuration as a web module"); + public static string ConfigTypeShort => LC.L("The config to get"); + public static string ConfigTypeLong => LC.L("Provide different config values"); + } + + internal static class S3IAM + { + public static string DisplayName => LC.L("S3 IAM support module"); + public static string Description => LC.L("Expose S3 IAM manipulation as a web module"); + public static string OperationShort => LC.L("The operation to perform"); + public static string OperationLong => LC.L("Select the operation to perform"); + public static string UsernameShort => LC.L("The username to use"); + public static string UsernameLong => LC.L("The Amazon Access Key ID"); + public static string PasswordShort => LC.L("The password to use"); + public static string PasswordLong => LC.L("The Amazon Secret Key"); + } + } From e89897829dae7889a6791afc3fc3d16ff5be769f Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 21 Mar 2025 14:13:21 +0100 Subject: [PATCH 20/48] Updated Jottacloud backend --- Duplicati/Library/Backend/FTP/FTPBackend.cs | 2 +- ...uplicati.Library.Backend.Jottacloud.csproj | 2 + .../Library/Backend/Jottacloud/Jottacloud.cs | 312 ++++++++++-------- .../Library/Backend/Jottacloud/Strings.cs | 5 +- Duplicati/Library/Backend/S3/S3AwsClient.cs | 2 +- Duplicati/Library/Backend/S3/S3Backend.cs | 2 +- Duplicati/Library/Backend/S3/S3MinioClient.cs | 2 +- 7 files changed, 179 insertions(+), 148 deletions(-) diff --git a/Duplicati/Library/Backend/FTP/FTPBackend.cs b/Duplicati/Library/Backend/FTP/FTPBackend.cs index ecb87bae4..35c8f6ca0 100644 --- a/Duplicati/Library/Backend/FTP/FTPBackend.cs +++ b/Duplicati/Library/Backend/FTP/FTPBackend.cs @@ -457,7 +457,7 @@ namespace Duplicati.Library.Backend { var client = await CreateClient(cancelToken).ConfigureAwait(false); var clientRemoteName = PreparePathForClient(remotename); - await using var inputStream = await client.OpenRead(clientRemoteName, token: cancelToken); + await using var inputStream = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancelToken, ct => client.OpenRead(clientRemoteName, token: ct)).ConfigureAwait(false); await using var timeoutStream = inputStream.ObserveReadTimeout(_timeouts.ReadWriteTimeout); await CoreUtility.CopyStreamAsync(inputStream, output, false, cancelToken).ConfigureAwait(false); } diff --git a/Duplicati/Library/Backend/Jottacloud/Duplicati.Library.Backend.Jottacloud.csproj b/Duplicati/Library/Backend/Jottacloud/Duplicati.Library.Backend.Jottacloud.csproj index 0b042b92e..3cba2fed0 100644 --- a/Duplicati/Library/Backend/Jottacloud/Duplicati.Library.Backend.Jottacloud.csproj +++ b/Duplicati/Library/Backend/Jottacloud/Duplicati.Library.Backend.Jottacloud.csproj @@ -5,6 +5,8 @@ Library Duplicati.Library.Backend Copyright © 2025 Team Duplicati, MIT license + enable + enable diff --git a/Duplicati/Library/Backend/Jottacloud/Jottacloud.cs b/Duplicati/Library/Backend/Jottacloud/Jottacloud.cs index 9a4c25336..ec4e5f30f 100644 --- a/Duplicati/Library/Backend/Jottacloud/Jottacloud.cs +++ b/Duplicati/Library/Backend/Jottacloud/Jottacloud.cs @@ -22,12 +22,11 @@ using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using Duplicati.Library.Localization.Short; -using System; -using System.Collections.Generic; +using Duplicati.Library.Utility; +using Duplicati.Library.Utility.Options; using System.Net; using System.Runtime.CompilerServices; using System.Security.Cryptography; -using System.Threading; using System.Threading.Tasks; namespace Duplicati.Library.Backend { @@ -35,7 +34,7 @@ namespace Duplicati.Library.Backend // This class is instantiated dynamically in the BackendLoader. public class Jottacloud : IBackend, IStreamingBackend { - private const string AUTHID_OPTION = "authid"; + private static readonly string TOKEN_URL = OAuthHelper.OAUTH_LOGIN_URL("jottacloud"); private const string JFS_ROOT = "https://jfs.jottacloud.com/jfs"; private const string JFS_ROOT_UPLOAD = "https://up.jottacloud.com/jfs"; // Separate host for uploading files private const string JFS_BUILTIN_DEVICE = "Jotta"; // The built-in device used for the built-in Sync and Archive mount points. @@ -60,6 +59,7 @@ namespace Duplicati.Library.Backend private static readonly string JFS_DEFAULT_THREADS = "4"; private readonly int m_threads; private readonly long m_chunksize; + private readonly TimeoutOptionsHelper.Timeouts m_timeouts; private readonly JottacloudAuthHelper m_oauth; @@ -70,23 +70,33 @@ namespace Duplicati.Library.Backend /// static Jottacloud() { - System.Net.ServicePointManager.DefaultConnectionLimit = 1000; + // TODO: Remove this when HttpClient migration is done + ServicePointManager.DefaultConnectionLimit = 1000; } public Jottacloud() { + m_device = null!; + m_mountPoint = null!; + m_path = null!; + m_url_device = null!; + m_url = null!; + m_url_upload = null!; + m_timeouts = null!; + m_oauth = null!; } - public Jottacloud(string url, Dictionary options) + public Jottacloud(string url, Dictionary options) { // Duplicati back-end url for Jottacloud is in format "jottacloud://folder/subfolder", we transform them to // the Jottacloud REST API (JFS) url format "https://jfs.jottacloud.com/jfs/[username]/[device]/[mountpoint]/[folder]/[subfolder]". // Find out what JFS device to use. - if (options.ContainsKey(JFS_DEVICE_OPTION)) + var device = options.GetValueOrDefault(JFS_DEVICE_OPTION); + if (!string.IsNullOrWhiteSpace(device)) { // Custom device specified. - m_device = options[JFS_DEVICE_OPTION]; + m_device = device; if (string.Equals(m_device, JFS_BUILTIN_DEVICE, StringComparison.OrdinalIgnoreCase)) { m_device_builtin = true; // Device is configured, but value set to the built-in device! @@ -105,10 +115,11 @@ namespace Duplicati.Library.Backend } // Find out what JFS mount point to use on the device. - if (options.ContainsKey(JFS_MOUNT_POINT_OPTION)) + var mountPoint = options.GetValueOrDefault(JFS_MOUNT_POINT_OPTION); + if (!string.IsNullOrWhiteSpace(mountPoint)) { // Custom mount point specified. - m_mountPoint = options[JFS_MOUNT_POINT_OPTION]; + m_mountPoint = mountPoint; // If we are using the built-in device make sure we have picked a mount point that we can use. if (m_device_builtin) @@ -136,10 +147,10 @@ namespace Duplicati.Library.Backend m_mountPoint = JFS_DEFAULT_CUSTOM_MOUNT_POINT; // Set a suitable default mount point for custom (backup) devices. } - string authid = null; - if (options.ContainsKey(AUTHID_OPTION)) - authid = options[AUTHID_OPTION]; - m_oauth = new JottacloudAuthHelper(authid); + var authId = AuthIdOptionsHelper.Parse(options) + .RequireCredentials(TOKEN_URL); + + m_oauth = new JottacloudAuthHelper(authId.AuthId!); // Build URL var u = new Utility.Uri(url); @@ -152,36 +163,32 @@ namespace Duplicati.Library.Backend m_url = m_url_device + "/" + m_mountPoint + "/" + m_path; m_url_upload = JFS_ROOT_UPLOAD + "/" + m_oauth.Username + "/" + m_device + "/" + m_mountPoint + "/" + m_path; // Different hostname, else identical to m_url. - m_threads = int.Parse(options.ContainsKey(JFS_THREADS) ? options[JFS_THREADS] : JFS_DEFAULT_THREADS); + var jfsThreads = options.GetValueOrDefault(JFS_THREADS); + if (string.IsNullOrWhiteSpace(jfsThreads)) + jfsThreads = JFS_DEFAULT_THREADS; - if (!options.TryGetValue(JFS_CHUNKSIZE, out var tmp)) - { - tmp = JFS_DEFAULT_CHUNKSIZE; - } + m_threads = int.Parse(jfsThreads); - var chunksize = Utility.Sizeparser.ParseSize(tmp, "mb"); + var jfsChunksize = options.GetValueOrDefault(JFS_CHUNKSIZE); + if (string.IsNullOrWhiteSpace(jfsChunksize)) + jfsChunksize = JFS_DEFAULT_CHUNKSIZE; + + var chunksize = Sizeparser.ParseSize(jfsChunksize, "mb"); // Chunk size is bound by BinaryReader.ReadBytes(length) where length is an int. if (chunksize > int.MaxValue || chunksize < 1024) - { - throw new ArgumentOutOfRangeException(nameof(chunksize), string.Format("The chunk size cannot be less than {0}, nor larger than {1}", Utility.Utility.FormatSizeString(1024), Utility.Utility.FormatSizeString(int.MaxValue))); - } + throw new ArgumentOutOfRangeException(nameof(chunksize), $"The chunk size cannot be less than {Utility.Utility.FormatSizeString(1024)}, nor larger than {Utility.Utility.FormatSizeString(int.MaxValue)}"); m_chunksize = chunksize; + m_timeouts = TimeoutOptionsHelper.Parse(options); } #region IBackend Members - public string DisplayName - { - get { return Strings.Jottacloud.DisplayName; } - } + public string DisplayName => Strings.Jottacloud.DisplayName; - public string ProtocolKey - { - get { return "jottacloud"; } - } + public string ProtocolKey => "jottacloud"; /// public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) @@ -193,14 +200,17 @@ namespace Duplicati.Library.Backend try { // Send request and load XML response. - var req = CreateRequest(System.Net.WebRequestMethods.Http.Get, "", "", false); - var areq = new Utility.AsyncHttpRequest(req); - using (var rs = areq.GetResponseStream()) - doc.Load(rs); + var req = CreateRequest(WebRequestMethods.Http.Get, "", "", false); + var areq = new AsyncHttpRequest(req); + await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, _ => + { + using (var rs = areq.GetResponseStream()) + doc.Load(rs); + }).ConfigureAwait(false); } - catch (System.Net.WebException wex) + catch (WebException wex) { - if (wex.Response is HttpWebResponse response && response.StatusCode == System.Net.HttpStatusCode.NotFound) + if (wex.Response is HttpWebResponse response && response.StatusCode == HttpStatusCode.NotFound) throw new FolderMissingException(wex); throw; } @@ -209,50 +219,61 @@ namespace Duplicati.Library.Backend // We must check for "deleted" attribute, because files/folders which has it is deleted (attribute contains the timestamp of deletion) // so we treat them as non-existent here. var xRoot = doc.DocumentElement; - if (xRoot.Attributes["deleted"] != null) - { + if (xRoot == null) throw new FolderMissingException(); - } - foreach (System.Xml.XmlNode xFolder in xRoot.SelectNodes("folders/folder[not(@deleted)]")) - { - // Subfolders are only listed with name. We can get a timestamp by sending a request for each folder, but that is probably not necessary? - FileEntry fe = new FileEntry(xFolder.Attributes["name"].Value); - fe.IsFolder = true; - yield return fe; - } - foreach (System.Xml.XmlNode xFile in xRoot.SelectNodes("files/file[not(@deleted)]")) - { - var fe = ToFileEntry(xFile); - if (fe != null) + if (xRoot.Attributes["deleted"] != null) + throw new FolderMissingException(); + + var folders = xRoot.SelectNodes("folders/folder[not(@deleted)]"); + if (folders != null) + foreach (System.Xml.XmlNode xFolder in folders) { - yield return fe; + // Subfolders are only listed with name. We can get a timestamp by sending a request for each folder, but that is probably not necessary? + var name = xFolder.Attributes?["name"]?.Value; + if (string.IsNullOrEmpty(name)) + continue; + + yield return new FileEntry(name) { IsFolder = true }; + } + + var files = xRoot.SelectNodes("files/file[not(@deleted)]"); + if (files != null) + foreach (System.Xml.XmlNode xFile in files) + { + var fe = ToFileEntry(xFile); + if (fe != null) + yield return fe; } - } } - public static IFileEntry ToFileEntry(System.Xml.XmlNode xFile) + public static IFileEntry? ToFileEntry(System.Xml.XmlNode? xFile) { - string name = xFile.Attributes["name"].Value; + if (xFile == null) + return null; + + var name = xFile.Attributes?["name"]?.Value; + if (string.IsNullOrWhiteSpace(name)) + return null; + // Normal files have an "currentRevision", which represent the most recent successfully upload // (could also checked that currentRevision/state is "COMPLETED", but should not be necessary). // There might also be a newer "latestRevision" coming from an incomplete or corrupt upload, // but we ignore that here and use the information about the last valid version. - System.Xml.XmlNode xRevision = xFile.SelectSingleNode("currentRevision"); + var xRevision = xFile.SelectSingleNode("currentRevision"); if (xRevision != null) { - System.Xml.XmlNode xState = xRevision.SelectSingleNode("state"); + var xState = xRevision.SelectSingleNode("state"); if (xState != null && xState.InnerText == "COMPLETED") // Think "currentRevision" always is a complete version, but just to be on the safe side.. { - System.Xml.XmlNode xSize = xRevision.SelectSingleNode("size"); + var xSize = xRevision.SelectSingleNode("size"); long size; if (xSize == null || !long.TryParse(xSize.InnerText, out size)) size = -1; - DateTime lastModified; - System.Xml.XmlNode xModified = xRevision.SelectSingleNode("modified"); // There is created, modified and updated time stamps, but not last accessed. - if (xModified == null || !DateTime.TryParseExact(xModified.InnerText, JFS_DATE_FORMAT, System.Globalization.CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.AdjustToUniversal, out lastModified)) + + var xModified = xRevision.SelectSingleNode("modified"); // There is created, modified and updated time stamps, but not last accessed. + if (xModified == null || !DateTime.TryParseExact(xModified.InnerText, JFS_DATE_FORMAT, System.Globalization.CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.AdjustToUniversal, out var lastModified)) lastModified = new DateTime(); - FileEntry fe = new FileEntry(name, size, lastModified, lastModified); - return fe; + return new FileEntry(name, size, lastModified, lastModified); } } return null; @@ -261,22 +282,26 @@ namespace Duplicati.Library.Backend /// /// Retrieves info for a single file (used to determine file size for chunking) /// - /// - /// - public IFileEntry Info(string remotename) + /// The remote file name + /// The cancellation token + /// The file entry + public async Task Info(string remotename, CancellationToken cancelToken) { var doc = new System.Xml.XmlDocument(); try { // Send request and load XML response. - var req = CreateRequest(System.Net.WebRequestMethods.Http.Get, remotename, "", false); + var req = CreateRequest(WebRequestMethods.Http.Get, remotename, "", false); var areq = new Utility.AsyncHttpRequest(req); - using (var rs = areq.GetResponseStream()) - doc.Load(rs); + await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => + { + using (var rs = areq.GetResponseStream()) + doc.Load(rs); + }).ConfigureAwait(false); } - catch (System.Net.WebException wex) + catch (WebException wex) { - if (wex.Response is HttpWebResponse response && response.StatusCode == System.Net.HttpStatusCode.NotFound) + if (wex.Response is HttpWebResponse response && response.StatusCode == HttpStatusCode.NotFound) throw new FileMissingException(wex); throw; } @@ -285,77 +310,74 @@ namespace Duplicati.Library.Backend // We must check for "deleted" attribute, because files/folders which has it is deleted (attribute contains the timestamp of deletion) // so we treat them as non-existent here. var xFile = doc.DocumentElement; - if (xFile.Attributes["deleted"] != null) - { + if (xFile?.Attributes["deleted"] != null) throw new FileMissingException(string.Format("{0}: {1}", LC.L("The requested file does not exist"), remotename)); - } return ToFileEntry(xFile); } public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) { - using (System.IO.FileStream fs = System.IO.File.OpenRead(filename)) + using (var fs = File.OpenRead(filename)) await PutAsync(remotename, fs, cancelToken).ConfigureAwait(false); } public async Task GetAsync(string remotename, string filename, CancellationToken cancelToken) { - using (System.IO.FileStream fs = System.IO.File.Create(filename)) + using (var fs = File.Create(filename)) await GetAsync(remotename, fs, cancelToken).ConfigureAwait(false); } public Task DeleteAsync(string remotename, CancellationToken cancelToken) { - System.Net.HttpWebRequest req = CreateRequest(System.Net.WebRequestMethods.Http.Post, remotename, "rm=true", false); // rm=true means permanent delete, dl=true would be move to trash. - Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req); - using (System.Net.HttpWebResponse resp = (System.Net.HttpWebResponse)areq.GetResponse()) - { } + var req = CreateRequest(WebRequestMethods.Http.Post, remotename, "rm=true", false); // rm=true means permanent delete, dl=true would be move to trash. + var areq = new Utility.AsyncHttpRequest(req); - return Task.CompletedTask; - } - - public IList SupportedCommands - { - get + return Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => { - return new List([ - new CommandLineArgument(AUTHID_OPTION, CommandLineArgument.ArgumentType.Password, Strings.Jottacloud.AuthidShort, Strings.Jottacloud.AuthidLong(OAuthHelper.OAUTH_LOGIN_URL("jottacloud"))), - new CommandLineArgument(JFS_DEVICE_OPTION, CommandLineArgument.ArgumentType.String, Strings.Jottacloud.DescriptionDeviceShort, Strings.Jottacloud.DescriptionDeviceLong(JFS_MOUNT_POINT_OPTION)), - new CommandLineArgument(JFS_MOUNT_POINT_OPTION, CommandLineArgument.ArgumentType.String, Strings.Jottacloud.DescriptionMountPointShort, Strings.Jottacloud.DescriptionMountPointLong(JFS_DEVICE_OPTION)), - new CommandLineArgument(JFS_THREADS, CommandLineArgument.ArgumentType.Integer, Strings.Jottacloud.ThreadsShort, Strings.Jottacloud.ThreadsLong, JFS_DEFAULT_THREADS), - new CommandLineArgument(JFS_CHUNKSIZE, CommandLineArgument.ArgumentType.Size, Strings.Jottacloud.ChunksizeShort, Strings.Jottacloud.ChunksizeLong, JFS_DEFAULT_CHUNKSIZE), - ]); - } + using (var resp = (HttpWebResponse)areq.GetResponse()) + { } + }); } - public string Description - { - get { return Strings.Jottacloud.Description; } - } + public IList SupportedCommands => [ + .. AuthIdOptionsHelper.GetOptions(TOKEN_URL), + new CommandLineArgument(JFS_DEVICE_OPTION, CommandLineArgument.ArgumentType.String, Strings.Jottacloud.DescriptionDeviceShort, Strings.Jottacloud.DescriptionDeviceLong(JFS_MOUNT_POINT_OPTION)), + new CommandLineArgument(JFS_MOUNT_POINT_OPTION, CommandLineArgument.ArgumentType.String, Strings.Jottacloud.DescriptionMountPointShort, Strings.Jottacloud.DescriptionMountPointLong(JFS_DEVICE_OPTION)), + new CommandLineArgument(JFS_THREADS, CommandLineArgument.ArgumentType.Integer, Strings.Jottacloud.ThreadsShort, Strings.Jottacloud.ThreadsLong, JFS_DEFAULT_THREADS), + new CommandLineArgument(JFS_CHUNKSIZE, CommandLineArgument.ArgumentType.Size, Strings.Jottacloud.ChunksizeShort, Strings.Jottacloud.ChunksizeLong, JFS_DEFAULT_CHUNKSIZE), + .. TimeoutOptionsHelper.GetOptions(), + ]; + + public string Description => Strings.Jottacloud.Description; public Task TestAsync(CancellationToken cancelToken) => this.TestListAsync(cancelToken); - public Task CreateFolderAsync(CancellationToken cancelToken) + public async Task CreateFolderAsync(CancellationToken cancelToken) { // When using custom (backup) device we must create the device first (if not already exists). if (!m_device_builtin) { - var req = CreateRequest(System.Net.WebRequestMethods.Http.Post, m_url_device, "type=WORKSTATION"); // Hard-coding device type. Must be one of "WORKSTATION", "LAPTOP", "IMAC", "MACBOOK", "IPAD", "ANDROID", "IPHONE" or "WINDOWS_PHONE". - var areq = new Utility.AsyncHttpRequest(req); - using (var resp = (System.Net.HttpWebResponse)areq.GetResponse()) - { } + var req = CreateRequest(WebRequestMethods.Http.Post, m_url_device, "type=WORKSTATION"); // Hard-coding device type. Must be one of "WORKSTATION", "LAPTOP", "IMAC", "MACBOOK", "IPAD", "ANDROID", "IPHONE" or "WINDOWS_PHONE". + var areq = new AsyncHttpRequest(req); + await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => + { + using (var resp = (HttpWebResponse)areq.GetResponse()) + { } + }).ConfigureAwait(false); } // Create the folder path, and if using custom mount point it will be created as well in the same operation. { - var req = CreateRequest(System.Net.WebRequestMethods.Http.Post, "", "mkDir=true", false); - var areq = new Utility.AsyncHttpRequest(req); - using (var resp = (System.Net.HttpWebResponse)areq.GetResponse()) - { } + var req = CreateRequest(WebRequestMethods.Http.Post, "", "mkDir=true", false); + var areq = new AsyncHttpRequest(req); + await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => + { + using (var resp = (HttpWebResponse)areq.GetResponse()) + { } + }).ConfigureAwait(false); } - return Task.CompletedTask; } #endregion @@ -368,44 +390,49 @@ namespace Duplicati.Library.Backend #endregion - private System.Net.HttpWebRequest CreateRequest(string method, string url, string queryparams) + private HttpWebRequest CreateRequest(string method, string url, string queryparams) { return m_oauth.CreateRequest(url + (string.IsNullOrEmpty(queryparams) || queryparams.Trim().Length == 0 ? "" : "?" + queryparams), method); } - private System.Net.HttpWebRequest CreateRequest(string method, string remotename, string queryparams, bool upload) + private HttpWebRequest CreateRequest(string method, string remotename, string queryparams, bool upload) { var url = (upload ? m_url_upload : m_url) + Library.Utility.Uri.UrlEncode(remotename).Replace("+", "%20"); return CreateRequest(method, url, queryparams); } public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new string[] { - new Uri(JFS_ROOT).Host, - new Uri(JFS_ROOT_UPLOAD).Host + new System.Uri(JFS_ROOT).Host, + new System.Uri(JFS_ROOT_UPLOAD).Host }); - public async Task GetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) + public async Task GetAsync(string remotename, Stream stream, CancellationToken cancelToken) { if (m_threads > 1) { - await ParallelGetAsync(remotename, stream, cancelToken).ConfigureAwait(false); + using (var timeoutStream = stream.ObserveWriteTimeout(m_timeouts.ReadWriteTimeout, false)) + await ParallelGetAsync(remotename, stream, cancelToken).ConfigureAwait(false); return; } // Downloading from Jottacloud: Will only succeed if the file has a completed revision, // and if there are multiple versions of the file we will only get the latest completed version, // ignoring any incomplete or corrupt versions. - var req = CreateRequest(System.Net.WebRequestMethods.Http.Get, remotename, "mode=bin", false); - var areq = new Utility.AsyncHttpRequest(req); - using (var s = areq.GetResponseStream()) - await Utility.Utility.CopyStreamAsync(s, stream, true, cancelToken).ConfigureAwait(false); + var req = CreateRequest(WebRequestMethods.Http.Get, remotename, "mode=bin", false); + var areq = new AsyncHttpRequest(req); + using (var s = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => areq.GetResponseStream()).ConfigureAwait(false)) + using (var t = s.ObserveReadTimeout(m_timeouts.ReadWriteTimeout)) + await Utility.Utility.CopyStreamAsync(t, stream, true, cancelToken).ConfigureAwait(false); } /// /// Fetches the file in chunks (parallelized) /// - public async Task ParallelGetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) + public async Task ParallelGetAsync(string remotename, Stream stream, CancellationToken cancelToken) { - var size = Info(remotename).Size; + var info = await Info(remotename, cancelToken); + if (info == null) + throw new FileMissingException(remotename); + var size = info.Size; var chunks = new Queue>(); // Tuple => Position (from), Position (to) @@ -428,11 +455,11 @@ namespace Duplicati.Library.Backend var item = chunks.Dequeue(); tasks.Enqueue(Task.Run(() => { - var req = CreateRequest(System.Net.WebRequestMethods.Http.Get, remotename, "mode=bin", false); + var req = CreateRequest(WebRequestMethods.Http.Get, remotename, "mode=bin", false); req.AddRange(item.Item1, item.Item2 - 1); - var areq = new Utility.AsyncHttpRequest(req); + var areq = new AsyncHttpRequest(req); using (var s = areq.GetResponseStream()) - using (var reader = new System.IO.BinaryReader(s)) + using (var reader = new BinaryReader(s)) { var length = item.Item2 - item.Item1; return reader.ReadBytes((int)length); @@ -444,7 +471,7 @@ namespace Duplicati.Library.Backend } } - public async Task PutAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) + public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken) { // Some challenges with uploading to Jottacloud: // - Jottacloud supports use of a custom header where we can tell the server the MD5 hash of the file @@ -461,32 +488,34 @@ namespace Duplicati.Library.Backend // request the server to remove the file again and throw an exception. But there is a requirement that // we specify the file size in a custom header. And if the stream is not seek-able we are not able // to use stream.Length, so we are back at square one. - Duplicati.Library.Utility.TempFile tmpFile = null; - var baseStream = stream; - while (baseStream is Duplicati.Library.Utility.OverrideableStream) - baseStream = typeof(Duplicati.Library.Utility.OverrideableStream).GetField("m_basestream", System.Reflection.BindingFlags.DeclaredOnly | System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic).GetValue(baseStream) as System.IO.Stream; + + // TODO: This unwrapping code is shared with CloudFiles + TempFile? tmpFile = null; + Stream? baseStream = stream; + while (baseStream is OverrideableStream) + baseStream = typeof(OverrideableStream).GetField("m_basestream", System.Reflection.BindingFlags.DeclaredOnly | System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic)?.GetValue(baseStream) as System.IO.Stream; if (baseStream == null) throw new Exception(string.Format("Unable to unwrap stream from: {0}", stream.GetType())); string md5Hash; if (baseStream.CanSeek) { var originalPosition = baseStream.Position; - using (var md5 = System.Security.Cryptography.MD5.Create()) - md5Hash = Library.Utility.Utility.ByteArrayAsHexString(md5.ComputeHash(baseStream)); + using (var md5 = MD5.Create()) + md5Hash = Utility.Utility.ByteArrayAsHexString(md5.ComputeHash(baseStream)); baseStream.Position = originalPosition; } else { // No seeking possible, use a temp file - tmpFile = new Duplicati.Library.Utility.TempFile(); - using (var os = System.IO.File.OpenWrite(tmpFile)) + tmpFile = new TempFile(); + using (var os = File.OpenWrite(tmpFile)) using (var hasher = MD5.Create()) - using (var md5 = new Utility.HashCalculatingStream(baseStream, hasher)) + using (var md5 = new HashCalculatingStream(baseStream, hasher)) { await Utility.Utility.CopyStreamAsync(md5, os, true, cancelToken).ConfigureAwait(false); md5Hash = md5.GetFinalHashString(); } - stream = System.IO.File.OpenRead(tmpFile); + stream = File.OpenRead(tmpFile); } try { @@ -497,7 +526,7 @@ namespace Duplicati.Library.Backend // HTTP 404 (Not Found) if file does not exists or it exists with a different hash, in which // case we must send a new request to upload the new content. var fileSize = stream.Length; - var req = CreateRequest(System.Net.WebRequestMethods.Http.Post, remotename, "umode=nomultipart", true); + var req = CreateRequest(WebRequestMethods.Http.Post, remotename, "umode=nomultipart", true); req.Headers.Add("JMd5", md5Hash); // Not required, but it will make the server verify the content and mark the file as corrupt if there is a mismatch. req.Headers.Add("JSize", fileSize.ToString()); // Required, and used to mark file as incomplete if we upload something be the total size of the original file! // File time stamp headers: Since we are working with a stream here we do not know the local file's timestamps, @@ -508,14 +537,15 @@ namespace Duplicati.Library.Backend req.ContentLength = fileSize; // Write post data request - var areq = new Utility.AsyncHttpRequest(req); + var areq = new AsyncHttpRequest(req); using (var rs = areq.GetRequestStream()) + using (var ts = rs.ObserveWriteTimeout(m_timeouts.ReadWriteTimeout)) await Utility.Utility.CopyStreamAsync(stream, rs, true, cancelToken).ConfigureAwait(false); // Send request, and check response - using (var resp = (System.Net.HttpWebResponse)areq.GetResponse()) + using (var resp = (HttpWebResponse)areq.GetResponse()) { - if (resp.StatusCode != System.Net.HttpStatusCode.Created) - throw new System.Net.WebException(Strings.Jottacloud.FileUploadError, null, System.Net.WebExceptionStatus.ProtocolError, resp); + if (resp.StatusCode != HttpStatusCode.Created) + throw new WebException(Strings.Jottacloud.FileUploadError, null, WebExceptionStatus.ProtocolError, resp); // Request seems to be successful, but we must verify the response XML content to be sure that the file // was correctly uploaded: The server will verify the JSize header and mark the file as incomplete if @@ -533,7 +563,7 @@ namespace Duplicati.Library.Backend try { doc.Load(rs); } catch (System.Xml.XmlException) { - throw new System.Net.WebException(Strings.Jottacloud.FileUploadError, System.Net.WebExceptionStatus.ProtocolError); + throw new WebException(Strings.Jottacloud.FileUploadError, WebExceptionStatus.ProtocolError); } bool uploadCompletedSuccessfully = false; var xFile = doc["file"]; @@ -548,7 +578,7 @@ namespace Duplicati.Library.Backend } } if (!uploadCompletedSuccessfully) // Report error (and we just let the incomplete/corrupt file revision stay on the server..) - throw new System.Net.WebException(Strings.Jottacloud.FileUploadError, System.Net.WebExceptionStatus.ProtocolError); + throw new WebException(Strings.Jottacloud.FileUploadError, WebExceptionStatus.ProtocolError); } } } diff --git a/Duplicati/Library/Backend/Jottacloud/Strings.cs b/Duplicati/Library/Backend/Jottacloud/Strings.cs index 876f8d0f0..163519c79 100644 --- a/Duplicati/Library/Backend/Jottacloud/Strings.cs +++ b/Duplicati/Library/Backend/Jottacloud/Strings.cs @@ -19,13 +19,12 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Localization.Short; -namespace Duplicati.Library.Backend.Strings { +namespace Duplicati.Library.Backend.Strings +{ internal static class Jottacloud { public static string Description { get { return LC.L(@"This backend can read and write data to Jottacloud using its REST protocol. Allowed format is ""jottacloud://folder/subfolder""."); } } public static string DisplayName { get { return LC.L(@"Jottacloud"); } } - public static string AuthidLong(string url) { return LC.L(@"The authorization token retrieved from {0}", url); } - public static string AuthidShort { get { return LC.L(@"The authorization code"); } } public static string NoUsernameError { get { return LC.L(@"No username found"); } } public static string NoPathError { get { return LC.L(@"No path given. Files cannot be uploaded to the root folder"); } } public static string IllegalMountPoint { get { return LC.L(@"Illegal mount point given."); } } diff --git a/Duplicati/Library/Backend/S3/S3AwsClient.cs b/Duplicati/Library/Backend/S3/S3AwsClient.cs index 9b7aa21ff..b044c732d 100644 --- a/Duplicati/Library/Backend/S3/S3AwsClient.cs +++ b/Duplicati/Library/Backend/S3/S3AwsClient.cs @@ -151,7 +151,7 @@ namespace Duplicati.Library.Backend public virtual async Task AddFileStreamAsync(string bucketName, string keyName, Stream source, CancellationToken cancelToken) { - using var ts = source.ObserveReadTimeout(m_timeouts.ReadWriteTimeout); + using var ts = source.ObserveReadTimeout(m_timeouts.ReadWriteTimeout, false); var objectAddRequest = new PutObjectRequest { BucketName = bucketName, diff --git a/Duplicati/Library/Backend/S3/S3Backend.cs b/Duplicati/Library/Backend/S3/S3Backend.cs index 6db810f3c..1b452ec4d 100644 --- a/Duplicati/Library/Backend/S3/S3Backend.cs +++ b/Duplicati/Library/Backend/S3/S3Backend.cs @@ -199,7 +199,7 @@ namespace Duplicati.Library.Backend if (!auth.HasUsername) throw new UserInformationException(Strings.S3Backend.NoAMZUserIDError, "S3NoAmzUserID"); - if (auth.HasPassword) + if (!auth.HasPassword) throw new UserInformationException(Strings.S3Backend.NoAMZKeyError, "S3NoAmzKey"); var useSSL = Utility.Utility.ParseBoolOption(options, SSL_OPTION); diff --git a/Duplicati/Library/Backend/S3/S3MinioClient.cs b/Duplicati/Library/Backend/S3/S3MinioClient.cs index 3c1fbb209..086704209 100644 --- a/Duplicati/Library/Backend/S3/S3MinioClient.cs +++ b/Duplicati/Library/Backend/S3/S3MinioClient.cs @@ -204,7 +204,7 @@ namespace Duplicati.Library.Backend try { - using var t = source.ObserveReadTimeout(m_timeouts.ReadWriteTimeout); + using var t = source.ObserveReadTimeout(m_timeouts.ReadWriteTimeout, false); await m_client.PutObjectAsync(bucketName, keyName, t, From 30d446f059060f6caaa88161a037aa12fceb4156 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 21 Mar 2025 14:31:13 +0100 Subject: [PATCH 21/48] Updated Mega backend --- .../Duplicati.Library.Backend.Mega.csproj | 2 + Duplicati/Library/Backend/Mega/MegaBackend.cs | 169 ++++++++---------- 2 files changed, 81 insertions(+), 90 deletions(-) diff --git a/Duplicati/Library/Backend/Mega/Duplicati.Library.Backend.Mega.csproj b/Duplicati/Library/Backend/Mega/Duplicati.Library.Backend.Mega.csproj index cf3460a26..c74f20ae6 100644 --- a/Duplicati/Library/Backend/Mega/Duplicati.Library.Backend.Mega.csproj +++ b/Duplicati/Library/Backend/Mega/Duplicati.Library.Backend.Mega.csproj @@ -4,6 +4,8 @@ net8.0 Library Copyright © 2025 Team Duplicati, MIT license + enable + enable diff --git a/Duplicati/Library/Backend/Mega/MegaBackend.cs b/Duplicati/Library/Backend/Mega/MegaBackend.cs index 573133f0d..4431c7ace 100644 --- a/Duplicati/Library/Backend/Mega/MegaBackend.cs +++ b/Duplicati/Library/Backend/Mega/MegaBackend.cs @@ -22,11 +22,8 @@ using CG.Web.MegaApiClient; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; +using Duplicati.Library.Utility; +using Duplicati.Library.Utility.Options; using OtpNet; using System.Runtime.CompilerServices; @@ -36,68 +33,67 @@ namespace Duplicati.Library.Backend.Mega // This class is instantiated dynamically in the BackendLoader. public class MegaBackend : IBackend, IStreamingBackend { - private readonly string m_username = null; - private readonly string m_password = null; - private readonly string m_twoFactorKey = null; - private Dictionary> m_filecache; - private INode m_currentFolder = null; - private readonly string m_prefix = null; + private readonly string m_username; + private readonly string m_password; + private readonly string? m_twoFactorKey; + private Dictionary>? m_filecache; + private INode? m_currentFolder = null; + private readonly string m_prefix; + private readonly TimeoutOptionsHelper.Timeouts m_timeouts; - private MegaApiClient m_client; + private MegaApiClient? m_client; public MegaBackend() { + m_username = null!; + m_password = null!; + m_twoFactorKey = null; + m_prefix = null!; + m_timeouts = null!; } - private MegaApiClient Client + private async Task GetClient(CancellationToken cancelToken) { - get - { - if (m_client == null) + if (m_client == null) + m_client = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, async ct => { var cl = new MegaApiClient(); if (m_twoFactorKey == null) - cl.Login(m_username, m_password); + await cl.LoginAsync(m_username, m_password).ConfigureAwait(false); else { var totp = new Totp(Base32Encoding.ToBytes(m_twoFactorKey)).ComputeTotp(); - cl.Login(m_username, m_password, totp); + await cl.LoginAsync(m_username, m_password, totp).ConfigureAwait(false); } - m_client = cl; - } + return cl; + }).ConfigureAwait(false); - return m_client; - } + return m_client; } - public MegaBackend(string url, Dictionary options) + public MegaBackend(string url, Dictionary options) { var uri = new Utility.Uri(url); - if (options.ContainsKey("auth-username")) - m_username = options["auth-username"]; - if (options.ContainsKey("auth-password")) - m_password = options["auth-password"]; + var auth = AuthOptionsHelper.Parse(options, uri); if (options.ContainsKey("auth-two-factor-key")) m_twoFactorKey = options["auth-two-factor-key"]; - if (!string.IsNullOrEmpty(uri.Username)) - m_username = uri.Username; - if (!string.IsNullOrEmpty(uri.Password)) - m_password = uri.Password; - if (string.IsNullOrEmpty(m_username)) throw new UserInformationException(Strings.MegaBackend.NoUsernameError, "MegaNoUsername"); if (string.IsNullOrEmpty(m_password)) throw new UserInformationException(Strings.MegaBackend.NoPasswordError, "MegaNoPassword"); + (m_username, m_password) = auth.GetCredentials(); m_prefix = uri.HostAndPath ?? ""; + m_timeouts = TimeoutOptionsHelper.Parse(options); } - private async Task FetchCurrentFolderAsync(bool autocreate, CancellationToken cancelToken) + private async Task FetchCurrentFolderAsync(bool autocreate, CancellationToken cancelToken) { + var client = await GetClient(cancelToken).ConfigureAwait(false); var parts = m_prefix.Split(new string[] { "/" }, StringSplitOptions.RemoveEmptyEntries); - var nodes = Client.GetNodes(); + var nodes = await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, _ => client.GetNodes()).ConfigureAwait(false); INode parent = nodes.First(x => x.Type == NodeType.Root); foreach (var n in parts) @@ -108,7 +104,7 @@ namespace Duplicati.Library.Backend.Mega if (!autocreate) throw new FolderMissingException(); - item = await Client.CreateFolderAsync(n, parent).ConfigureAwait(false); + item = await client.CreateFolderAsync(n, parent).ConfigureAwait(false); } parent = item; @@ -117,14 +113,17 @@ namespace Duplicati.Library.Backend.Mega m_currentFolder = parent; await ResetFileCacheAsync(nodes, cancelToken).ConfigureAwait(false); + + return m_currentFolder; } private async Task GetCurrentFolderAsync(CancellationToken cancelToken) { - if (m_currentFolder == null) - await FetchCurrentFolderAsync(false, cancelToken).ConfigureAwait(false); + var folder = m_currentFolder; + if (folder == null) + folder = await FetchCurrentFolderAsync(false, cancelToken).ConfigureAwait(false); - return m_currentFolder; + return folder; } private async Task GetFileNodeAsync(string name, CancellationToken cancelToken) @@ -140,17 +139,23 @@ namespace Duplicati.Library.Backend.Mega throw new FileMissingException(); } - private async Task ResetFileCacheAsync(IEnumerable list, CancellationToken cancelToken) + private async Task>> ResetFileCacheAsync(IEnumerable? list, CancellationToken cancelToken) { if (m_currentFolder == null) { await FetchCurrentFolderAsync(false, cancelToken).ConfigureAwait(false); + // We build it as part of the fetch step + return m_filecache!; } else { + var client = await GetClient(cancelToken).ConfigureAwait(false); + var currentFolder = await GetCurrentFolderAsync(cancelToken).ConfigureAwait(false); - m_filecache = - (list ?? Client.GetNodes()).Where(x => x.Type == NodeType.File && x.ParentId == currentFolder.Id) + if (list == null) + list = await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, _ => client.GetNodes()).ConfigureAwait(false); + return m_filecache = + list.Where(x => x.Type == NodeType.File && x.ParentId == currentFolder.Id) .GroupBy(x => x.Name, x => x, (k, g) => new KeyValuePair>(k, g.ToList())) .ToDictionary(x => x.Key, x => x.Value); } @@ -158,19 +163,22 @@ namespace Duplicati.Library.Backend.Mega #region IStreamingBackend implementation - public async Task PutAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) + public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken) { try { - if (m_filecache == null) - await ResetFileCacheAsync(null, cancelToken).ConfigureAwait(false); + var filecache = m_filecache; + if (filecache == null) + filecache = await ResetFileCacheAsync(null, cancelToken).ConfigureAwait(false); + var client = await GetClient(cancelToken).ConfigureAwait(false); var currentFolder = await GetCurrentFolderAsync(cancelToken).ConfigureAwait(false); - var el = await Client.UploadAsync(stream, remotename, currentFolder, new Progress(), null, cancelToken).ConfigureAwait(false); - if (m_filecache.ContainsKey(remotename)) + using var ts = stream.ObserveReadTimeout(m_timeouts.ReadWriteTimeout, false); + var el = await client.UploadAsync(ts, remotename, currentFolder, new Progress(), null, cancelToken).ConfigureAwait(false); + if (filecache.ContainsKey(remotename)) await DeleteAsync(remotename, cancelToken).ConfigureAwait(false); - m_filecache[remotename] = [el]; + filecache[remotename] = [el]; } catch { @@ -181,9 +189,11 @@ namespace Duplicati.Library.Backend.Mega public async Task GetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) { + var client = await GetClient(cancelToken).ConfigureAwait(false); var node = await GetFileNodeAsync(remotename, cancelToken).ConfigureAwait(false); - using (var s = Client.Download(node)) - await Library.Utility.Utility.CopyStreamAsync(s, stream, cancelToken).ConfigureAwait(false); + using (var s = await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, _ => client.Download(node)).ConfigureAwait(false)) + using (var t = s.ObserveReadTimeout(m_timeouts.ReadWriteTimeout)) + await Utility.Utility.CopyStreamAsync(t, stream, cancelToken).ConfigureAwait(false); } #endregion @@ -193,10 +203,11 @@ namespace Duplicati.Library.Backend.Mega /// public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { - if (m_filecache == null) - await ResetFileCacheAsync(null, cancelToken).ConfigureAwait(false); + var filecache = m_filecache; + if (filecache == null) + filecache = await ResetFileCacheAsync(null, cancelToken).ConfigureAwait(false); - foreach (var n in m_filecache.Values) + foreach (var n in filecache.Values) { var item = n.OrderByDescending(x => x.ModificationDate).First(); yield return new FileEntry(item.Name, item.Size, item.ModificationDate ?? new DateTime(0), item.ModificationDate ?? new DateTime(0)); @@ -219,16 +230,18 @@ namespace Duplicati.Library.Backend.Mega { try { - if (m_filecache == null || !m_filecache.ContainsKey(remotename)) - await ResetFileCacheAsync(null, cancelToken).ConfigureAwait(false); + var filecache = m_filecache; + if (filecache == null || !filecache.ContainsKey(remotename)) + filecache = await ResetFileCacheAsync(null, cancelToken).ConfigureAwait(false); - if (!m_filecache.ContainsKey(remotename)) + if (!filecache.ContainsKey(remotename)) throw new FileMissingException(); - foreach (var n in m_filecache[remotename]) - await Client.DeleteAsync(n, false).ConfigureAwait(false); + var client = await GetClient(cancelToken).ConfigureAwait(false); + foreach (var n in filecache[remotename]) + await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, _ => client.DeleteAsync(n, false)).ConfigureAwait(false); - m_filecache.Remove(remotename); + filecache.Remove(remotename); } catch { @@ -245,41 +258,17 @@ namespace Duplicati.Library.Backend.Mega return FetchCurrentFolderAsync(true, cancelToken); } - public string DisplayName - { - get - { - return Strings.MegaBackend.DisplayName; - } - } + public string DisplayName => Strings.MegaBackend.DisplayName; - public string ProtocolKey - { - get - { - return "mega"; - } - } + public string ProtocolKey => "mega"; - public IList SupportedCommands - { - get - { - return new List(new ICommandLineArgument[] { - new CommandLineArgument("auth-password", CommandLineArgument.ArgumentType.Password, Strings.MegaBackend.AuthPasswordDescriptionShort, Strings.MegaBackend.AuthPasswordDescriptionLong), - new CommandLineArgument("auth-username", CommandLineArgument.ArgumentType.String, Strings.MegaBackend.AuthUsernameDescriptionShort, Strings.MegaBackend.AuthUsernameDescriptionLong), - new CommandLineArgument("auth-two-factor-key", CommandLineArgument.ArgumentType.Password, Strings.MegaBackend.AuthTwoFactorKeyDescriptionShort, Strings.MegaBackend.AuthTwoFactorKeyDescriptionLong), - }); - } - } + public IList SupportedCommands => [ + .. AuthOptionsHelper.GetOptions(), + new CommandLineArgument("auth-two-factor-key", CommandLineArgument.ArgumentType.Password, Strings.MegaBackend.AuthTwoFactorKeyDescriptionShort, Strings.MegaBackend.AuthTwoFactorKeyDescriptionLong), + .. TimeoutOptionsHelper.GetOptions() + ]; - public string Description - { - get - { - return Strings.MegaBackend.Description; - } - } + public string Description => Strings.MegaBackend.Description; public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(Array.Empty()); From b27acf52d57e1f4fc302e24e95ef5735ebd041ef Mon Sep 17 00:00:00 2001 From: "Marcelo C." Date: Fri, 21 Mar 2025 10:43:51 -0300 Subject: [PATCH 22/48] Removing custom boundary, leaving for framework to add its own value --- Duplicati/Library/Backend/Box/BoxBackend.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Backend/Box/BoxBackend.cs b/Duplicati/Library/Backend/Box/BoxBackend.cs index 4e49abcc7..7ed4d77a7 100644 --- a/Duplicati/Library/Backend/Box/BoxBackend.cs +++ b/Duplicati/Library/Backend/Box/BoxBackend.cs @@ -204,7 +204,7 @@ namespace Duplicati.Library.Backend.Box var existing = _fileCache.ContainsKey(remotename); - var multipartForm = new MultipartFormDataContent("----DuplicatiFormBoundary" + Guid.NewGuid().ToString("N")); + var multipartForm = new MultipartFormDataContent(); try { From 5c1e17b0c291689e6ced0981ff9f97bef041c802 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 21 Mar 2025 15:58:56 +0100 Subject: [PATCH 23/48] Updated OneDrive and other MsGraph based backends --- .../Duplicati.Library.Backend.OneDrive.csproj | 2 + .../Library/Backend/OneDrive/Exceptions.cs | 151 ++++----- .../Backend/OneDrive/MicrosoftGraphBackend.cs | 286 +++++++----------- .../Backend/OneDrive/MicrosoftGraphTypes.cs | 95 +++--- .../Backend/OneDrive/MicrosoftGroup.cs | 117 +++---- .../Library/Backend/OneDrive/OneDrive.cs | 60 ++-- .../Library/Backend/OneDrive/SharePointV2.cs | 97 +++--- Duplicati/Library/Backend/OneDrive/Strings.cs | 43 ++- 8 files changed, 336 insertions(+), 515 deletions(-) diff --git a/Duplicati/Library/Backend/OneDrive/Duplicati.Library.Backend.OneDrive.csproj b/Duplicati/Library/Backend/OneDrive/Duplicati.Library.Backend.OneDrive.csproj index 8aeed71cd..515043adb 100644 --- a/Duplicati/Library/Backend/OneDrive/Duplicati.Library.Backend.OneDrive.csproj +++ b/Duplicati/Library/Backend/OneDrive/Duplicati.Library.Backend.OneDrive.csproj @@ -5,6 +5,8 @@ Library Duplicati.Library.Backend Copyright © 2025 Team Duplicati, MIT license + enable + enable diff --git a/Duplicati/Library/Backend/OneDrive/Exceptions.cs b/Duplicati/Library/Backend/OneDrive/Exceptions.cs index d30f190b2..bb542ab94 100644 --- a/Duplicati/Library/Backend/OneDrive/Exceptions.cs +++ b/Duplicati/Library/Backend/OneDrive/Exceptions.cs @@ -1,30 +1,25 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. -using System; -using System.IO; +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. using System.Net; -using System.Net.Http; -using System.Net.Http.Headers; using System.Text.RegularExpressions; -using System.Threading.Tasks; using Duplicati.Library.Utility; @@ -36,11 +31,11 @@ namespace Duplicati.Library.Backend.MicrosoftGraph { private static readonly Regex authorizationHeaderRemover = new Regex(@"Authorization:\s*Bearer\s+\S+", RegexOptions.IgnoreCase); - private readonly HttpResponseMessage responseMessage; - private readonly HttpWebResponse webResponse; + private readonly HttpResponseMessage? responseMessage; + private readonly HttpWebResponse? webResponse; public MicrosoftGraphException(HttpResponseMessage response) - : this(string.Format("{0}: {1} error from request {2}", response.StatusCode, response.ReasonPhrase, response.RequestMessage.RequestUri), response) + : this(string.Format("{0}: {1} error from request {2}", response.StatusCode, response.ReasonPhrase, response.RequestMessage?.RequestUri), response) { } @@ -49,7 +44,7 @@ namespace Duplicati.Library.Backend.MicrosoftGraph { } - public MicrosoftGraphException(string message, HttpResponseMessage response, Exception innerException) + public MicrosoftGraphException(string message, HttpResponseMessage response, Exception? innerException) : base(BuildFullMessage(message, response), innerException) { this.responseMessage = response; @@ -65,59 +60,40 @@ namespace Duplicati.Library.Backend.MicrosoftGraph { } - public MicrosoftGraphException(string message, HttpWebResponse response, Exception innerException) + public MicrosoftGraphException(string message, HttpWebResponse response, Exception? innerException) : base(BuildFullMessage(message, response), innerException) { this.webResponse = response; } - public string RequestUrl - { - get - { - if (this.responseMessage != null) - { - return this.responseMessage.RequestMessage.RequestUri.ToString(); - } - else - { - return this.webResponse.ResponseUri.ToString(); - } - } - } - public HttpStatusCode StatusCode { get { - if (this.responseMessage != null) - { - return this.responseMessage.StatusCode; - } - else - { - return this.webResponse.StatusCode; - } + if (responseMessage != null) + return responseMessage.StatusCode; + if (webResponse != null) + return webResponse.StatusCode; + return HttpStatusCode.InternalServerError; } } - protected static string ResponseToString(HttpResponseMessage response) + protected static string? ResponseToString(HttpResponseMessage response) { - if (response != null) - { - // Start to read the content - using (Task content = response.Content.ReadAsStringAsync()) - { - // Since the exception message may be saved / sent in logs, we want to prevent the authorization header from being included. - // it wouldn't be as bad as recording the username/password in logs, since the token will expire, but it doesn't hurt to be safe. - // So we replace anything in the request that looks like the auth header with a safe version. - string requestMessage = authorizationHeaderRemover.Replace(response.RequestMessage.ToString(), "Authorization: Bearer ABC...XYZ"); - return string.Format("{0}\n{1}\n{2}", requestMessage, response, PrettifyJson(content.Await())); - } - } - else - { + if (response == null) return null; + + // Start to read the content + using (Task content = response.Content.ReadAsStringAsync()) + { + // Since the exception message may be saved / sent in logs, we want to prevent the authorization header from being included. + // it wouldn't be as bad as recording the username/password in logs, since the token will expire, but it doesn't hurt to be safe. + // So we replace anything in the request that looks like the auth header with a safe version. + var requestMessage = response.RequestMessage == null + ? "" + : authorizationHeaderRemover.Replace(response.RequestMessage.ToString(), "Authorization: Bearer ABC...XYZ"); + + return string.Format("{0}\n{1}\n{2}", requestMessage, response, PrettifyJson(content.Await())); } } @@ -133,41 +109,28 @@ namespace Duplicati.Library.Backend.MicrosoftGraph } } - protected static string ResponseToString(HttpWebResponse response) + protected static string? ResponseToString(HttpWebResponse response) { - if (response != null) - { - // Start to read the content - using (var responseStream = response.GetResponseStream()) - using (var textReader = new StreamReader(responseStream)) - { - return string.Format("{0}\n{1}", response, PrettifyJson(textReader.ReadToEnd())); - } - } - else - { + if (response == null) return null; - } + + // Start to read the content + using (var responseStream = response.GetResponseStream()) + using (var textReader = new StreamReader(responseStream)) + return string.Format("{0}\n{1}", response, PrettifyJson(textReader.ReadToEnd())); } private static string BuildFullMessage(string message, HttpWebResponse response) { - if (response != null) - { - return string.Format("{0}\n{1}", message, ResponseToString(response)); - } - else - { + if (response == null) return message; - } + return string.Format("{0}\n{1}", message, ResponseToString(response)); } private static string PrettifyJson(string json) { if (string.IsNullOrWhiteSpace(json)) - { return json; - } if (json[0] == '<') { @@ -191,12 +154,12 @@ namespace Duplicati.Library.Backend.MicrosoftGraph public class DriveItemNotFoundException : MicrosoftGraphException { public DriveItemNotFoundException(HttpResponseMessage response) - : base(string.Format("Item at {0} was not found", response?.RequestMessage?.RequestUri?.ToString() ?? ""), response) + : base(string.Format("Item at {0} was not found", response.RequestMessage?.RequestUri?.ToString() ?? ""), response) { } public DriveItemNotFoundException(HttpWebResponse response) - : base(string.Format("Item at {0} was not found", response?.ResponseUri?.ToString() ?? ""), response) + : base(string.Format("Item at {0} was not found", response.ResponseUri?.ToString() ?? ""), response) { } } @@ -209,7 +172,7 @@ namespace Duplicati.Library.Backend.MicrosoftGraph int fragmentCount, Exception fragmentException) : base( - string.Format("Error uploading fragment {0} of {1} for {2}", fragment, fragmentCount, originalResponse?.RequestMessage?.RequestUri?.ToString() ?? ""), + string.Format("Error uploading fragment {0} of {1} for {2}", fragment, fragmentCount, originalResponse.RequestMessage?.RequestUri?.ToString() ?? ""), originalResponse, fragmentException) { @@ -223,7 +186,7 @@ namespace Duplicati.Library.Backend.MicrosoftGraph int fragmentCount, Exception fragmentException) : base( - string.Format("Error uploading fragment {0} of {1} for {2}", fragment, fragmentCount, originalResponse?.ResponseUri?.ToString() ?? ""), + string.Format("Error uploading fragment {0} of {1} for {2}", fragment, fragmentCount, originalResponse.ResponseUri?.ToString() ?? ""), originalResponse, fragmentException) { diff --git a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs index 5726522cd..7387c0a80 100644 --- a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs +++ b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs @@ -23,18 +23,12 @@ using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using Duplicati.Library.Logging; using Duplicati.Library.Utility; +using Duplicati.Library.Utility.Options; using Newtonsoft.Json; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; using System.Net; -using System.Net.Http; using System.Net.Http.Headers; using System.Runtime.CompilerServices; using System.Text; -using System.Threading; -using System.Threading.Tasks; namespace Duplicati.Library.Backend { @@ -59,7 +53,6 @@ namespace Duplicati.Library.Backend private const string BASE_ADDRESS = "https://graph.microsoft.com"; - private const string AUTHID_OPTION = "authid"; private const string UPLOAD_SESSION_FRAGMENT_SIZE_OPTION = "fragment-size"; private const string UPLOAD_SESSION_FRAGMENT_RETRY_COUNT_OPTION = "fragment-retry-count"; private const string UPLOAD_SESSION_FRAGMENT_RETRY_DELAY_OPTION = "fragment-retry-delay"; @@ -116,23 +109,29 @@ namespace Duplicati.Library.Backend // send a request. And before sending any requests, we'll make sure to wait until at least this time. // Since this may be read and written by multiple threads, it is stored as a long and updated using Interlocked.Exchange. private readonly RetryAfterHelper m_retryAfter; + protected readonly TimeoutOptionsHelper.Timeouts m_timeouts; - private string[] dnsNames = null; + private string[]? dnsNames = null; private readonly Lazy rootPathFromURL; private string RootPath => this.rootPathFromURL.Value; + private string TokenUrl => OAuthHelper.OAUTH_LOGIN_URL(ProtocolKey); - protected MicrosoftGraphBackend() { } // Constructor needed for dynamic loading to find it - - protected MicrosoftGraphBackend(string url, string protocolKey, Dictionary options) + protected MicrosoftGraphBackend() { - string authid; - options.TryGetValue(AUTHID_OPTION, out authid); - if (string.IsNullOrEmpty(authid)) - throw new UserInformationException(Strings.MicrosoftGraph.MissingAuthId(OAuthHelper.OAUTH_LOGIN_URL(protocolKey)), "MicrosoftGraphBackendMissingAuthId"); + // Constructor needed for dynamic loading to find it + m_client = null!; + m_retryAfter = null!; + m_timeouts = null!; + rootPathFromURL = null!; + } - string fragmentSizeStr; - if (options.TryGetValue(UPLOAD_SESSION_FRAGMENT_SIZE_OPTION, out fragmentSizeStr) && int.TryParse(fragmentSizeStr, out this.fragmentSize)) + protected MicrosoftGraphBackend(string url, string protocolKey, Dictionary options) + { + var authId = AuthIdOptionsHelper.Parse(options) + .RequireCredentials(TokenUrl); + + if (options.TryGetValue(UPLOAD_SESSION_FRAGMENT_SIZE_OPTION, out var fragmentSizeStr) && int.TryParse(fragmentSizeStr, out this.fragmentSize)) { // Make sure the fragment size is a multiple of the desired multiple size. // If it isn't, we round down to the nearest multiple below it. @@ -146,22 +145,17 @@ namespace Duplicati.Library.Backend this.fragmentSize = UPLOAD_SESSION_FRAGMENT_DEFAULT_SIZE; } - string fragmentRetryCountStr; - if (!(options.TryGetValue(UPLOAD_SESSION_FRAGMENT_RETRY_COUNT_OPTION, out fragmentRetryCountStr) && int.TryParse(fragmentRetryCountStr, out this.fragmentRetryCount))) - { + if (!(options.TryGetValue(UPLOAD_SESSION_FRAGMENT_RETRY_COUNT_OPTION, out var fragmentRetryCountStr) && int.TryParse(fragmentRetryCountStr, out this.fragmentRetryCount))) this.fragmentRetryCount = UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_COUNT; - } - string fragmentRetryDelayStr; - if (!(options.TryGetValue(UPLOAD_SESSION_FRAGMENT_RETRY_DELAY_OPTION, out fragmentRetryDelayStr) && int.TryParse(fragmentRetryDelayStr, out this.fragmentRetryDelay))) - { + if (!(options.TryGetValue(UPLOAD_SESSION_FRAGMENT_RETRY_DELAY_OPTION, out var fragmentRetryDelayStr) && int.TryParse(fragmentRetryDelayStr, out this.fragmentRetryDelay))) this.fragmentRetryDelay = UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_DELAY; - } - this.m_client = new OAuthHttpClient(authid, protocolKey); + this.m_client = new OAuthHttpClient(authId.AuthId!, protocolKey); this.m_client.BaseAddress = new System.Uri(BASE_ADDRESS); this.m_retryAfter = RetryAfterHelper.CreateOrGetRetryAfterHelper(url); + this.m_timeouts = TimeoutOptionsHelper.Parse(options); // Extract out the path to the backup root folder from the given URI. Since this can be an expensive operation, // we will cache the value using a lazy initializer. @@ -173,78 +167,67 @@ namespace Duplicati.Library.Backend public abstract string DisplayName { get; } - public string Description - { - get - { - return this.DescriptionTemplate( + public string Description => this.DescriptionTemplate( "Microsoft Service Agreement", SERVICES_AGREEMENT, "Microsoft Online Privacy Statement", PRIVACY_STATEMENT); - } - } - public IList SupportedCommands - { - get - { - return new[] - { - new CommandLineArgument(AUTHID_OPTION, CommandLineArgument.ArgumentType.Password, Strings.MicrosoftGraph.AuthIdShort, Strings.MicrosoftGraph.AuthIdLong(OAuthHelper.OAUTH_LOGIN_URL(this.ProtocolKey))), - new CommandLineArgument(UPLOAD_SESSION_FRAGMENT_SIZE_OPTION, CommandLineArgument.ArgumentType.Integer, Strings.MicrosoftGraph.FragmentSizeShort, Strings.MicrosoftGraph.FragmentSizeLong, Library.Utility.Utility.FormatSizeString(UPLOAD_SESSION_FRAGMENT_DEFAULT_SIZE)), - new CommandLineArgument(UPLOAD_SESSION_FRAGMENT_RETRY_COUNT_OPTION, CommandLineArgument.ArgumentType.Integer, Strings.MicrosoftGraph.FragmentRetryCountShort, Strings.MicrosoftGraph.FragmentRetryCountLong, UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_COUNT.ToString()), - new CommandLineArgument(UPLOAD_SESSION_FRAGMENT_RETRY_DELAY_OPTION, CommandLineArgument.ArgumentType.Integer, Strings.MicrosoftGraph.FragmentRetryDelayShort, Strings.MicrosoftGraph.FragmentRetryDelayLong, UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_DELAY.ToString()), - new CommandLineArgument("use-http-client", CommandLineArgument.ArgumentType.Boolean, Strings.MicrosoftGraph.UseHttpClientShort, Strings.MicrosoftGraph.UseHttpClientLong, "true", null, null, Strings.MicrosoftGraph.UseHttpClientDeprecated), - } - .Concat(this.AdditionalSupportedCommands).ToList(); - } - } + public IList SupportedCommands => [ + .. AuthIdOptionsHelper.GetOptions(TokenUrl), + new CommandLineArgument(UPLOAD_SESSION_FRAGMENT_SIZE_OPTION, CommandLineArgument.ArgumentType.Integer, Strings.MicrosoftGraph.FragmentSizeShort, Strings.MicrosoftGraph.FragmentSizeLong, Library.Utility.Utility.FormatSizeString(UPLOAD_SESSION_FRAGMENT_DEFAULT_SIZE)), + new CommandLineArgument(UPLOAD_SESSION_FRAGMENT_RETRY_COUNT_OPTION, CommandLineArgument.ArgumentType.Integer, Strings.MicrosoftGraph.FragmentRetryCountShort, Strings.MicrosoftGraph.FragmentRetryCountLong, UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_COUNT.ToString()), + new CommandLineArgument(UPLOAD_SESSION_FRAGMENT_RETRY_DELAY_OPTION, CommandLineArgument.ArgumentType.Integer, Strings.MicrosoftGraph.FragmentRetryDelayShort, Strings.MicrosoftGraph.FragmentRetryDelayLong, UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_DELAY.ToString()), + new CommandLineArgument("use-http-client", CommandLineArgument.ArgumentType.Boolean, Strings.MicrosoftGraph.UseHttpClientShort, Strings.MicrosoftGraph.UseHttpClientLong, "true", null, null, Strings.MicrosoftGraph.UseHttpClientDeprecated), + .. TimeoutOptionsHelper.GetOptions() + ]; public async Task GetDNSNamesAsync(CancellationToken cancelToken) { - if (this.dnsNames == null) + if (dnsNames == null) { // The DNS names that this instance may need to access include: // - Core graph API endpoint // - Upload session endpoint (which seems to be different depending on the drive being accessed - not sure if it can vary for a single drive) // To get the upload session endpoint, we can start an upload session and then immediately cancel it. // We pick a random file name (using a guid) to make sure we don't conflict with an existing file - string dnsTestFile = string.Format("DNSNameTest-{0}", Guid.NewGuid()); - var uploadSession = await this.PostAsync(string.Format("{0}/root:{1}{2}:/createUploadSession", this.DrivePrefix, this.RootPath, NormalizeSlashes(dnsTestFile)), MicrosoftGraphBackend.dummyUploadSession, cancelToken).ConfigureAwait(false); + var dnsTestFile = string.Format("DNSNameTest-{0}", Guid.NewGuid()); + var drivePrefix = await GetDrivePrefix(cancelToken).ConfigureAwait(false); + var uploadSession = await this.PostAsync(string.Format("{0}/root:{1}{2}:/createUploadSession", drivePrefix, this.RootPath, NormalizeSlashes(dnsTestFile)), dummyUploadSession, cancelToken).ConfigureAwait(false); // Canceling an upload session is done by sending a DELETE to the upload URL await m_retryAfter.WaitForRetryAfterAsync(cancelToken).ConfigureAwait(false); using (var request = new HttpRequestMessage(HttpMethod.Delete, uploadSession.UploadUrl)) - using (var response = await this.m_client.SendAsync(request, false, cancelToken).ConfigureAwait(false)) - { - this.CheckResponse(response); - } + using (var response = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, + ct => m_client.SendAsync(request, false, ct)).ConfigureAwait(false)) + CheckResponse(response); - this.dnsNames = new[] - { - new System.Uri(BASE_ADDRESS).Host, - new System.Uri(uploadSession.UploadUrl).Host, - } + dnsNames = new[] { + new System.Uri(BASE_ADDRESS).Host, + string.IsNullOrWhiteSpace(uploadSession.UploadUrl) ? null : new System.Uri(uploadSession.UploadUrl).Host, + } + .WhereNotNullOrWhiteSpace() .Distinct(StringComparer.OrdinalIgnoreCase) .ToArray(); } - return this.dnsNames; + return dnsNames; } - public async Task GetQuotaInfoAsync(CancellationToken cancelToken) + public async Task GetQuotaInfoAsync(CancellationToken cancelToken) { - Drive driveInfo = await this.GetAsync(this.DrivePrefix, cancelToken).ConfigureAwait(false); - if (driveInfo.Quota != null) + var drivePrefix = await GetDrivePrefix(cancelToken).ConfigureAwait(false); + var driveInfo = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, + ct => GetAsync(drivePrefix, ct) + ).ConfigureAwait(false); + + if (driveInfo?.Quota != null) { // Some sources (SharePoint for example) seem to return 0 for these values even when the quota isn't exceeded.. // As a special test, if all the returned values are 0, we pretend that no quota was reported. // This way we don't send spurious warnings because the quota looks like it is exceeded. if (driveInfo.Quota.Total != 0 || driveInfo.Quota.Remaining != 0 || driveInfo.Quota.Used != 0) - { return new QuotaInfo(driveInfo.Quota.Total, driveInfo.Quota.Remaining); - } } return null; @@ -254,54 +237,36 @@ namespace Duplicati.Library.Backend /// /// Override-able fragment indicating the API version to use each query /// - protected virtual string ApiVersion - { - get { return "/v1.0"; } - } + protected virtual string ApiVersion => "/v1.0"; /// /// Normalized fragment (starting with a slash and ending without one) for the path to the drive to be used. /// For example: "/me/drive" for the default drive for a user. /// - protected abstract string DrivePath { get; } + protected abstract Task GetDrivePath(CancellationToken cancelToken); protected abstract DescriptionTemplateDelegate DescriptionTemplate { get; } - protected virtual IList AdditionalSupportedCommands - { - get - { - return new ICommandLineArgument[0]; - } - } + protected virtual IList AdditionalSupportedCommands => []; - private string DrivePrefix - { - get - { - if (this.m_client != null) - { - return this.ApiVersion + this.DrivePath; - } - else - { - // When not using the HttpClient for requests, the base address needs to be included in this prefix - return BASE_ADDRESS + this.ApiVersion + this.DrivePath; - } - } - } + private async Task GetDrivePrefix(CancellationToken cancelToken) + => $"{ApiVersion}{await GetDrivePath(cancelToken).ConfigureAwait(false)}"; public async Task CreateFolderAsync(CancellationToken cancelToken) { - string parentFolder = "root"; - string parentFolderPath = string.Empty; + var parentFolder = "root"; + var parentFolderPath = string.Empty; + var drivePrefix = await GetDrivePrefix(cancelToken).ConfigureAwait(false); + foreach (string folder in this.RootPath.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries)) { string nextPath = parentFolderPath + "/" + folder; DriveItem folderItem; try { - folderItem = await this.GetAsync(string.Format("{0}/root:{1}", this.DrivePrefix, NormalizeSlashes(nextPath)), cancelToken).ConfigureAwait(false); + folderItem = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, + ct => GetAsync(string.Format("{0}/root:{1}", drivePrefix, NormalizeSlashes(nextPath)), ct) + ).ConfigureAwait(false); } catch (DriveItemNotFoundException) { @@ -311,7 +276,9 @@ namespace Duplicati.Library.Backend Folder = new FolderFacet(), }; - folderItem = await this.PostAsync(string.Format("{0}/items/{1}/children", this.DrivePrefix, parentFolder), newFolder, cancelToken).ConfigureAwait(false); + folderItem = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, + ct => PostAsync(string.Format("{0}/items/{1}/children", drivePrefix, parentFolder), newFolder, ct) + ).ConfigureAwait(false); } parentFolder = folderItem.Id; @@ -321,9 +288,9 @@ namespace Duplicati.Library.Backend /// public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) - { - await foreach (var item in this.Enumerate(string.Format("{0}/root:{1}:/children", this.DrivePrefix, this.RootPath), cancelToken).ConfigureAwait(false)) + var drivePrefix = await GetDrivePrefix(cancelToken).ConfigureAwait(false); + await foreach (var item in this.Enumerate(string.Format("{0}/root:{1}:/children", drivePrefix, this.RootPath), cancelToken).ConfigureAwait(false)) { // Exclude non-files and deleted items (not sure if they show up in this listing, but make sure anyway) if (item.IsFile && !item.IsDeleted) @@ -347,13 +314,18 @@ namespace Duplicati.Library.Backend { try { + var drivePrefix = await GetDrivePrefix(cancelToken).ConfigureAwait(false); m_retryAfter.WaitForRetryAfter(); - string getUrl = string.Format("{0}/root:{1}{2}:/content", this.DrivePrefix, this.RootPath, NormalizeSlashes(remotename)); - using (var response = await this.m_client.GetAsync(getUrl, HttpCompletionOption.ResponseHeadersRead, cancelToken).ConfigureAwait(false)) + string getUrl = string.Format("{0}/root:{1}{2}:/content", drivePrefix, this.RootPath, NormalizeSlashes(remotename)); + using (var response = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, + ct => m_client.GetAsync(getUrl, HttpCompletionOption.ResponseHeadersRead, ct) + ).ConfigureAwait(false)) { - this.CheckResponse(response); - using (var responseStream = await response.Content.ReadAsStreamAsync(cancelToken).ConfigureAwait(false)) - await Library.Utility.Utility.CopyStreamAsync(responseStream, stream, cancelToken).ConfigureAwait(false); + CheckResponse(response); + using (var responseStream = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, + ct => response.Content.ReadAsStreamAsync(ct)).ConfigureAwait(false)) + using (var timeoutSream = responseStream.ObserveReadTimeout(m_timeouts.ReadWriteTimeout)) + await Utility.Utility.CopyStreamAsync(responseStream, stream, cancelToken).ConfigureAwait(false); } } catch (DriveItemNotFoundException ex) @@ -367,7 +339,10 @@ namespace Duplicati.Library.Backend { try { - await this.PatchAsync(string.Format("{0}/root:{1}{2}", this.DrivePrefix, this.RootPath, NormalizeSlashes(oldname)), new DriveItem() { Name = newname }, cancelToken).ConfigureAwait(false); + var drivePrefix = await GetDrivePrefix(cancelToken).ConfigureAwait(false); + await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, + ct => PatchAsync(string.Format("{0}/root:{1}{2}", drivePrefix, this.RootPath, NormalizeSlashes(oldname)), new DriveItem() { Name = newname }, ct) + ).ConfigureAwait(false); } catch (DriveItemNotFoundException ex) { @@ -379,19 +354,20 @@ namespace Duplicati.Library.Backend public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) { using (FileStream fileStream = File.OpenRead(filename)) - { await PutAsync(remotename, fileStream, cancelToken).ConfigureAwait(false); - } } public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken) { + var drivePrefix = await GetDrivePrefix(cancelToken).ConfigureAwait(false); + // PUT only supports up to 4 MB file uploads. There's a separate process for larger files. if (stream.Length < PUT_MAX_SIZE) { await m_retryAfter.WaitForRetryAfterAsync(cancelToken).ConfigureAwait(false); - string putUrl = string.Format("{0}/root:{1}{2}:/content", this.DrivePrefix, this.RootPath, NormalizeSlashes(remotename)); - using (StreamContent streamContent = new StreamContent(stream)) + string putUrl = string.Format("{0}/root:{1}{2}:/content", drivePrefix, this.RootPath, NormalizeSlashes(remotename)); + using (var timeoutStream = stream.ObserveReadTimeout(m_timeouts.ReadWriteTimeout, false)) + using (var streamContent = new StreamContent(timeoutStream)) { streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); using (var response = await this.m_client.PutAsync(putUrl, streamContent, cancelToken).ConfigureAwait(false)) @@ -408,10 +384,10 @@ namespace Duplicati.Library.Backend // The documentation seems somewhat contradictory - it states that uploads must be done sequentially, // but also states that the nextExpectedRanges value returned may indicate multiple ranges... // For now, this plays it safe and does a sequential upload. - string createSessionUrl = string.Format("{0}/root:{1}{2}:/createUploadSession", this.DrivePrefix, this.RootPath, NormalizeSlashes(remotename)); + string createSessionUrl = string.Format("{0}/root:{1}{2}:/createUploadSession", drivePrefix, this.RootPath, NormalizeSlashes(remotename)); await m_retryAfter.WaitForRetryAfterAsync(cancelToken).ConfigureAwait(false); - using (HttpRequestMessage createSessionRequest = new HttpRequestMessage(HttpMethod.Post, createSessionUrl)) - using (HttpResponseMessage createSessionResponse = await this.m_client.SendAsync(createSessionRequest, cancelToken).ConfigureAwait(false)) + using (var createSessionRequest = new HttpRequestMessage(HttpMethod.Post, createSessionUrl)) + using (var createSessionResponse = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => this.m_client.SendAsync(createSessionRequest, ct)).ConfigureAwait(false)) { var uploadSession = await this.ParseResponseAsync(createSessionResponse, cancelToken).ConfigureAwait(false); @@ -428,7 +404,7 @@ namespace Duplicati.Library.Backend currentBufferSize = stream.Length - offset; } - using (Stream subStream = new ReadLimitLengthStream(stream, offset, currentBufferSize)) + using (var subStream = new ReadLimitLengthStream(stream, offset, currentBufferSize)) { read = subStream.Length; @@ -447,8 +423,9 @@ namespace Duplicati.Library.Backend fragmentCount, remotename); - using (HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Put, uploadSession.UploadUrl)) - using (StreamContent fragmentContent = new StreamContent(subStream)) + using (var request = new HttpRequestMessage(HttpMethod.Put, uploadSession.UploadUrl)) + using (var timeoutStream = subStream.ObserveReadTimeout(m_timeouts.ReadWriteTimeout, false)) + using (var fragmentContent = new StreamContent(timeoutStream)) { fragmentContent.Headers.ContentLength = read; fragmentContent.Headers.ContentRange = new ContentRangeHeaderValue(offset, offset + read - 1, stream.Length); @@ -458,7 +435,7 @@ namespace Duplicati.Library.Backend try { // The uploaded put requests will error if they are authenticated - using (HttpResponseMessage response = await this.m_client.SendAsync(request, false, cancelToken).ConfigureAwait(false)) + using (var response = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => this.m_client.SendAsync(request, false, ct)).ConfigureAwait(false)) { // Note: On the last request, the json result includes the default properties of the item that was uploaded await this.ParseResponseAsync(response, cancelToken).ConfigureAwait(false); @@ -585,12 +562,11 @@ namespace Duplicati.Library.Backend { try { + var drivePrefix = await GetDrivePrefix(cancelToken).ConfigureAwait(false); await m_retryAfter.WaitForRetryAfterAsync(cancelToken).ConfigureAwait(false); - string deleteUrl = string.Format("{0}/root:{1}{2}", this.DrivePrefix, this.RootPath, NormalizeSlashes(remotename)); - using (var response = await this.m_client.DeleteAsync(deleteUrl, cancelToken).ConfigureAwait(false)) - { + string deleteUrl = string.Format("{0}/root:{1}{2}", drivePrefix, this.RootPath, NormalizeSlashes(remotename)); + using (var response = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => this.m_client.DeleteAsync(deleteUrl, ct)).ConfigureAwait(false)) this.CheckResponse(response); - } } catch (DriveItemNotFoundException ex) { @@ -603,7 +579,8 @@ namespace Duplicati.Library.Backend { try { - string rootPath = string.Format("{0}/root:{1}", this.DrivePrefix, this.RootPath); + var drivePrefix = await GetDrivePrefix(cancelToken).ConfigureAwait(false); + string rootPath = string.Format("{0}/root:{1}", drivePrefix, this.RootPath); await this.GetAsync(rootPath, cancelToken).ConfigureAwait(false); } catch (DriveItemNotFoundException ex) @@ -672,13 +649,13 @@ namespace Duplicati.Library.Backend private async IAsyncEnumerable Enumerate(string url, [EnumeratorCancellation] CancellationToken cancelToken) { - string nextUrl = url; + var nextUrl = url; while (!string.IsNullOrEmpty(nextUrl)) { GraphCollection results; try { - results = await this.GetAsync>(nextUrl, cancelToken).ConfigureAwait(false); + results = await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, ct => this.GetAsync>(nextUrl, ct)).ConfigureAwait(false); } catch (DriveItemNotFoundException ex) { @@ -686,10 +663,8 @@ namespace Duplicati.Library.Backend throw new FolderMissingException(ex); } - foreach (T result in results.Value) - { + foreach (T result in results.Value ?? []) yield return result; - } nextUrl = results.ODataNextLink; } @@ -714,49 +689,15 @@ namespace Duplicati.Library.Backend } } - private void CheckResponse(HttpWebResponse response) - { - string retryAfterHeader = response.Headers[HttpResponseHeader.RetryAfter]; - if (retryAfterHeader != null && RetryConditionHeaderValue.TryParse(retryAfterHeader, out RetryConditionHeaderValue retryAfter)) - { - m_retryAfter.SetRetryAfter(retryAfter); - } - - if (!((int)response.StatusCode >= 200 && (int)response.StatusCode < 300)) - { - if (response.StatusCode == HttpStatusCode.NotFound) - { - // It looks like this is an 'item not found' exception, so wrap it in a new exception class to make it easier to pick things out. - throw new DriveItemNotFoundException(response); - } - else - { - // Throw a wrapper exception to make it easier for the caller to look at specific status codes, etc. - throw new MicrosoftGraphException(response); - } - } - } - private async Task ParseResponseAsync(HttpResponseMessage response, CancellationToken cancelToken) { - this.CheckResponse(response); - using (var responseStream = await response.Content.ReadAsStreamAsync(cancelToken).ConfigureAwait(false)) - using (var reader = new StreamReader(responseStream)) + CheckResponse(response); + using (var responseStream = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => response.Content.ReadAsStreamAsync(ct)).ConfigureAwait(false)) + using (var timeoutStream = responseStream.ObserveReadTimeout(m_timeouts.ReadWriteTimeout)) + using (var reader = new StreamReader(timeoutStream)) using (var jsonReader = new JsonTextReader(reader)) - { - return this.m_serializer.Deserialize(jsonReader); - } - } - - private Task ParseResponseAsync(HttpWebResponse response, CancellationToken cancelToken) - { - this.CheckResponse(response); - using (Stream responseStream = response.GetResponseStream()) - using (StreamReader reader = new StreamReader(responseStream)) - using (JsonTextReader jsonReader = new JsonTextReader(reader)) - { - return Task.FromResult(this.m_serializer.Deserialize(jsonReader)); - } + return m_serializer.Deserialize(jsonReader) + ?? throw new MicrosoftGraphException("Failed to parse response", response); } private async Task ThrowUploadSessionException( @@ -801,16 +742,11 @@ namespace Duplicati.Library.Backend return url; } - private StringContent PrepareContent(T body) + private StringContent? PrepareContent(T body) { - if (body != null) - { - return new StringContent(JsonConvert.SerializeObject(body), Encoding.UTF8, "application/json"); - } - else - { + if (body == null) return null; - } + return new StringContent(JsonConvert.SerializeObject(body), Encoding.UTF8, "application/json"); } } } diff --git a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphTypes.cs b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphTypes.cs index b1ac34a61..50100a8d4 100644 --- a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphTypes.cs +++ b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphTypes.cs @@ -18,9 +18,6 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System; -using System.Linq; - using Newtonsoft.Json; using Newtonsoft.Json.Converters; @@ -34,10 +31,10 @@ namespace Duplicati.Library.Backend.MicrosoftGraph public class Identity { [JsonProperty("id", NullValueHandling = NullValueHandling.Ignore)] - public string Id { get; set; } + public string? Id { get; set; } [JsonProperty("displayName", NullValueHandling = NullValueHandling.Ignore)] - public string DisplayName { get; set; } + public string? DisplayName { get; set; } } public class IdentitySet @@ -46,37 +43,37 @@ namespace Duplicati.Library.Backend.MicrosoftGraph /// The optional application associated with this action /// [JsonProperty("application", NullValueHandling = NullValueHandling.Ignore)] - public Identity Application { get; set; } + public Identity? Application { get; set; } /// /// The optional device associated with this action /// [JsonProperty("device", NullValueHandling = NullValueHandling.Ignore)] - public Identity Device { get; set; } + public Identity? Device { get; set; } /// /// The optional user associated with this action /// [JsonProperty("user", NullValueHandling = NullValueHandling.Ignore)] - public Identity User { get; set; } + public Identity? User { get; set; } } public class BaseItem { [JsonProperty("id", NullValueHandling = NullValueHandling.Ignore)] - public string Id { get; set; } + public string? Id { get; set; } [JsonProperty("createdBy", NullValueHandling = NullValueHandling.Ignore)] - public IdentitySet CreatedBy { get; set; } + public IdentitySet? CreatedBy { get; set; } [JsonProperty("createdDateTime", NullValueHandling = NullValueHandling.Ignore)] public DateTimeOffset? CreatedDateTime { get; set; } [JsonProperty("description", NullValueHandling = NullValueHandling.Ignore)] - public string Description { get; set; } + public string? Description { get; set; } [JsonProperty("lastModifiedBy", NullValueHandling = NullValueHandling.Ignore)] - public IdentitySet LastModifiedBy { get; set; } + public IdentitySet? LastModifiedBy { get; set; } [JsonProperty("lastModifiedDateTime", NullValueHandling = NullValueHandling.Ignore)] public DateTimeOffset? LastModifiedDateTime { get; set; } @@ -90,10 +87,10 @@ namespace Duplicati.Library.Backend.MicrosoftGraph /// https://support.microsoft.com/en-us/help/2829981/support-for-whitespace-characters-in-file-and-folder-names-for-windows /// [JsonProperty("name", NullValueHandling = NullValueHandling.Ignore)] - public string Name { get; set; } + public string? Name { get; set; } [JsonProperty("webUrl", NullValueHandling = NullValueHandling.Ignore)] - public string WebUrl { get; set; } + public string? WebUrl { get; set; } } public enum QuotaState @@ -176,10 +173,10 @@ namespace Duplicati.Library.Backend.MicrosoftGraph public DriveType DriveType { get; set; } [JsonProperty("owner", NullValueHandling = NullValueHandling.Ignore)] - public IdentitySet Owner { get; set; } + public IdentitySet? Owner { get; set; } [JsonProperty("quota", NullValueHandling = NullValueHandling.Ignore)] - public Quota Quota { get; set; } + public Quota? Quota { get; set; } } public class HashesType @@ -189,30 +186,30 @@ namespace Duplicati.Library.Backend.MicrosoftGraph /// Available only in OneDrive personal /// [JsonProperty("sha1Hash", NullValueHandling = NullValueHandling.Ignore)] - public string Sha1Hash { get; set; } + public string? Sha1Hash { get; set; } /// /// CRC32 hash /// Available only in OneDrive personal /// [JsonProperty("crc32Hash", NullValueHandling = NullValueHandling.Ignore)] - public string Crc32Hash { get; set; } + public string? Crc32Hash { get; set; } /// /// A proprietary hash of the file that can be used to determine if the file's contents have changed. /// Available on in OneDrive for Business and SharePoint Server 2016. /// [JsonProperty("quickXorHash", NullValueHandling = NullValueHandling.Ignore)] - public string QuickXorHash { get; set; } + public string? QuickXorHash { get; set; } } public class FileFacet { [JsonProperty("hashes", NullValueHandling = NullValueHandling.Ignore)] - public HashesType Hashes { get; set; } + public HashesType? Hashes { get; set; } [JsonProperty("mimeType", NullValueHandling = NullValueHandling.Ignore)] - public string MimeType { get; set; } + public string? MimeType { get; set; } } public enum SortBy @@ -307,7 +304,7 @@ namespace Duplicati.Library.Backend.MicrosoftGraph public long? ChildCount { get; set; } [JsonProperty("view", NullValueHandling = NullValueHandling.Ignore)] - public FolderView View { get; set; } + public FolderView? View { get; set; } } /// @@ -337,19 +334,19 @@ namespace Duplicati.Library.Backend.MicrosoftGraph /// Package type - currently only oneNote is defined, but others could be used. /// [JsonProperty("type", NullValueHandling = NullValueHandling.Ignore)] - public string Type { get; set; } + public string? Type { get; set; } } public class DeletedFacet { [JsonProperty("state", NullValueHandling = NullValueHandling.Ignore)] - public string State { get; set; } + public string? State { get; set; } } public class RemoteItemFacet { [JsonProperty("remoteItem", NullValueHandling = NullValueHandling.Ignore)] - public RemoteItemFacet RemoteItem { get; set; } + public RemoteItemFacet? RemoteItem { get; set; } } public class RootFacet @@ -362,7 +359,7 @@ namespace Duplicati.Library.Backend.MicrosoftGraph /// The unique identifier which can be used under /drive/special/ /// [JsonProperty("name", NullValueHandling = NullValueHandling.Ignore)] - public string Name { get; set; } + public string? Name { get; set; } } /// @@ -372,19 +369,19 @@ namespace Duplicati.Library.Backend.MicrosoftGraph public class ItemReference { [JsonProperty("driveId", NullValueHandling = NullValueHandling.Ignore)] - public string DriveId { get; set; } + public string? DriveId { get; set; } [JsonProperty("driveType", NullValueHandling = NullValueHandling.Ignore)] public DriveType DriveType { get; set; } [JsonProperty("id", NullValueHandling = NullValueHandling.Ignore)] - public string Id { get; set; } + public string? Id { get; set; } [JsonProperty("name", NullValueHandling = NullValueHandling.Ignore)] - public string Name { get; set; } + public string? Name { get; set; } [JsonProperty("path", NullValueHandling = NullValueHandling.Ignore)] - public string Path { get; set; } + public string? Path { get; set; } } public enum ConflictBehavior @@ -403,41 +400,41 @@ namespace Duplicati.Library.Backend.MicrosoftGraph public class DriveItem : BaseItem { [JsonProperty("parentReference", NullValueHandling = NullValueHandling.Ignore)] - public ItemReference ParentReference { get; set; } + public ItemReference? ParentReference { get; set; } [JsonProperty("size", NullValueHandling = NullValueHandling.Ignore)] public long? Size { get; set; } [JsonProperty("file", NullValueHandling = NullValueHandling.Ignore)] - public FileFacet File { get; set; } + public FileFacet? File { get; set; } [JsonProperty("folder", NullValueHandling = NullValueHandling.Ignore)] - public FolderFacet Folder { get; set; } + public FolderFacet? Folder { get; set; } [JsonProperty("fileSystemInfo", NullValueHandling = NullValueHandling.Ignore)] - public FileSystemInfoFacet FileSystemInfo { get; set; } + public FileSystemInfoFacet? FileSystemInfo { get; set; } [JsonProperty("package", NullValueHandling = NullValueHandling.Ignore)] - public PackageFacet Package { get; set; } + public PackageFacet? Package { get; set; } [JsonProperty("deleted", NullValueHandling = NullValueHandling.Ignore)] - public DeletedFacet Deleted { get; set; } + public DeletedFacet? Deleted { get; set; } [JsonProperty("remoteItem", NullValueHandling = NullValueHandling.Ignore)] - public RemoteItemFacet RemoteItem { get; set; } + public RemoteItemFacet? RemoteItem { get; set; } [JsonProperty("root", NullValueHandling = NullValueHandling.Ignore)] - public RootFacet Root { get; set; } + public RootFacet? Root { get; set; } [JsonProperty("specialFolder", NullValueHandling = NullValueHandling.Ignore)] - public SpecialFolderFacet SpecialFolder { get; set; } + public SpecialFolderFacet? SpecialFolder { get; set; } [JsonProperty("@microsoft.graph.conflictBehavior", NullValueHandling = NullValueHandling.Ignore)] [JsonConverter(typeof(StringEnumConverter))] public ConflictBehavior? ConflictBehavior { get; set; } [JsonProperty("@microsoft.graph.downloadUrl", NullValueHandling = NullValueHandling.Ignore)] - public string DownloadUrl { get; set; } + public string? DownloadUrl { get; set; } [JsonIgnore] public bool IsFile @@ -515,25 +512,25 @@ namespace Duplicati.Library.Backend.MicrosoftGraph public class GraphCollection { [JsonProperty("value", NullValueHandling = NullValueHandling.Ignore)] - public T[] Value { get; set; } + public T[]? Value { get; set; } [JsonProperty("@odata.nextLink", NullValueHandling = NullValueHandling.Ignore)] - public string ODataNextLink { get; set; } + public string? ODataNextLink { get; set; } } public class UploadSession { [JsonProperty("uploadUrl", NullValueHandling = NullValueHandling.Ignore)] - public string UploadUrl { get; set; } + public string? UploadUrl { get; set; } [JsonProperty("expirationDateTime", NullValueHandling = NullValueHandling.Ignore)] public DateTimeOffset? ExpirationDateTime { get; set; } [JsonProperty("nextExpectedRanges", NullValueHandling = NullValueHandling.Ignore)] - public string[] NextExpectedRanges { get; set; } + public string[]? NextExpectedRanges { get; set; } [JsonProperty("item", NullValueHandling = NullValueHandling.Ignore)] - public DriveItem Item { get; set; } + public DriveItem? Item { get; set; } } public class SharePointSite : BaseItem @@ -543,16 +540,16 @@ namespace Duplicati.Library.Backend.MicrosoftGraph public class Group : BaseItem { [JsonProperty("mail", NullValueHandling = NullValueHandling.Ignore)] - public string Mail { get; set; } + public string? Mail { get; set; } [JsonProperty("proxyAddresses", NullValueHandling = NullValueHandling.Ignore)] - public string[] ProxyAddresses { get; set; } + public string[]? ProxyAddresses { get; set; } [JsonProperty("mailNickname", NullValueHandling = NullValueHandling.Ignore)] - public string MailNickname { get; set; } + public string? MailNickname { get; set; } [JsonProperty("groupTypes", NullValueHandling = NullValueHandling.Ignore)] - public string[] GroupTypes { get; set; } + public string[]? GroupTypes { get; set; } [JsonIgnore] public bool IsUnifiedGroup diff --git a/Duplicati/Library/Backend/OneDrive/MicrosoftGroup.cs b/Duplicati/Library/Backend/OneDrive/MicrosoftGroup.cs index 0e3aa5cd7..f5bca8593 100644 --- a/Duplicati/Library/Backend/OneDrive/MicrosoftGroup.cs +++ b/Duplicati/Library/Backend/OneDrive/MicrosoftGroup.cs @@ -18,13 +18,9 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; + using Duplicati.Library.Backend.MicrosoftGraph; using Duplicati.Library.Interface; -using Duplicati.Library.Utility; namespace Duplicati.Library.Backend { @@ -34,93 +30,76 @@ namespace Duplicati.Library.Backend private const string GROUP_ID_OPTION = "group-id"; private const string PROTOCOL_KEY = "msgroup"; - private readonly string drivePath; + private readonly Dictionary options; + private string? drivePath; - public MicrosoftGroup() { } // Constructor needed for dynamic loading to find it - - public MicrosoftGroup(string url, Dictionary options) - : base(url, MicrosoftGroup.PROTOCOL_KEY, options) + public MicrosoftGroup() { - string groupId = null; - string groupEmail; - if (options.TryGetValue(GROUP_EMAIL_OPTION, out groupEmail)) - { - // TODO: Should not make network requests in constructor - groupId = this.GetGroupIdFromEmailAsync(groupEmail, CancellationToken.None).Await(); - } + // Constructor needed for dynamic loading to find it + options = null!; + } - string groupIdOption; - if (options.TryGetValue(GROUP_ID_OPTION, out groupIdOption)) + public MicrosoftGroup(string url, Dictionary options) + : base(url, PROTOCOL_KEY, options) + { + this.options = options; + } + + public override string ProtocolKey => PROTOCOL_KEY; + + public override string DisplayName => Strings.MicrosoftGroup.DisplayName; + + protected override async Task GetDrivePath(CancellationToken cancelToken) + { + if (string.IsNullOrEmpty(drivePath)) { - if (!string.IsNullOrEmpty(groupId) && !string.Equals(groupId, groupIdOption)) + string? groupId = null; + var groupEmail = options.GetValueOrDefault(GROUP_EMAIL_OPTION); + if (!string.IsNullOrWhiteSpace(groupEmail)) + groupId = await this.GetGroupIdFromEmailAsync(groupEmail, cancelToken).ConfigureAwait(false); + + var groupIdOption = options.GetValueOrDefault(GROUP_ID_OPTION); + if (!string.IsNullOrWhiteSpace(groupIdOption)) { - throw new UserInformationException(Strings.MicrosoftGroup.ConflictingGroupId(groupIdOption, groupId), "MicrosoftGroupConflictingGroupId"); + if (!string.IsNullOrEmpty(groupId) && !string.Equals(groupId, groupIdOption)) + throw new UserInformationException(Strings.MicrosoftGroup.ConflictingGroupId(groupIdOption, groupId), "MicrosoftGroupConflictingGroupId"); + + groupId = groupIdOption; } - groupId = groupIdOption; + if (string.IsNullOrWhiteSpace(groupId)) + throw new UserInformationException(Strings.MicrosoftGroup.MissingGroupIdAndEmailAddress, "MicrosoftGroupMissingGroupIdAndEmailAddress"); + + drivePath = string.Format("/groups/{0}/drive", groupId); } - if (string.IsNullOrEmpty(groupId)) - { - throw new UserInformationException(Strings.MicrosoftGroup.MissingGroupIdAndEmailAddress, "MicrosoftGroupMissingGroupIdAndEmailAddress"); - } - - this.drivePath = string.Format("/groups/{0}/drive", groupId); + return drivePath; } - public override string ProtocolKey - { - get { return MicrosoftGroup.PROTOCOL_KEY; } - } + protected override DescriptionTemplateDelegate DescriptionTemplate => Strings.MicrosoftGroup.Description; - public override string DisplayName - { - get { return Strings.MicrosoftGroup.DisplayName; } - } - - protected override string DrivePath - { - get { return this.drivePath; } - } - - protected override DescriptionTemplateDelegate DescriptionTemplate - { - get - { - return Strings.MicrosoftGroup.Description; - } - } - - protected override IList AdditionalSupportedCommands - { - get - { - return - [ + protected override IList AdditionalSupportedCommands => [ new CommandLineArgument(GROUP_ID_OPTION, CommandLineArgument.ArgumentType.String, Strings.MicrosoftGroup.GroupIdShort, Strings.MicrosoftGroup.GroupIdLong), new CommandLineArgument(GROUP_EMAIL_OPTION, CommandLineArgument.ArgumentType.String, Strings.MicrosoftGroup.GroupEmailShort, Strings.MicrosoftGroup.GroupEmailLong), ]; - } - } private async Task GetGroupIdFromEmailAsync(string email, CancellationToken cancelToken) { // We can get all groups that have the given email as one of their addresses with: // https://graph.microsoft.com/v1.0/groups?$filter=mail eq '{email}' or proxyAddresses/any(x:x eq 'smtp:{email}') string request = string.Format("{0}/groups?$filter=mail eq '{1}' or proxyAddresses/any(x:x eq 'smtp:{1}')", this.ApiVersion, email); - var groups = await this.GetAsync>(request, cancelToken).ConfigureAwait(false); - if (groups.Value.Length == 0) - { + var groups = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => GetAsync>(request, ct)).ConfigureAwait(false); + if (groups.Value == null || groups.Value.Length == 0) throw new UserInformationException(Strings.MicrosoftGroup.NoGroupsWithEmail(email), "MicrosoftGroupNoGroupsWithEmail"); - } - else if (groups.Value.Length > 1) - { + + if (groups.Value.Length > 1) throw new UserInformationException(Strings.MicrosoftGroup.MultipleGroupsWithEmail(email), "MicrosoftGroupMultipleGroupsWithEmail"); - } - else - { - return groups.Value.Single().Id; - } + + var id = groups.Value.Single().Id; + if (string.IsNullOrEmpty(id)) + throw new UserInformationException(Strings.MicrosoftGroup.NoGroupsWithEmail(email), "MicrosoftGroupNoGroupsWithEmail"); + + return id; } } } diff --git a/Duplicati/Library/Backend/OneDrive/OneDrive.cs b/Duplicati/Library/Backend/OneDrive/OneDrive.cs index 52c6b060a..c263e422d 100644 --- a/Duplicati/Library/Backend/OneDrive/OneDrive.cs +++ b/Duplicati/Library/Backend/OneDrive/OneDrive.cs @@ -18,7 +18,6 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System.Collections.Generic; using Duplicati.Library.Interface; @@ -30,56 +29,33 @@ namespace Duplicati.Library.Backend private const string DEFAULT_DRIVE_PATH = "/me/drive"; private const string PROTOCOL_KEY = "onedrivev2"; - private readonly string drivePath; + private readonly Task drivePath; - public OneDrive() { } // Constructor needed for dynamic loading to find it - - public OneDrive(string url, Dictionary options) - : base(url, OneDrive.PROTOCOL_KEY, options) + public OneDrive() { - string driveId; - if (options.TryGetValue(DRIVE_ID_OPTION, out driveId)) - { - this.drivePath = string.Format("/drives/{0}", driveId); - } - else - { - this.drivePath = DEFAULT_DRIVE_PATH; - } + // Constructor needed for dynamic loading to find it + drivePath = null!; } - public override string ProtocolKey + public OneDrive(string url, Dictionary options) + : base(url, PROTOCOL_KEY, options) { - get { return OneDrive.PROTOCOL_KEY; } + var driveId = options.GetValueOrDefault(DRIVE_ID_OPTION); + drivePath = Task.FromResult(string.IsNullOrWhiteSpace(driveId) + ? DEFAULT_DRIVE_PATH + : string.Format("/drives/{0}", driveId)); } - public override string DisplayName - { - get { return Strings.OneDrive.DisplayName; } - } + public override string ProtocolKey => PROTOCOL_KEY; - protected override string DrivePath - { - get { return this.drivePath; } - } + public override string DisplayName => Strings.OneDrive.DisplayName; - protected override DescriptionTemplateDelegate DescriptionTemplate - { - get - { - return Strings.OneDrive.Description; - } - } + protected override Task GetDrivePath(CancellationToken cancelToken) => drivePath; - protected override IList AdditionalSupportedCommands - { - get - { - return new ICommandLineArgument[] - { - new CommandLineArgument(DRIVE_ID_OPTION, CommandLineArgument.ArgumentType.String, Strings.OneDrive.DriveIdShort, Strings.OneDrive.DriveIdLong(DEFAULT_DRIVE_PATH)), - }; - } - } + protected override DescriptionTemplateDelegate DescriptionTemplate => Strings.OneDrive.Description; + + protected override IList AdditionalSupportedCommands => [ + new CommandLineArgument(DRIVE_ID_OPTION, CommandLineArgument.ArgumentType.String, Strings.OneDrive.DriveIdShort, Strings.OneDrive.DriveIdLong(DEFAULT_DRIVE_PATH)), + ]; } } diff --git a/Duplicati/Library/Backend/OneDrive/SharePointV2.cs b/Duplicati/Library/Backend/OneDrive/SharePointV2.cs index 345622453..e130cac88 100644 --- a/Duplicati/Library/Backend/OneDrive/SharePointV2.cs +++ b/Duplicati/Library/Backend/OneDrive/SharePointV2.cs @@ -18,12 +18,7 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; + using Duplicati.Library.Backend.MicrosoftGraph; using Duplicati.Library.Interface; @@ -34,67 +29,40 @@ namespace Duplicati.Library.Backend private const string SITE_ID_OPTION = "site-id"; private const string PROTOCOL_KEY = "sharepoint"; - private readonly string drivePath; - private string siteId = null; + private readonly Task drivePath; + private string siteId; - public SharePointV2() { } // Constructor needed for dynamic loading to find it + public SharePointV2() + { + // Constructor needed for dynamic loading to find it + drivePath = null!; + siteId = null!; + } - public SharePointV2(string url, Dictionary options) - : base(url, SharePointV2.PROTOCOL_KEY, options) + public SharePointV2(string url, Dictionary options) + : base(url, PROTOCOL_KEY, options) { // Check to see if a site ID was explicitly provided - string siteIdOption; - if (options.TryGetValue(SITE_ID_OPTION, out siteIdOption)) - { - if (!string.IsNullOrEmpty(this.siteId) && !string.Equals(this.siteId, siteIdOption)) - { - throw new UserInformationException(Strings.SharePointV2.ConflictingSiteId(siteIdOption, this.siteId), "SharePointConflictingSiteId"); - } - - this.siteId = siteIdOption; - } - - if (string.IsNullOrEmpty(this.siteId)) - { + var siteIdOption = options.GetValueOrDefault(SITE_ID_OPTION); + if (!string.IsNullOrWhiteSpace(siteIdOption)) + siteId = siteIdOption; + else throw new UserInformationException(Strings.SharePointV2.MissingSiteId, "SharePointMissingSiteId"); - } - this.drivePath = string.Format("/sites/{0}/drive", this.siteId); + drivePath = Task.FromResult(string.Format("/sites/{0}/drive", siteId)); } - public override string ProtocolKey - { - get { return SharePointV2.PROTOCOL_KEY; } - } + public override string ProtocolKey => PROTOCOL_KEY; - public override string DisplayName - { - get { return Strings.SharePointV2.DisplayName; } - } + public override string DisplayName => Strings.SharePointV2.DisplayName; - protected override string DrivePath - { - get { return this.drivePath; } - } + protected override Task GetDrivePath(CancellationToken cancelToken) => drivePath; - protected override DescriptionTemplateDelegate DescriptionTemplate - { - get - { - return Strings.SharePointV2.Description; - } - } + protected override DescriptionTemplateDelegate DescriptionTemplate => Strings.SharePointV2.Description; - protected override IList AdditionalSupportedCommands - { - get - { - return new ICommandLineArgument[] - { - new CommandLineArgument(SITE_ID_OPTION, CommandLineArgument.ArgumentType.String, Strings.SharePointV2.SiteIdShort, Strings.SharePointV2.SiteIdLong), - }; - } - } + protected override IList AdditionalSupportedCommands => [ + new CommandLineArgument(SITE_ID_OPTION, CommandLineArgument.ArgumentType.String, Strings.SharePointV2.SiteIdShort, Strings.SharePointV2.SiteIdLong), + ]; /// /// This method takes an input URL, which could be either in the format that the old SharePoint backend accepted or @@ -107,7 +75,7 @@ namespace Duplicati.Library.Backend /// Path within the drive protected override async Task GetRootPathFromUrlAsync(string url, CancellationToken cancelToken) { - Uri uri = new Uri(url); + var uri = new Uri(url); // If the user gave a URL like "https://{tenant}.sharepoint.com/path" in the UI, // it might appear here as "sharepoint://https://{tenant}.sharepoint.com/path". @@ -124,13 +92,16 @@ namespace Duplicati.Library.Backend // Get the web URL of the site's main drive try { - var drive = await this.GetAsync(string.Format("{0}/sites/{1}/drive", this.ApiVersion, site.Id), cancelToken).ConfigureAwait(false); + var drive = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => this.GetAsync(string.Format("{0}/sites/{1}/drive", this.ApiVersion, site.Id), ct)).ConfigureAwait(false); - this.siteId = site.Id; - var driveWebUrl = new Uri(drive.WebUrl); + if (drive != null && !string.IsNullOrEmpty(drive.WebUrl) && !string.IsNullOrWhiteSpace(site.Id)) + { + siteId = site.Id; + var driveWebUrl = new Uri(drive.WebUrl); - // Make sure to replace any "//" in the original path with "/", so the substrings line up. - return uri.LocalPath.Replace("//", "/").Substring(driveWebUrl.LocalPath.Length); + // Make sure to replace any "//" in the original path with "/", so the substrings line up. + return uri.LocalPath.Replace("//", "/").Substring(driveWebUrl.LocalPath.Length); + } } catch (MicrosoftGraphException) { @@ -157,7 +128,7 @@ namespace Duplicati.Library.Backend try { var request = string.Format("{0}:/{1}", requestBase, uri.Path.Substring(0, siteHint)); - return await this.GetAsync(request, cancelToken).ConfigureAwait(false); + return await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => this.GetAsync(request, ct)).ConfigureAwait(false); } catch (MicrosoftGraphException) { @@ -171,7 +142,7 @@ namespace Duplicati.Library.Backend try { string request = string.Format("{0}:/{1}", requestBase, string.Join("/", pathPieces.Take(i))); - return await this.GetAsync(request, cancelToken).ConfigureAwait(false); + return await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => this.GetAsync(request, ct)).ConfigureAwait(false); } catch (MicrosoftGraphException) { diff --git a/Duplicati/Library/Backend/OneDrive/Strings.cs b/Duplicati/Library/Backend/OneDrive/Strings.cs index a018f4e79..7a7c3ea2f 100644 --- a/Duplicati/Library/Backend/OneDrive/Strings.cs +++ b/Duplicati/Library/Backend/OneDrive/Strings.cs @@ -1,31 +1,28 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Localization.Short; namespace Duplicati.Library.Backend.Strings { internal static class MicrosoftGraph { - public static string AuthIdLong(string url) { return LC.L(@"The authorization token retrieved from {0}", url); } - public static string AuthIdShort { get { return LC.L(@"The authorization code"); } } - public static string MissingAuthId(string url) { return LC.L(@"No Auth-ID was provided - you can get one from {0}", url); } public static string FragmentSizeLong { get { return LC.L(@"Size of individual fragments which are uploaded separately for large files. It is recommended to be between 5-10 MiB (though a smaller value may work better on a slower or less reliable connection), and to be a multiple of 320 KiB."); } } public static string FragmentSizeShort { get { return LC.L(@"Fragment size for large uploads"); } } public static string FragmentRetryCountLong { get { return LC.L(@"Number of retry attempts made for each fragment before failing the overall file upload."); } } @@ -52,7 +49,7 @@ namespace Duplicati.Library.Backend.Strings public static string SiteIdLong { get { return LC.L(@"ID of the site to store data in."); } } public static string SiteIdShort { get { return LC.L(@"ID of the site"); } } public static string MissingSiteId { get { return LC.L(@"No site ID was provided"); } } - public static string ConflictingSiteId(string given, string found) { return LC.L(@"Conflicting site IDs used: given {0} but found {1}", given, found); } + public static string MissingDriveInfo { get { return LC.L(@"No drive information was returned"); } } } internal static class MicrosoftGroup From 53972a4c780d02931861680f5ad25d5d52b82f16 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 21 Mar 2025 16:35:29 +0100 Subject: [PATCH 24/48] Updated OpenStack --- ...Duplicati.Library.Backend.OpenStack.csproj | 2 + .../Backend/OpenStack/OpenStackConfig.cs | 76 ++--- .../Backend/OpenStack/OpenStackStorage.cs | 314 +++++++++--------- .../Library/Backend/OpenStack/Strings.cs | 46 +-- 4 files changed, 224 insertions(+), 214 deletions(-) diff --git a/Duplicati/Library/Backend/OpenStack/Duplicati.Library.Backend.OpenStack.csproj b/Duplicati/Library/Backend/OpenStack/Duplicati.Library.Backend.OpenStack.csproj index b9c7aa57c..7de58c33f 100644 --- a/Duplicati/Library/Backend/OpenStack/Duplicati.Library.Backend.OpenStack.csproj +++ b/Duplicati/Library/Backend/OpenStack/Duplicati.Library.Backend.OpenStack.csproj @@ -4,6 +4,8 @@ net8.0 Library Copyright © 2025 Team Duplicati, MIT license + enable + enable diff --git a/Duplicati/Library/Backend/OpenStack/OpenStackConfig.cs b/Duplicati/Library/Backend/OpenStack/OpenStackConfig.cs index b07f9b580..280d22cbf 100644 --- a/Duplicati/Library/Backend/OpenStack/OpenStackConfig.cs +++ b/Duplicati/Library/Backend/OpenStack/OpenStackConfig.cs @@ -1,36 +1,32 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System; using Duplicati.Library.Interface; -using System.Collections.Generic; -using System.Linq; -using Duplicati.Library.Localization.Short; namespace Duplicati.Library.Backend.OpenStack { public class SwiftConfig : IWebModule { private const ConfigType DEFAULT_CONFIG_TYPE = ConfigType.Providers; - private static readonly string DEFAULT_CONFIG_TYPE_STR = Enum.GetName(typeof(ConfigType), DEFAULT_CONFIG_TYPE); + private static readonly string DEFAULT_CONFIG_TYPE_STR = DEFAULT_CONFIG_TYPE.ToString(); private const string KEY_CONFIGTYPE = "openstack-config"; public enum ConfigType @@ -42,42 +38,32 @@ namespace Duplicati.Library.Backend.OpenStack { } - public System.Collections.Generic.IDictionary Execute(System.Collections.Generic.IDictionary options) + public IDictionary Execute(IDictionary options) { - string k; - options.TryGetValue(KEY_CONFIGTYPE, out k); + options.TryGetValue(KEY_CONFIGTYPE, out var k); if (string.IsNullOrWhiteSpace(k)) k = DEFAULT_CONFIG_TYPE_STR; - ConfigType ct; - if (!Enum.TryParse(k, true, out ct)) + if (!Enum.TryParse(k, true, out var ct)) ct = DEFAULT_CONFIG_TYPE; switch (ct) { case ConfigType.Versions: - return OpenStack.OpenStackStorage.OPENSTACK_VERSIONS.ToDictionary((x) => x.Key, (y) => y.Value); + return OpenStackStorage.OPENSTACK_VERSIONS.ToDictionary((x) => x.Key, (y) => y.Value); default: - return OpenStack.OpenStackStorage.KNOWN_OPENSTACK_PROVIDERS.ToDictionary((x) => x.Key, (y) => y.Value); + return OpenStackStorage.KNOWN_OPENSTACK_PROVIDERS.ToDictionary((x) => x.Key, (y) => y.Value); } } - public string Key { get { return "openstack-getconfig"; } } + public string Key => "openstack-getconfig"; - public string DisplayName { get { return LC.L("OpenStack configuration module"); } } + public string DisplayName => Strings.SwiftConfig.DisplayName; - public string Description { get { return LC.L("Expose OpenStack configuration as a web module"); } } + public string Description => Strings.SwiftConfig.Description; - - public System.Collections.Generic.IList SupportedCommands - { - get - { - return new List([ - new CommandLineArgument(KEY_CONFIGTYPE, CommandLineArgument.ArgumentType.Enumeration, LC.L("The config to get"), LC.L("Provide different config values"), DEFAULT_CONFIG_TYPE_STR, Enum.GetNames(typeof(ConfigType))) - - ]); - } - } + public IList SupportedCommands => [ + new CommandLineArgument(KEY_CONFIGTYPE, CommandLineArgument.ArgumentType.Enumeration, Strings.SwiftConfig.ConfigTypeOptionShort, Strings.SwiftConfig.ConfigTypeOptionLong, DEFAULT_CONFIG_TYPE_STR, Enum.GetNames(typeof(ConfigType))) + ]; } } diff --git a/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs b/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs index 5731a8a31..416200c64 100644 --- a/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs +++ b/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs @@ -19,28 +19,22 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System; -using System.Collections.Generic; using Duplicati.Library.Interface; -using System.Linq; -using System.IO; using Duplicati.Library.Utility; using Newtonsoft.Json; using Newtonsoft.Json.Converters; using System.Net; using System.Text; using Duplicati.Library.Common.IO; -using System.Threading; -using System.Threading.Tasks; using System.Runtime.CompilerServices; +using Duplicati.Library.Utility.Options; +using System.Threading.Tasks; namespace Duplicati.Library.Backend.OpenStack { public class OpenStackStorage : IBackend, IStreamingBackend { private const string DOMAINNAME_OPTION = "openstack-domain-name"; - private const string USERNAME_OPTION = "auth-username"; - private const string PASSWORD_OPTION = "auth-password"; private const string TENANTNAME_OPTION = "openstack-tenant-name"; private const string AUTHURI_OPTION = "openstack-authuri"; private const string VERSION_OPTION = "openstack-version"; @@ -53,19 +47,20 @@ namespace Duplicati.Library.Backend.OpenStack private readonly string m_container; private readonly string m_prefix; - private readonly string m_domainName; + private readonly string? m_domainName; private readonly string m_username; - private readonly string m_password; + private readonly string? m_password; private readonly string m_authUri; - private readonly string m_version; - private readonly string m_tenantName; - private readonly string m_apikey; - private readonly string m_region; + private readonly string? m_version; + private readonly string? m_tenantName; + private readonly string? m_apikey; + private readonly string? m_region; + private readonly TimeoutOptionsHelper.Timeouts m_timeouts; - protected string m_simplestorageendpoint; + protected string? m_simplestorageendpoint; private readonly WebHelper m_helper; - private OpenStackAuthResponse.TokenClass m_accessToken; + private OpenStackAuthResponse.TokenClass? m_accessToken; public static readonly KeyValuePair[] KNOWN_OPENSTACK_PROVIDERS = { new KeyValuePair("Rackspace US", "https://identity.api.rackspacecloud.com/v2.0"), @@ -93,8 +88,8 @@ namespace Duplicati.Library.Backend.OpenStack { public class AuthContainer { - public Identity identity { get; set; } - public Scope scope { get; set; } + public Identity? identity { get; set; } + public Scope? scope { get; set; } } public class Identity @@ -103,7 +98,7 @@ namespace Duplicati.Library.Backend.OpenStack public IdentityMethods[] methods { get; set; } [JsonProperty("password", NullValueHandling = NullValueHandling.Ignore)] - public PasswordBasedRequest PasswordCredentials { get; set; } + public PasswordBasedRequest? PasswordCredentials { get; set; } public Identity() { @@ -113,7 +108,7 @@ namespace Duplicati.Library.Backend.OpenStack public class Scope { - public Project project; + public Project? project; } public enum IdentityMethods @@ -123,7 +118,7 @@ namespace Duplicati.Library.Backend.OpenStack public class PasswordBasedRequest { - public UserCredentials user { get; set; } + public UserCredentials? user { get; set; } } public class UserCredentials @@ -134,6 +129,9 @@ namespace Duplicati.Library.Backend.OpenStack public UserCredentials() { + domain = null!; + name = null!; + password = null!; } public UserCredentials(Domain domain, string name, string password) { @@ -146,9 +144,9 @@ namespace Duplicati.Library.Backend.OpenStack public class Domain { - public string name { get; set; } + public string? name { get; set; } - public Domain(string name) + public Domain(string? name) { this.name = name; } @@ -168,16 +166,16 @@ namespace Duplicati.Library.Backend.OpenStack public AuthContainer auth { get; set; } - public Keystone3AuthRequest(string domain_name, string username, string password, string project_name) + public Keystone3AuthRequest(string? domain_name, string username, string? password, string? project_name) { Domain domain = new Domain(domain_name); this.auth = new AuthContainer(); this.auth.identity = new Identity(); this.auth.identity.PasswordCredentials = new PasswordBasedRequest(); - this.auth.identity.PasswordCredentials.user = new UserCredentials(domain, username, password); + this.auth.identity.PasswordCredentials.user = new UserCredentials(domain, username, password!); this.auth.scope = new Scope(); - this.auth.scope.project = new Project(domain, project_name); + this.auth.scope.project = new Project(domain, project_name!); } } @@ -186,41 +184,41 @@ namespace Duplicati.Library.Backend.OpenStack public class AuthContainer { [JsonProperty("RAX-KSKEY:apiKeyCredentials", NullValueHandling = NullValueHandling.Ignore)] - public ApiKeyBasedRequest ApiCredentials { get; set; } + public ApiKeyBasedRequest? ApiCredentials { get; set; } [JsonProperty("passwordCredentials", NullValueHandling = NullValueHandling.Ignore)] - public PasswordBasedRequest PasswordCredentials { get; set; } + public PasswordBasedRequest? PasswordCredentials { get; set; } [JsonProperty("tenantName", NullValueHandling = NullValueHandling.Ignore)] - public string TenantName { get; set; } + public string? TenantName { get; set; } [JsonProperty("token", NullValueHandling = NullValueHandling.Ignore)] - public TokenBasedRequest Token { get; set; } + public TokenBasedRequest? Token { get; set; } } public class ApiKeyBasedRequest { - public string username { get; set; } - public string apiKey { get; set; } + public string? username { get; set; } + public string? apiKey { get; set; } } public class PasswordBasedRequest { - public string username { get; set; } - public string password { get; set; } - public string tenantName { get; set; } + public string? username { get; set; } + public string? password { get; set; } + public string? tenantName { get; set; } } public class TokenBasedRequest { - public string id { get; set; } + public string? id { get; set; } } public AuthContainer auth { get; set; } - public OpenStackAuthRequest(string tenantname, string username, string password, string apikey) + public OpenStackAuthRequest(string? tenantname, string username, string? password, string? apikey) { this.auth = new AuthContainer(); this.auth.TenantName = tenantname; @@ -247,70 +245,70 @@ namespace Duplicati.Library.Backend.OpenStack private class Keystone3AuthResponse { - public TokenClass token { get; set; } + public TokenClass? token { get; set; } public class EndpointItem { // 'interface' is a reserved keyword, so we need this decorator to map it [JsonProperty(PropertyName = "interface")] - public string interface_name { get; set; } - public string region { get; set; } - public string url { get; set; } + public string? interface_name { get; set; } + public string? region { get; set; } + public string? url { get; set; } } public class CatalogItem { - public EndpointItem[] endpoints { get; set; } - public string name { get; set; } - public string type { get; set; } + public EndpointItem[]? endpoints { get; set; } + public string? name { get; set; } + public string? type { get; set; } } public class TokenClass { - public CatalogItem[] catalog { get; set; } + public CatalogItem[]? catalog { get; set; } public DateTime? expires_at { get; set; } } } private class OpenStackAuthResponse { - public AccessClass access { get; set; } + public AccessClass? access { get; set; } public class TokenClass { - public string id { get; set; } + public string? id { get; set; } public DateTime? expires { get; set; } } public class EndpointItem { - public string region { get; set; } - public string tenantId { get; set; } - public string publicURL { get; set; } - public string internalURL { get; set; } + public string? region { get; set; } + public string? tenantId { get; set; } + public string? publicURL { get; set; } + public string? internalURL { get; set; } } public class ServiceItem { - public string name { get; set; } - public string type { get; set; } - public EndpointItem[] endpoints { get; set; } + public string? name { get; set; } + public string? type { get; set; } + public EndpointItem[]? endpoints { get; set; } } public class AccessClass { - public TokenClass token { get; set; } - public ServiceItem[] serviceCatalog { get; set; } + public TokenClass? token { get; set; } + public ServiceItem[]? serviceCatalog { get; set; } } } private class OpenStackStorageItem { - public string name { get; set; } + public string? name { get; set; } public DateTime? last_modified { get; set; } public long? bytes { get; set; } - public string content_type { get; set; } - public string subdir { get; set; } + public string? content_type { get; set; } + public string? subdir { get; set; } } private class WebHelper : JSONWebHelper @@ -319,43 +317,54 @@ namespace Duplicati.Library.Backend.OpenStack public WebHelper(OpenStackStorage parent) { m_parent = parent; } - public override HttpWebRequest CreateRequest(string url, string method = null) + public override HttpWebRequest CreateRequest(string url, string? method = null) { var req = base.CreateRequest(url, method); - req.Headers["X-Auth-Token"] = m_parent.AccessToken; + req.Headers["X-Auth-Token"] = m_parent.GetAccessToken(CancellationToken.None).Await(); return req; } } public OpenStackStorage() { + m_container = null!; + m_prefix = null!; + m_timeouts = null!; + m_username = null!; + m_authUri = null!; + m_helper = null!; } - public OpenStackStorage(string url, Dictionary options) + public OpenStackStorage(string url, Dictionary options) { var uri = new Utility.Uri(url); m_container = uri.Host; m_prefix = Util.AppendDirSeparator("/" + uri.Path, "/"); + m_timeouts = TimeoutOptionsHelper.Parse(options); // For OpenStack we do not use a leading slash if (m_prefix.StartsWith("/", StringComparison.Ordinal)) m_prefix = m_prefix.Substring(1); + var auth = AuthOptionsHelper.Parse(options, uri); + options.TryGetValue(DOMAINNAME_OPTION, out m_domainName); - options.TryGetValue(USERNAME_OPTION, out m_username); - options.TryGetValue(PASSWORD_OPTION, out m_password); options.TryGetValue(TENANTNAME_OPTION, out m_tenantName); - options.TryGetValue(AUTHURI_OPTION, out m_authUri); options.TryGetValue(VERSION_OPTION, out m_version); options.TryGetValue(APIKEY_OPTION, out m_apikey); options.TryGetValue(REGION_OPTION, out m_region); - if (string.IsNullOrWhiteSpace(m_username)) - throw new UserInformationException(Strings.OpenStack.MissingOptionError(USERNAME_OPTION), "OpenStackMissingUsername"); - if (string.IsNullOrWhiteSpace(m_authUri)) + if (!auth.HasUsername) + throw new UserInformationException(Strings.OpenStack.MissingOptionError(AuthOptionsHelper.AuthUsername), "OpenStackMissingUsername"); + + m_username = auth.Username!; + m_password = auth.Password; + var authUri = options.GetValueOrDefault(AUTHURI_OPTION); + if (string.IsNullOrWhiteSpace(authUri)) throw new UserInformationException(Strings.OpenStack.MissingOptionError(AUTHURI_OPTION), "OpenStackMissingAuthUri"); + m_authUri = authUri; if (string.IsNullOrWhiteSpace(m_version) && m_authUri.EndsWith("/v3", StringComparison.OrdinalIgnoreCase)) m_version = "v3"; @@ -363,7 +372,7 @@ namespace Duplicati.Library.Backend.OpenStack { case "v3": if (string.IsNullOrWhiteSpace(m_password)) - throw new UserInformationException(Strings.OpenStack.MissingOptionError(PASSWORD_OPTION), "OpenStackMissingPassword"); + throw new UserInformationException(Strings.OpenStack.MissingOptionError(AuthOptionsHelper.AuthPassword), "OpenStackMissingPassword"); if (string.IsNullOrWhiteSpace(m_domainName)) throw new UserInformationException(Strings.OpenStack.MissingOptionError(DOMAINNAME_OPTION), "OpenStackMissingDomainName"); if (string.IsNullOrWhiteSpace(m_tenantName)) @@ -374,7 +383,7 @@ namespace Duplicati.Library.Backend.OpenStack if (string.IsNullOrWhiteSpace(m_apikey)) { if (string.IsNullOrWhiteSpace(m_password)) - throw new UserInformationException(Strings.OpenStack.MissingOptionError(PASSWORD_OPTION), "OpenStackMissingPassword"); + throw new UserInformationException(Strings.OpenStack.MissingOptionError(AuthOptionsHelper.AuthPassword), "OpenStackMissingPassword"); if (string.IsNullOrWhiteSpace(m_tenantName)) throw new UserInformationException(Strings.OpenStack.MissingOptionError(TENANTNAME_OPTION), "OpenStackMissingTenantName"); } @@ -384,15 +393,12 @@ namespace Duplicati.Library.Backend.OpenStack m_helper = new WebHelper(this); } - protected virtual string AccessToken + protected async Task GetAccessToken(CancellationToken cancelToken) { - get - { - if (m_accessToken == null || (m_accessToken.expires.HasValue && (m_accessToken.expires.Value - DateTime.UtcNow).TotalSeconds < 30)) - GetAuthResponse(); + if (m_accessToken == null || (m_accessToken.expires.HasValue && (m_accessToken.expires.Value - DateTime.UtcNow).TotalSeconds < 30)) + await GetAuthResponse(cancelToken).ConfigureAwait(false); - return m_accessToken.id; - } + return m_accessToken!.id!; } private static string JoinUrls(string uri, string fragment) @@ -405,21 +411,19 @@ namespace Duplicati.Library.Backend.OpenStack return JoinUrls(JoinUrls(uri, fragment1), fragment2); } - private void GetAuthResponse() + private Task GetAuthResponse(CancellationToken cancellationToken) { switch (this.m_version) { case "v3": - GetKeystone3AuthResponse(); - break; + return GetKeystone3AuthResponse(cancellationToken); case "v2": default: - GetOpenstackAuthResponse(); - break; + return GetOpenstackAuthResponse(cancellationToken); } } - private Keystone3AuthResponse GetKeystone3AuthResponse() + private async Task GetKeystone3AuthResponse(CancellationToken cancellationToken) { var helper = new JSONWebHelper(); @@ -435,35 +439,50 @@ namespace Duplicati.Library.Backend.OpenStack req.ContentLength = data.Length; req.ContentType = "application/json; charset=UTF-8"; - using (var rs = req.GetRequestStream()) - rs.Write(data, 0, data.Length); - - WebResponse http_response = req.GetResponse(); - - Keystone3AuthResponse resp; - using (var reader = new StreamReader(http_response.GetResponseStream())) + await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancellationToken, async ct => { - resp = Newtonsoft.Json.JsonConvert.DeserializeObject( - reader.ReadToEnd()); - } + using (var rs = req.GetRequestStream()) + await rs.WriteAsync(data, 0, data.Length, ct); + }).ConfigureAwait(false); - string token = http_response.Headers["X-Subject-Token"]; - this.m_accessToken = new OpenStackAuthResponse.TokenClass(); - this.m_accessToken.id = token; - this.m_accessToken.expires = resp.token.expires_at; + + var http_response = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancellationToken, ct => req.GetResponse()).ConfigureAwait(false); + + var resp = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancellationToken, ct => + { + using (var reader = new StreamReader(http_response.GetResponseStream())) + return JsonConvert.DeserializeObject(reader.ReadToEnd()) + ?? throw new Exception("Failed to parse response"); + }).ConfigureAwait(false); + + if (resp.token == null) + throw new Exception("No token received"); + + var token = http_response.Headers["X-Subject-Token"]; + if (string.IsNullOrWhiteSpace(token)) + throw new Exception("No token received"); + + this.m_accessToken = new OpenStackAuthResponse.TokenClass() + { + id = token, + expires = resp.token.expires_at + }; // Grab the endpoint now that we have received it anyway - var fileservice = resp.token.catalog.FirstOrDefault(x => string.Equals(x.type, "object-store", StringComparison.OrdinalIgnoreCase)); + var fileservice = (resp.token.catalog ?? []).FirstOrDefault(x => string.Equals(x.type, "object-store", StringComparison.OrdinalIgnoreCase)); if (fileservice == null) throw new Exception("No object-store service found, is this service supported by the provider?"); + if (fileservice.endpoints == null || fileservice.endpoints.Length == 0) + throw new Exception("No endpoints found for object-store service"); + var endpoint = fileservice.endpoints.FirstOrDefault(x => (string.Equals(m_region, x.region) && string.Equals(x.interface_name, "public", StringComparison.OrdinalIgnoreCase))) ?? fileservice.endpoints.First(); m_simplestorageendpoint = endpoint.url; return resp; } - private OpenStackAuthResponse GetOpenstackAuthResponse() + private async Task GetOpenstackAuthResponse(CancellationToken cancellationToken) { var helper = new JSONWebHelper(); @@ -471,53 +490,60 @@ namespace Duplicati.Library.Backend.OpenStack req.Accept = "application/json"; req.Method = "POST"; - var resp = helper.ReadJSONResponse( + var resp = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancellationToken, ct => helper.ReadJSONResponse( req, new OpenStackAuthRequest(m_tenantName, m_username, m_password, m_apikey) - ); + )).ConfigureAwait(false); + + if (resp.access?.token == null) + throw new Exception("No token received"); m_accessToken = resp.access.token; // Grab the endpoint now that we have received it anyway - var fileservice = resp.access.serviceCatalog.FirstOrDefault(x => string.Equals(x.type, "object-store", StringComparison.OrdinalIgnoreCase)); + var fileservice = (resp.access.serviceCatalog ?? []).FirstOrDefault(x => string.Equals(x.type, "object-store", StringComparison.OrdinalIgnoreCase)); if (fileservice == null) throw new Exception("No object-store service found, is this service supported by the provider?"); - var endpoint = fileservice.endpoints.FirstOrDefault(x => string.Equals(m_region, x.region)) ?? fileservice.endpoints.First(); + var fileserviceendpoints = fileservice.endpoints ?? []; + var endpoint = fileserviceendpoints.FirstOrDefault(x => string.Equals(m_region, x.region)) + ?? fileserviceendpoints.FirstOrDefault(); + + if (endpoint == null) + throw new Exception("No endpoint found for object-store service"); m_simplestorageendpoint = endpoint.publicURL; return resp; } - protected virtual string SimpleStorageEndPoint + protected async Task GetSimpleStorageEndPoint(CancellationToken cancelToken) { - get - { - if (m_simplestorageendpoint == null) - GetAuthResponse(); + if (m_simplestorageendpoint == null) + await GetAuthResponse(cancelToken); - return m_simplestorageendpoint; - } + return m_simplestorageendpoint!; } #region IStreamingBackend implementation public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken) { - var url = JoinUrls(SimpleStorageEndPoint, m_container, Utility.Uri.UrlPathEncode(m_prefix + remotename)); - using (await m_helper.GetResponseAsync(url, cancelToken, stream, "PUT").ConfigureAwait(false)) + var url = JoinUrls(await GetSimpleStorageEndPoint(cancelToken).ConfigureAwait(false), m_container, Utility.Uri.UrlPathEncode(m_prefix + remotename)); + using (var ts = stream.ObserveReadTimeout(m_timeouts.ShortTimeout, false)) + using (await m_helper.GetResponseAsync(url, cancelToken, ts, "PUT").ConfigureAwait(false)) { } } public async Task GetAsync(string remotename, Stream stream, CancellationToken cancelToken) { - var url = JoinUrls(SimpleStorageEndPoint, m_container, Utility.Uri.UrlPathEncode(m_prefix + remotename)); + var url = JoinUrls(await GetSimpleStorageEndPoint(cancelToken).ConfigureAwait(false), m_container, Utility.Uri.UrlPathEncode(m_prefix + remotename)); try { - using (var resp = m_helper.GetResponse(url)) - using (var rs = AsyncHttpRequest.TrySetTimeout(resp.GetResponseStream())) - await Library.Utility.Utility.CopyStreamAsync(rs, stream, cancelToken).ConfigureAwait(false); + using (var resp = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => m_helper.GetResponse(url)).ConfigureAwait(false)) + using (var rs = AsyncHttpRequest.TrySetTimeout(await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => resp.GetResponseStream()).ConfigureAwait(false))) + using (var ts = rs.ObserveReadTimeout(m_timeouts.ShortTimeout)) + await Utility.Utility.CopyStreamAsync(ts, stream, cancelToken).ConfigureAwait(false); } catch (WebException wex) { @@ -549,9 +575,9 @@ namespace Duplicati.Library.Backend.OpenStack /// public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { - var plainurl = JoinUrls(SimpleStorageEndPoint, m_container) + string.Format("?format=json&delimiter=/&limit={0}", PAGE_LIMIT); + var plainurl = JoinUrls(await GetSimpleStorageEndPoint(cancelToken).ConfigureAwait(false), m_container) + string.Format("?format=json&delimiter=/&limit={0}", PAGE_LIMIT); if (!string.IsNullOrEmpty(m_prefix)) - plainurl += "&prefix=" + Library.Utility.Uri.UrlEncode(m_prefix); + plainurl += "&prefix=" + Utility.Uri.UrlEncode(m_prefix); var url = plainurl; @@ -560,10 +586,13 @@ namespace Duplicati.Library.Backend.OpenStack var req = m_helper.CreateRequest(url); req.Accept = "application/json"; - var items = await HandleListExceptions(() => m_helper.ReadJSONResponseAsync(req, cancelToken)).ConfigureAwait(false); + var items = await HandleListExceptions(async () => await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, ct => m_helper.ReadJSONResponseAsync(req, ct))).ConfigureAwait(false); foreach (var n in items) { var name = n.name; + if (string.IsNullOrWhiteSpace(name)) + continue; + if (name.StartsWith(m_prefix, StringComparison.Ordinal)) name = name.Substring(m_prefix.Length); @@ -594,34 +623,22 @@ namespace Duplicati.Library.Backend.OpenStack using (FileStream fs = File.Create(filename)) await GetAsync(remotename, fs, cancelToken).ConfigureAwait(false); } - public Task DeleteAsync(string remotename, CancellationToken cancelToken) + public async Task DeleteAsync(string remotename, CancellationToken cancelToken) { - var url = JoinUrls(SimpleStorageEndPoint, m_container, Library.Utility.Uri.UrlPathEncode(m_prefix + remotename)); - return m_helper.ReadJSONResponseAsync(url, cancelToken, null, "DELETE"); + var url = JoinUrls(await GetSimpleStorageEndPoint(cancelToken).ConfigureAwait(false), m_container, Library.Utility.Uri.UrlPathEncode(m_prefix + remotename)); + await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => m_helper.ReadJSONResponseAsync(url, ct, null, "DELETE")).ConfigureAwait(false); } public Task TestAsync(CancellationToken cancelToken) => this.TestListAsync(cancelToken); public async Task CreateFolderAsync(CancellationToken cancelToken) { - var url = JoinUrls(SimpleStorageEndPoint, m_container); - using (await m_helper.GetResponseAsync(url, cancelToken, null, "PUT").ConfigureAwait(false)) + var url = JoinUrls(await GetSimpleStorageEndPoint(cancelToken).ConfigureAwait(false), m_container); + using (await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => m_helper.GetResponseAsync(url, cancelToken, null, "PUT")).ConfigureAwait(false)) { } } - public string DisplayName - { - get - { - return Strings.OpenStack.DisplayName; - } - } - public string ProtocolKey - { - get - { - return "openstack"; - } - } + public string DisplayName => Strings.OpenStack.DisplayName; + public string ProtocolKey => "openstack"; public IList SupportedCommands { get @@ -630,30 +647,27 @@ namespace Duplicati.Library.Backend.OpenStack foreach (var s in KNOWN_OPENSTACK_PROVIDERS) authuris.AppendLine(string.Format("{0}: {1}", s.Key, s.Value)); - return new List(new[] { + return [ new CommandLineArgument(DOMAINNAME_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.DomainnameOptionShort, Strings.OpenStack.DomainnameOptionLong), - new CommandLineArgument(USERNAME_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.UsernameOptionShort, Strings.OpenStack.UsernameOptionLong), - new CommandLineArgument(PASSWORD_OPTION, CommandLineArgument.ArgumentType.Password, Strings.OpenStack.PasswordOptionShort, Strings.OpenStack.PasswordOptionLong(TENANTNAME_OPTION)), + new CommandLineArgument(AuthOptionsHelper.AuthUsername, CommandLineArgument.ArgumentType.String, Strings.OpenStack.UsernameOptionShort, Strings.OpenStack.UsernameOptionLong), + new CommandLineArgument(AuthOptionsHelper.AuthPassword, CommandLineArgument.ArgumentType.Password, Strings.OpenStack.PasswordOptionShort, Strings.OpenStack.PasswordOptionLong(TENANTNAME_OPTION)), new CommandLineArgument(TENANTNAME_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.TenantnameOptionShort, Strings.OpenStack.TenantnameOptionLong), new CommandLineArgument(APIKEY_OPTION, CommandLineArgument.ArgumentType.Password, Strings.OpenStack.ApikeyOptionShort, Strings.OpenStack.ApikeyOptionLong), new CommandLineArgument(AUTHURI_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.AuthuriOptionShort, Strings.OpenStack.AuthuriOptionLong(authuris.ToString())), new CommandLineArgument(VERSION_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.VersionOptionShort, Strings.OpenStack.VersionOptionLong), new CommandLineArgument(REGION_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.RegionOptionShort, Strings.OpenStack.RegionOptionLong), - }); - } - } - public string Description - { - get - { - return Strings.OpenStack.Description; + .. TimeoutOptionsHelper.GetOptions(), + ]; } } + public string Description => Strings.OpenStack.Description; public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new[] { new System.Uri(m_authUri).Host, string.IsNullOrWhiteSpace(m_simplestorageendpoint) ? null : new System.Uri(m_simplestorageendpoint).Host - }); + } + .WhereNotNullOrWhiteSpace() + .ToArray()); #endregion #region IDisposable implementation diff --git a/Duplicati/Library/Backend/OpenStack/Strings.cs b/Duplicati/Library/Backend/OpenStack/Strings.cs index 834a9d360..6abff1fff 100644 --- a/Duplicati/Library/Backend/OpenStack/Strings.cs +++ b/Duplicati/Library/Backend/OpenStack/Strings.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Localization.Short; @@ -44,6 +44,14 @@ namespace Duplicati.Library.Backend.Strings public static string VersionOptionShort { get { return LC.L(@"The keystone API version to use"); } } public static string RegionOptionLong { get { return LC.L(@"By default, the first reported endpoint will be used for file transfers. To select a specific region, provide the region name. If no such region is supported, the default (first reported) endpoint is used."); } } public static string RegionOptionShort { get { return LC.L(@"Supply the prefered region for endpoints"); } } + } + + internal static class SwiftConfig + { + public static string Description { get { return LC.L(@"Expose OpenStack configuration as a web module"); } } + public static string DisplayName { get { return LC.L(@"OpenStack configuration module"); } } + public static string ConfigTypeOptionLong { get { return LC.L(@"Provide different config values"); } } + public static string ConfigTypeOptionShort { get { return LC.L(@"The config to get"); } } } } From 9cecee04b942b8a038352ba7c4884c7695b7f024 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 21 Mar 2025 17:01:37 +0100 Subject: [PATCH 25/48] Updated pCloud --- .../Duplicati.Library.Backend.pCloud.csproj | 3 + .../Model/pCloudCreateFolderResponse.cs | 3 +- .../pCloud/Model/pCloudDeleteResponse.cs | 1 - .../pCloud/Model/pCloudDownloadResponse.cs | 4 +- .../Backend/pCloud/Model/pCloudErrorList.cs | 20 +- .../Backend/pCloud/Model/pCloudFileEntry.cs | 1 - .../pCloud/Model/pCloudFolderContent.cs | 1 - .../pCloud/Model/pCloudListFolderResponse.cs | 1 - .../pCloud/Model/pCloudListFoldersMetadata.cs | 2 - .../pCloud/Model/pCloudUploadResponse.cs | 2 - Duplicati/Library/Backend/pCloud/Strings.cs | 9 +- .../Library/Backend/pCloud/pCloudBackend.cs | 252 +++++++----------- .../Utility/Options/AuthIdOptionsHelper.cs | 8 +- 13 files changed, 123 insertions(+), 184 deletions(-) diff --git a/Duplicati/Library/Backend/pCloud/Duplicati.Library.Backend.pCloud.csproj b/Duplicati/Library/Backend/pCloud/Duplicati.Library.Backend.pCloud.csproj index 84812a427..b78f03557 100644 --- a/Duplicati/Library/Backend/pCloud/Duplicati.Library.Backend.pCloud.csproj +++ b/Duplicati/Library/Backend/pCloud/Duplicati.Library.Backend.pCloud.csproj @@ -4,9 +4,12 @@ net8.0 Library Copyright © 2025 Team Duplicati, MIT license + enable + enable + diff --git a/Duplicati/Library/Backend/pCloud/Model/pCloudCreateFolderResponse.cs b/Duplicati/Library/Backend/pCloud/Model/pCloudCreateFolderResponse.cs index a9f9bb704..fd9f2883d 100644 --- a/Duplicati/Library/Backend/pCloud/Model/pCloudCreateFolderResponse.cs +++ b/Duplicati/Library/Backend/pCloud/Model/pCloudCreateFolderResponse.cs @@ -19,7 +19,6 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -#nullable enable namespace Duplicati.Library.Backend.pCloud; /// @@ -42,5 +41,5 @@ internal record pCloudCreateFolderResponse /// /// Detailed metadata about the created or existing folder /// - public pCloudFolderMetadata? metadata { get; init; } + public pCloudFolderMetadata? metadata { get; init; } } \ No newline at end of file diff --git a/Duplicati/Library/Backend/pCloud/Model/pCloudDeleteResponse.cs b/Duplicati/Library/Backend/pCloud/Model/pCloudDeleteResponse.cs index c329e2ece..cbfd6994b 100644 --- a/Duplicati/Library/Backend/pCloud/Model/pCloudDeleteResponse.cs +++ b/Duplicati/Library/Backend/pCloud/Model/pCloudDeleteResponse.cs @@ -19,7 +19,6 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -#nullable enable namespace Duplicati.Library.Backend.pCloud; /// diff --git a/Duplicati/Library/Backend/pCloud/Model/pCloudDownloadResponse.cs b/Duplicati/Library/Backend/pCloud/Model/pCloudDownloadResponse.cs index df8b2f502..c0ad67927 100644 --- a/Duplicati/Library/Backend/pCloud/Model/pCloudDownloadResponse.cs +++ b/Duplicati/Library/Backend/pCloud/Model/pCloudDownloadResponse.cs @@ -19,8 +19,6 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System.Collections.Generic; - namespace Duplicati.Library.Backend.pCloud; /// @@ -44,5 +42,5 @@ internal record pCloudDownloadResponse /// Download path to be used with the host /// public string path { get; init; } = string.Empty; - + } \ No newline at end of file diff --git a/Duplicati/Library/Backend/pCloud/Model/pCloudErrorList.cs b/Duplicati/Library/Backend/pCloud/Model/pCloudErrorList.cs index 53b7cb281..c775ad989 100644 --- a/Duplicati/Library/Backend/pCloud/Model/pCloudErrorList.cs +++ b/Duplicati/Library/Backend/pCloud/Model/pCloudErrorList.cs @@ -19,18 +19,16 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System.Collections.Generic; - namespace Duplicati.Library.Backend.pCloud; - /// - /// Error codes/strings for pcloud. They will not be subject to translation so messages - /// can be googled to help users diagnose issues. - /// - /// - internal static class pCloudErrorList - { - internal static readonly Dictionary ErrorMessages = new() +/// +/// Error codes/strings for pcloud. They will not be subject to translation so messages +/// can be googled to help users diagnose issues. +/// +/// +internal static class pCloudErrorList +{ + internal static readonly Dictionary ErrorMessages = new() { { 1000, "Log in required." }, { 1001, "No full path or name/folderid provided." }, @@ -46,4 +44,4 @@ namespace Duplicati.Library.Backend.pCloud; { 4000, "Too many login tries from this IP address." }, { 5002, "Internal error, no servers available. Try again later." } }; - } \ No newline at end of file +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/pCloud/Model/pCloudFileEntry.cs b/Duplicati/Library/Backend/pCloud/Model/pCloudFileEntry.cs index 179c277c6..b280ed94e 100644 --- a/Duplicati/Library/Backend/pCloud/Model/pCloudFileEntry.cs +++ b/Duplicati/Library/Backend/pCloud/Model/pCloudFileEntry.cs @@ -19,7 +19,6 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System; using Duplicati.Library.Interface; namespace Duplicati.Library.Backend.pCloud; diff --git a/Duplicati/Library/Backend/pCloud/Model/pCloudFolderContent.cs b/Duplicati/Library/Backend/pCloud/Model/pCloudFolderContent.cs index 97ca3ac0e..1f08b584e 100644 --- a/Duplicati/Library/Backend/pCloud/Model/pCloudFolderContent.cs +++ b/Duplicati/Library/Backend/pCloud/Model/pCloudFolderContent.cs @@ -19,7 +19,6 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -#nullable enable namespace Duplicati.Library.Backend.pCloud; /// diff --git a/Duplicati/Library/Backend/pCloud/Model/pCloudListFolderResponse.cs b/Duplicati/Library/Backend/pCloud/Model/pCloudListFolderResponse.cs index a6249af53..91345ee8f 100644 --- a/Duplicati/Library/Backend/pCloud/Model/pCloudListFolderResponse.cs +++ b/Duplicati/Library/Backend/pCloud/Model/pCloudListFolderResponse.cs @@ -19,7 +19,6 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -#nullable enable namespace Duplicati.Library.Backend.pCloud; /// diff --git a/Duplicati/Library/Backend/pCloud/Model/pCloudListFoldersMetadata.cs b/Duplicati/Library/Backend/pCloud/Model/pCloudListFoldersMetadata.cs index dc1b6c741..4b74fbaea 100644 --- a/Duplicati/Library/Backend/pCloud/Model/pCloudListFoldersMetadata.cs +++ b/Duplicati/Library/Backend/pCloud/Model/pCloudListFoldersMetadata.cs @@ -19,8 +19,6 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System.Collections.Generic; - namespace Duplicati.Library.Backend.pCloud; /// diff --git a/Duplicati/Library/Backend/pCloud/Model/pCloudUploadResponse.cs b/Duplicati/Library/Backend/pCloud/Model/pCloudUploadResponse.cs index f79e23e19..ead719b5b 100644 --- a/Duplicati/Library/Backend/pCloud/Model/pCloudUploadResponse.cs +++ b/Duplicati/Library/Backend/pCloud/Model/pCloudUploadResponse.cs @@ -19,8 +19,6 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System.Collections.Generic; - namespace Duplicati.Library.Backend.pCloud; /// diff --git a/Duplicati/Library/Backend/pCloud/Strings.cs b/Duplicati/Library/Backend/pCloud/Strings.cs index 9ed6ac0c0..aeffffe54 100644 --- a/Duplicati/Library/Backend/pCloud/Strings.cs +++ b/Duplicati/Library/Backend/pCloud/Strings.cs @@ -21,12 +21,13 @@ using Duplicati.Library.Localization.Short; -namespace Duplicati.Library.Backend.Strings { - internal static class pCloudBackend { - +namespace Duplicati.Library.Backend.Strings +{ + internal static class pCloudBackend + { public static string Description => LC.L(@"This backend can read and write data to pCloud with native API. Allowed format is ""pcloud://api.pcloud.com""."); public static string DisplayName => LC.L(@"pCloud (Native API)"); - public static string AuthPasswordDescriptionLong => LC.L(@"The oAuth token used to connect to the server. This may also be supplied as the environment variable ""AUTHID""."); + public static string AuthPasswordDescriptionLong(string tokenurl) => LC.L(@"The oAuth token used to connect to the server. This may also be supplied as the environment variable ""AUTHID"". Visit {0} to get a new one", tokenurl); public static string AuthPasswordDescriptionShort => LC.L(@"Supply the oAuth token used to connect to the server"); public static string NoServerSpecified => LC.L(@"No server specified, must be either api.pcloud.com or eapi.pcloud.com for European hosting"); public static string InvalidServerSpecified => LC.L(@"Invalid server specified, must be either api.pcloud.com or eapi.pcloud.com for European hosting"); diff --git a/Duplicati/Library/Backend/pCloud/pCloudBackend.cs b/Duplicati/Library/Backend/pCloud/pCloudBackend.cs index a7a0ee75f..9b208f03a 100644 --- a/Duplicati/Library/Backend/pCloud/pCloudBackend.cs +++ b/Duplicati/Library/Backend/pCloud/pCloudBackend.cs @@ -19,21 +19,15 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System; using Duplicati.Library.Interface; -using System.Collections.Generic; -using System.IO; -using System.Linq; using System.Net; -using System.Net.Http; using System.Net.Http.Headers; using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; using Duplicati.Library.Backend.pCloud; using Duplicati.Library.Utility; using Uri = System.Uri; using System.Runtime.CompilerServices; +using Duplicati.Library.Utility.Options; namespace Duplicati.Library.Backend; @@ -42,6 +36,10 @@ namespace Duplicati.Library.Backend; /// public class pCloudBackend : IStreamingBackend { + /// + /// The URL used to get a new token + /// + private static readonly string TOKEN_URL = OAuthHelper.OAUTH_LOGIN_URL("pcloud"); /// /// Implementation of interface property for the backend key /// @@ -57,20 +55,10 @@ public class pCloudBackend : IStreamingBackend /// public string Description => Strings.pCloudBackend.Description; - /// - /// The default timeout in seconds for PUT/GET file operations - /// - private const int LONG_OPERATION_TIMEOUT_SECONDS = 30000; - - /// - /// The default timeout in seconds for LIST/CreateFolder operations - /// - private const int SHORT_OPERATION_TIMEOUT_SECONDS = 30; - /// /// The server URL to be used (pcloud uses 2 different endpoints depending if its an european or non european hosting) /// - private string _ServerUrl; + private readonly string _ServerUrl; /// /// Bearer token to be using in the API @@ -80,12 +68,22 @@ public class pCloudBackend : IStreamingBackend /// /// Remote path/folder to use used in the backend /// - private string _Path; + private readonly string _Path; /// /// Hostname only (no ports or paths) to be used on DNS resolutions. /// - private string _DnsName; + private readonly string _DnsName; + + /// + /// The timeouts to be used in the backend + /// + private readonly TimeoutOptionsHelper.Timeouts _Timeouts; + + /// + /// HttpClient to be used for requests + /// + private readonly HttpClient _HttpClient; /// /// Variable being used to cache the folder ID, as it is required to upload files @@ -94,11 +92,6 @@ public class pCloudBackend : IStreamingBackend /// private ulong? _CachedFolderID; - /// - /// Name of the authentication parameter/option - /// - private const string AUTHENTICATION_OPTION = "authid"; - /// /// Path separators (both Windows \ and unix /) to be used in path manipulation /// @@ -118,6 +111,12 @@ public class pCloudBackend : IStreamingBackend /// public pCloudBackend() { + _ServerUrl = null!; + _Path = null!; + _DnsName = null!; + _Token = null!; + _Timeouts = null!; + _HttpClient = null!; } /// @@ -125,14 +124,15 @@ public class pCloudBackend : IStreamingBackend /// /// URL in Duplicati Uri format /// options to be used in the backend - public pCloudBackend(string url, Dictionary options) + public pCloudBackend(string url, Dictionary options) { var uri = new Utility.Uri(url); uri.RequireHost(); _DnsName = uri.Host; - if (options.TryGetValue(AUTHENTICATION_OPTION, out var option)) - _Token = option; + _Token = AuthIdOptionsHelper.Parse(options) + .RequireCredentials(TOKEN_URL) + .AuthId!; if (!PCLOUD_SERVERS.ContainsValue(uri.Host)) throw new UserInformationException(Strings.pCloudBackend.InvalidServerSpecified, @@ -144,24 +144,29 @@ public class pCloudBackend : IStreamingBackend // Ensure that the path is in the correct format, without starting or tailing slashes _Path = uri.Path.TrimStart(PATH_SEPARATORS).TrimEnd(PATH_SEPARATORS).Trim(); _ServerUrl = uri.Host; + _Timeouts = TimeoutOptionsHelper.Parse(options); + + var httpHandler = new HttpClientHandler(); + _HttpClient = HttpClientHelper.CreateClient(httpHandler); + _HttpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( + "Bearer", + _Token + ); + + // Set the timeout to infinite, all methods are called with cancelationTokens. + _HttpClient.Timeout = Timeout.InfiniteTimeSpan; + } /// /// Implementation of interface property to return supported command parameters /// - public IList SupportedCommands - { - get - { - return new List(new ICommandLineArgument[] - { - new CommandLineArgument(AUTHENTICATION_OPTION, - CommandLineArgument.ArgumentType.Password, - Strings.pCloudBackend.AuthPasswordDescriptionShort, - Strings.pCloudBackend.AuthPasswordDescriptionLong), - }); - } - } + public IList SupportedCommands => [ + new CommandLineArgument(AuthIdOptionsHelper.AuthId, + CommandLineArgument.ArgumentType.Password, + Strings.pCloudBackend.AuthPasswordDescriptionShort, + Strings.pCloudBackend.AuthPasswordDescriptionLong(TOKEN_URL)), + ]; /// /// Implementation of interface method for listing remote folder contents. @@ -195,20 +200,20 @@ public class pCloudBackend : IStreamingBackend /// private async Task> ListWithMetadata(ulong folderId, CancellationToken cancellationToken) { - using var timeoutToken = new CancellationTokenSource(); - timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS)); - using var combinedTokens = - CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken); + using var request = CreateRequest($"/listfolder?folderid={folderId}", HttpMethod.Get); - using var requestResources = CreateRequest($"/listfolder?folderid={folderId}", HttpMethod.Get); - - using var response = await requestResources.HttpClient - .SendAsync(requestResources.RequestMessage, HttpCompletionOption.ResponseContentRead, - timeoutToken.Token).ConfigureAwait(false); + using var response = await Utility.Utility.WithTimeout(_Timeouts.ListTimeout, cancellationToken, + ct => _HttpClient + .SendAsync(request, HttpCompletionOption.ResponseContentRead, + ct)).ConfigureAwait(false); response.EnsureSuccessStatusCode(); - var content = await response.Content.ReadAsStringAsync(timeoutToken.Token).ConfigureAwait(false); - var listFolderResponse = JsonSerializer.Deserialize(content); + var content = await Utility.Utility.WithTimeout(_Timeouts.ListTimeout, cancellationToken, + ct => response.Content.ReadAsStringAsync(ct) + ).ConfigureAwait(false); + + var listFolderResponse = JsonSerializer.Deserialize(content) + ?? throw new Exception("Failed to deserialize list folder response"); if (pCloudErrorList.ErrorMessages.TryGetValue(listFolderResponse.result, out var message)) throw new Exception(message); @@ -257,28 +262,22 @@ public class pCloudBackend : IStreamingBackend { _CachedFolderID ??= await GetFolderId(cancellationToken).ConfigureAwait(false); - using var timeoutToken = new CancellationTokenSource(); - timeoutToken.CancelAfter(TimeSpan.FromSeconds(LONG_OPERATION_TIMEOUT_SECONDS)); - using var combinedTokens = - CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken); - var encodedPath = Uri.EscapeDataString(remotename); - using var requestResources = + using var request = CreateRequest($"/uploadfile?folderid={_CachedFolderID}&filename={encodedPath}&nopartial=1", HttpMethod.Post); - - requestResources.RequestMessage.Content = new StreamContent(input); - requestResources.RequestMessage.Content.Headers.ContentLength = input.Length; - requestResources.RequestMessage.Content.Headers.ContentType = + using var timeoutStream = input.ObserveReadTimeout(_Timeouts.ReadWriteTimeout, false); + request.Content = new StreamContent(timeoutStream); + request.Content.Headers.ContentLength = timeoutStream.Length; + request.Content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); - using var response = await requestResources.HttpClient.SendAsync( - requestResources.RequestMessage, - HttpCompletionOption.ResponseContentRead, combinedTokens.Token).ConfigureAwait(false); + using var response = await _HttpClient.SendAsync(request, HttpCompletionOption.ResponseContentRead, cancellationToken).ConfigureAwait(false); - var content = await response.Content.ReadAsStringAsync(combinedTokens.Token).ConfigureAwait(false); - var uploadResponse = JsonSerializer.Deserialize(content); + var content = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + var uploadResponse = JsonSerializer.Deserialize(content) + ?? throw new Exception("Failed to deserialize upload response"); if (pCloudErrorList.ErrorMessages.TryGetValue(uploadResponse.result, out var message)) throw new Exception(message); @@ -314,22 +313,18 @@ public class pCloudBackend : IStreamingBackend /// Exceptions arising from either code execution private async Task GetFileLink(string filename, CancellationToken cancellationToken) { - using var timeoutToken = new CancellationTokenSource(); - timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS)); - using var combinedTokens = - CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken); + using var request = CreateRequest($"/getfilelink?fileid={await GetFileId(filename, cancellationToken).ConfigureAwait(false)}", HttpMethod.Get); - using var requestResources = CreateRequest($"/getfilelink?fileid={await GetFileId(filename, cancellationToken).ConfigureAwait(false)}", HttpMethod.Get); - - using var response = await requestResources.HttpClient.SendAsync( - requestResources.RequestMessage, - HttpCompletionOption.ResponseContentRead, combinedTokens.Token).ConfigureAwait(false); + using var response = await Utility.Utility.WithTimeout(_Timeouts.ShortTimeout, cancellationToken, + ct => _HttpClient.SendAsync(request, HttpCompletionOption.ResponseContentRead, ct) + ).ConfigureAwait(false); if (!response.IsSuccessStatusCode) throw new Exception($"Failed to get download link. Status: {response.StatusCode}"); - var content = await response.Content.ReadAsStringAsync(combinedTokens.Token).ConfigureAwait(false); - var getFileIdResponse = JsonSerializer.Deserialize(content); + var content = await Utility.Utility.WithTimeout(_Timeouts.ShortTimeout, cancellationToken, ct => response.Content.ReadAsStringAsync(ct)).ConfigureAwait(false); + var getFileIdResponse = JsonSerializer.Deserialize(content) + ?? throw new Exception("Failed to deserialize getfilelink response"); if (pCloudErrorList.ErrorMessages.TryGetValue(getFileIdResponse.result, out var message)) throw new FileMissingException(message); @@ -353,17 +348,12 @@ public class pCloudBackend : IStreamingBackend { try { - using var timeoutToken = new CancellationTokenSource(); - timeoutToken.CancelAfter(TimeSpan.FromSeconds(LONG_OPERATION_TIMEOUT_SECONDS)); - using var combinedTokens = - CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken); + using var request = CreateRequest(string.Empty, HttpMethod.Get); + request.RequestUri = new Uri(await GetFileLink(remotename, cancellationToken).ConfigureAwait(false)); - using var requestResources = CreateRequest(string.Empty, HttpMethod.Get); - - requestResources.RequestMessage.RequestUri = new Uri(await GetFileLink(remotename, cancellationToken).ConfigureAwait(false)); - - await requestResources.HttpClient.DownloadFile(requestResources.RequestMessage, output, null, - combinedTokens.Token).ConfigureAwait(false); + using var timeoutStream = output.ObserveWriteTimeout(_Timeouts.ReadWriteTimeout, false); + await _HttpClient.DownloadFile(request, timeoutStream, null, + cancellationToken).ConfigureAwait(false); } catch (HttpRequestException wex) { @@ -396,21 +386,19 @@ public class pCloudBackend : IStreamingBackend /// Exceptions arising from either code execution or business logic when return code from pcloud indicates an error. public async Task DeleteAsync(string remotename, CancellationToken cancellationToken) { - using var timeoutToken = new CancellationTokenSource(); - timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS)); - using var combinedTokens = - CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken); + using var request = CreateRequest($"/deletefile?fileid={await GetFileId(remotename, cancellationToken).ConfigureAwait(false)}", HttpMethod.Get); - using var requestResources = CreateRequest($"/deletefile?fileid={await GetFileId(remotename, cancellationToken).ConfigureAwait(false)}", HttpMethod.Get); - - using var response = await requestResources.HttpClient.SendAsync( - requestResources.RequestMessage, - HttpCompletionOption.ResponseContentRead, combinedTokens.Token).ConfigureAwait(false); + using var response = await Utility.Utility.WithTimeout(_Timeouts.ShortTimeout, cancellationToken, + ct => _HttpClient.SendAsync(request, HttpCompletionOption.ResponseContentRead, ct) + ).ConfigureAwait(false); response.EnsureSuccessStatusCode(); - var content = await response.Content.ReadAsStringAsync(combinedTokens.Token).ConfigureAwait(false); - var deleteFileResponse = JsonSerializer.Deserialize(content); + var content = await Utility.Utility.WithTimeout(_Timeouts.ShortTimeout, cancellationToken, + ct => response.Content.ReadAsStringAsync(ct) + ).ConfigureAwait(false); + var deleteFileResponse = JsonSerializer.Deserialize(content) + ?? throw new Exception("Failed to deserialize delete file response"); // If no error code is matched, result was == 0 so it successfully created the folder if (deleteFileResponse.result == 2009) @@ -422,7 +410,6 @@ public class pCloudBackend : IStreamingBackend if (deleteFileResponse.result != 0) throw new Exception( Strings.pCloudBackend.FailedWithUnexpectedErrorCode("delete", deleteFileResponse.result)); - } /// @@ -477,21 +464,19 @@ public class pCloudBackend : IStreamingBackend /// The folderID of the newly created folder private async Task CreateFolder(CancellationToken cancellationToken, ulong parentFolderId, string folderName) { - using var timeoutToken = new CancellationTokenSource(); - timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS)); - using var combinedTokens = - CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken); + using var request = CreateRequest($"/createfolderifnotexists?folderid={parentFolderId}&name={folderName}", HttpMethod.Get); - using var requestResources = CreateRequest($"/createfolderifnotexists?folderid={parentFolderId}&name={folderName}", HttpMethod.Get); - - using var response = await requestResources.HttpClient.SendAsync( - requestResources.RequestMessage, - HttpCompletionOption.ResponseContentRead, combinedTokens.Token).ConfigureAwait(false); + using var response = await Utility.Utility.WithTimeout(_Timeouts.ShortTimeout, cancellationToken, + ct => _HttpClient.SendAsync(request, HttpCompletionOption.ResponseContentRead, ct) + ).ConfigureAwait(false); response.EnsureSuccessStatusCode(); - var content = await response.Content.ReadAsStringAsync(combinedTokens.Token).ConfigureAwait(false); - var createFolderResponse = JsonSerializer.Deserialize(content); + var content = await Utility.Utility.WithTimeout(_Timeouts.ShortTimeout, cancellationToken, + ct => response.Content.ReadAsStringAsync(ct) + ).ConfigureAwait(false); + var createFolderResponse = JsonSerializer.Deserialize(content) + ?? throw new Exception("Failed to deserialize create folder response"); if (pCloudErrorList.ErrorMessages.TryGetValue(createFolderResponse.result, out var message)) throw new Exception(message); @@ -544,55 +529,14 @@ public class pCloudBackend : IStreamingBackend return currentFolderId; } - /// - /// Wrapper for the tupple of HttpClient and HttpRequestMessage used in web requests. - /// - /// The HTTPClient - /// The HttpRequestMessage object - private record RequestResources(HttpClient HttpClient, HttpRequestMessage RequestMessage) : IDisposable - { - public void Dispose() - { - try - { - RequestMessage?.Dispose(); - } - catch - { - // ignored - } - - try - { - HttpClient?.Dispose(); - } - catch - { - // ignored - } - } - } - /// /// Helper method to create request resources with the bearer token and default headers /// /// url to be appended after host /// Http Method /// - private RequestResources CreateRequest(string url, HttpMethod method = null) + private HttpRequestMessage CreateRequest(string url, HttpMethod? method = null) { - HttpClient httpClient; - HttpClientHandler httpHandler = new HttpClientHandler(); - - httpClient = HttpClientHelper.CreateClient(httpHandler); - httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( - "Bearer", - _Token - ); - - // Set the timeout to infinite, all methods are called with cancelationTokens. - httpClient.Timeout = Timeout.InfiniteTimeSpan; - var request = new HttpRequestMessage(HttpMethod.Get, $"https://{_ServerUrl}/{url}"); request.Headers.Add(HttpRequestHeader.UserAgent.ToString(), "Duplicati pCloud Client v" + System.Reflection.Assembly.GetExecutingAssembly().GetName().Version); @@ -600,7 +544,7 @@ public class pCloudBackend : IStreamingBackend if (method != null) request.Method = method; - return new RequestResources(httpClient, request); + return request; } /// diff --git a/Duplicati/Library/Utility/Options/AuthIdOptionsHelper.cs b/Duplicati/Library/Utility/Options/AuthIdOptionsHelper.cs index ff0b1c42d..c2126a0d9 100644 --- a/Duplicati/Library/Utility/Options/AuthIdOptionsHelper.cs +++ b/Duplicati/Library/Utility/Options/AuthIdOptionsHelper.cs @@ -32,6 +32,10 @@ namespace Duplicati.Library.Utility.Options; /// public static class AuthIdOptionsHelper { + /// + /// The authentication ID option, without a prefix + /// + public const string AuthId = "authid"; /// /// Gets the authentication options /// @@ -40,7 +44,7 @@ public static class AuthIdOptionsHelper /// The authentication options public static CommandLineArgument[] GetOptions(string tokenurl, string? prefix = null) => [ - new CommandLineArgument($"{prefix}authid", CommandLineArgument.ArgumentType.String, Strings.AuthIdSettingsHelper.AuthidShort, Strings.AuthIdSettingsHelper.AuthidLong(tokenurl)) + new CommandLineArgument($"{prefix}{AuthId}", CommandLineArgument.ArgumentType.String, Strings.AuthIdSettingsHelper.AuthidShort, Strings.AuthIdSettingsHelper.AuthidLong(tokenurl)) ]; /// @@ -50,7 +54,7 @@ public static class AuthIdOptionsHelper /// An optional prefix for the options /// The parsed authentication options public static AuthIdOptions Parse(IReadOnlyDictionary options, string? prefix = null) - => new AuthIdOptions(options.GetValueOrDefault($"{prefix}authid")); + => new AuthIdOptions(options.GetValueOrDefault($"{prefix}{AuthId}")); /// /// Structure to hold the authentication options From 769f68769378c1c02f0e47b4e0dc867cdbe5402a Mon Sep 17 00:00:00 2001 From: "Marcelo C." Date: Fri, 21 Mar 2025 13:29:29 -0300 Subject: [PATCH 26/48] Converting AccessToken helper method to be fully async --- Duplicati/Library/Backend/Box/BoxBackend.cs | 15 ++++++--- .../OAuthHelper/OAuthHelperHttpClient.cs | 31 +++++++++---------- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/Duplicati/Library/Backend/Box/BoxBackend.cs b/Duplicati/Library/Backend/Box/BoxBackend.cs index 7ed4d77a7..37aefd83d 100644 --- a/Duplicati/Library/Backend/Box/BoxBackend.cs +++ b/Duplicati/Library/Backend/Box/BoxBackend.cs @@ -32,6 +32,7 @@ using System.Threading.Tasks; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using Duplicati.Library.Logging; +using Duplicati.Library.Utility; using Newtonsoft.Json; using Uri = Duplicati.Library.Utility.Uri; @@ -65,16 +66,22 @@ namespace Duplicati.Library.Backend.Box } public override void AttemptParseAndThrowException(Exception ex, HttpResponseMessage responseContext = null) + { + AttemptParseAndThrowExceptionAsync(ex, responseContext).Await(); + } + + public override async Task AttemptParseAndThrowExceptionAsync(Exception ex, HttpResponseMessage responseContext = null, + CancellationToken cancellationToken = default) { if (ex is not HttpRequestException || responseContext == null) return; - - if (ex is HttpRequestException && responseContext is { StatusCode: HttpStatusCode.TooManyRequests }) + + if (responseContext is { StatusCode: HttpStatusCode.TooManyRequests }) throw new TooManyRequestException(responseContext.Headers.RetryAfter); - using var stream = responseContext.Content.ReadAsStream(); + await using var stream = await responseContext.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); using var reader = new StreamReader(stream); - var rawData = reader.ReadToEnd(); + var rawData = await reader.ReadToEndAsync(cancellationToken).ConfigureAwait(false); var errorResponse = JsonConvert.DeserializeObject(rawData); throw new UserInformationException($"Box.com ErrorResponse: {errorResponse.Status} - {errorResponse.Code}: {errorResponse.Message}", "box.com"); } diff --git a/Duplicati/Library/Backend/OAuthHelper/OAuthHelperHttpClient.cs b/Duplicati/Library/Backend/OAuthHelper/OAuthHelperHttpClient.cs index 83bf40618..8f35628f0 100644 --- a/Duplicati/Library/Backend/OAuthHelper/OAuthHelperHttpClient.cs +++ b/Duplicati/Library/Backend/OAuthHelper/OAuthHelperHttpClient.cs @@ -25,6 +25,7 @@ using System.Net; using System.Net.Http; using System.Net.Http.Headers; using System.Threading; +using System.Threading.Tasks; using Duplicati.Library.Utility; namespace Duplicati.Library; @@ -83,10 +84,12 @@ public class OAuthHelperHttpClient : JsonWebHelperHttpClient Strings.OAuthHelper.MissingAuthID(OAuthLoginUrl), "MissingAuthID"); } - private HttpRequestMessage CreateRequest(string url, string method, bool noAuthorization) + private HttpRequestMessage CreateRequest(string url, string method, bool noAuthorization, CancellationToken cancellationToken = default) { - var request = base.CreateRequest(url, method); - if (!noAuthorization && AutoAuthHeader && !string.Equals(OAuthContextSettings.ServerURL, url)) request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", AccessToken); + var request = new HttpRequestMessage(string.IsNullOrEmpty(method) ? HttpMethod.Get : new HttpMethod(method), url); + request.Headers.Add("User-Agent", UserAgent); + if (!noAuthorization && AutoAuthHeader && !string.Equals(OAuthContextSettings.ServerURL, url)) request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", + GetAccessTokenAsync(cancellationToken).Await()); return request; } @@ -94,11 +97,9 @@ public class OAuthHelperHttpClient : JsonWebHelperHttpClient { return CreateRequest(url, method, false); } - - private string AccessToken + + private async Task GetAccessTokenAsync(CancellationToken cancellationToken) { - get - { if (AccessTokenOnly) return _Authid; @@ -111,17 +112,17 @@ public class OAuthHelperHttpClient : JsonWebHelperHttpClient HttpResponseMessage response = null; try { - using var request = base.CreateRequest(OAuthContextSettings.ServerURL); - - request.Headers.TryAddWithoutValidation("X-AuthID", _Authid); - using var timeoutToken = new CancellationTokenSource(); timeoutToken.CancelAfter(TimeSpan.FromSeconds(AUTHENTICATION_TIMEOUT_SECONDS)); + using var combinedCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken); + + using var request = CreateRequest(OAuthContextSettings.ServerURL, "GET", false, combinedCancellationToken.Token); - response = _httpClient.Send(request, timeoutToken.Token); + request.Headers.TryAddWithoutValidation("X-AuthID", _Authid); + response = await _httpClient.SendAsync(request, combinedCancellationToken.Token).ConfigureAwait(false); response.EnsureSuccessStatusCode(); - var res = ReadJsonResponse(response); + var res = await ReadJsonResponseAsync(response, combinedCancellationToken.Token).ConfigureAwait(false); _mTokenExpires = DateTime.UtcNow.AddSeconds(res.expires - 30); if (AutoV2 && !string.IsNullOrWhiteSpace(res.v2_authid)) @@ -153,7 +154,6 @@ public class OAuthHelperHttpClient : JsonWebHelperHttpClient if (string.IsNullOrWhiteSpace(msg)) msg = response.StatusCode.ToString(); - //TODO: Test; this was ported from old code but looks fragile. if (response.StatusCode == HttpStatusCode.ServiceUnavailable) { string errorKey = msg == response.StatusCode.ToString() @@ -171,7 +171,7 @@ public class OAuthHelperHttpClient : JsonWebHelperHttpClient if (retries >= (clientError ? 1 : MAX_AUTHORIZATION_RETRIES)) { - AttemptParseAndThrowException(ex, response); + await AttemptParseAndThrowExceptionAsync(ex, response, cancellationToken).ConfigureAwait(false); throw; } @@ -187,6 +187,5 @@ public class OAuthHelperHttpClient : JsonWebHelperHttpClient } return _Token; - } } } \ No newline at end of file From 9821fa79354bc3f6a5c50685a0cf6ed93b0c315d Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sat, 22 Mar 2025 07:49:29 +0100 Subject: [PATCH 27/48] Fixed casing and file rename --- .../CIFS/{CIFSTests.cs => SMBTests.cs} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename LiveTests/Duplicati.Backend.Tests/CIFS/{CIFSTests.cs => SMBTests.cs} (96%) diff --git a/LiveTests/Duplicati.Backend.Tests/CIFS/CIFSTests.cs b/LiveTests/Duplicati.Backend.Tests/CIFS/SMBTests.cs similarity index 96% rename from LiveTests/Duplicati.Backend.Tests/CIFS/CIFSTests.cs rename to LiveTests/Duplicati.Backend.Tests/CIFS/SMBTests.cs index 375435cad..400370764 100644 --- a/LiveTests/Duplicati.Backend.Tests/CIFS/CIFSTests.cs +++ b/LiveTests/Duplicati.Backend.Tests/CIFS/SMBTests.cs @@ -31,12 +31,12 @@ public sealed class SMBTests : BaseSftpgoTest { /// - /// Test SMB with TestContainers creating a Samba Server with TestContainers. + /// Test Smb with TestContainers creating a Samba Server with TestContainers. /// /// This test has no requirement of environment variables. /// [TestMethod] - public async Task TestSMB() + public async Task TestSmb() { var outputConsumer = new OutputConsumer(); var randomPassword = GeneratePassword(); @@ -92,7 +92,7 @@ smbd --foreground --no-process-group --debug-stdout"; var exitCode = CommandLine.BackendTester.Program.Main( new[] { - $"sMB://localhost/testshare1/new/?transport=directtcp&auth-domain&auth-username=smbuser1&auth-password={randomPassword}", + $"smb://localhost/testshare1/new/?transport=directtcp&auth-domain&auth-username=smbuser1&auth-password={randomPassword}", }.Concat(Parameters.GlobalTestParameters).ToArray()); Console.WriteLine(await outputConsumer.GetStreamsOutput()); From fc927b881fb085861cf86119b3870715b9e003eb Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sat, 22 Mar 2025 08:08:54 +0100 Subject: [PATCH 28/48] Updated rclone backend --- .../Duplicati.Library.Backend.Rclone.csproj | 2 + Duplicati/Library/Backend/Rclone/Rclone.cs | 133 +++++++++--------- .../Utility/Options/TimeoutOptionsHelper.cs | 25 +++- 3 files changed, 87 insertions(+), 73 deletions(-) diff --git a/Duplicati/Library/Backend/Rclone/Duplicati.Library.Backend.Rclone.csproj b/Duplicati/Library/Backend/Rclone/Duplicati.Library.Backend.Rclone.csproj index 6560931e8..4e9b541f8 100644 --- a/Duplicati/Library/Backend/Rclone/Duplicati.Library.Backend.Rclone.csproj +++ b/Duplicati/Library/Backend/Rclone/Duplicati.Library.Backend.Rclone.csproj @@ -4,6 +4,8 @@ net8.0 Library Copyright © 2025 Team Duplicati, MIT license + enable + enable diff --git a/Duplicati/Library/Backend/Rclone/Rclone.cs b/Duplicati/Library/Backend/Rclone/Rclone.cs index 7bdd2f8c2..e05081dbd 100644 --- a/Duplicati/Library/Backend/Rclone/Rclone.cs +++ b/Duplicati/Library/Backend/Rclone/Rclone.cs @@ -21,16 +21,12 @@ using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; +using Duplicati.Library.Utility.Options; using Newtonsoft.Json; using Newtonsoft.Json.Linq; -using System; -using System.Collections.Generic; using System.Diagnostics; -using System.IO; using System.Runtime.CompilerServices; using System.Text; -using System.Threading; -using System.Threading.Tasks; namespace Duplicati.Library.Backend { @@ -51,22 +47,28 @@ namespace Duplicati.Library.Backend private readonly string remote_path; private readonly string opt_rclone; private readonly string rclone_executable; + private readonly TimeoutOptionsHelper.Timeouts timeouts; public Rclone() { - + local_repo = null!; + remote_repo = null!; + remote_path = null!; + opt_rclone = null!; + rclone_executable = null!; + timeouts = null!; } - public Rclone(string url, Dictionary options) + public Rclone(string url, Dictionary options) { var uri = new Utility.Uri(url); /*should check here if program is installed */ - local_repo = options.GetValueOrDefault(OPTION_LOCAL_REPO, local_repo); - remote_repo = options.GetValueOrDefault(OPTION_REMOTE_REPO, remote_repo); - remote_path = options.GetValueOrDefault(OPTION_REMOTE_PATH, remote_path); - opt_rclone = options.GetValueOrDefault(OPTION_RCLONE, opt_rclone) ?? ""; - rclone_executable = options.GetValueOrDefault(OPTION_RCLONE_EXECUTABLE, rclone_executable); + local_repo = options.GetValueOrDefault(OPTION_LOCAL_REPO) ?? ""; + remote_repo = options.GetValueOrDefault(OPTION_REMOTE_REPO) ?? ""; + remote_path = options.GetValueOrDefault(OPTION_REMOTE_PATH) ?? ""; + opt_rclone = options.GetValueOrDefault(OPTION_RCLONE) ?? ""; + rclone_executable = options.GetValueOrDefault(OPTION_RCLONE_EXECUTABLE) ?? ""; if (string.IsNullOrWhiteSpace(local_repo)) local_repo = "local"; @@ -77,6 +79,8 @@ namespace Duplicati.Library.Backend if (string.IsNullOrWhiteSpace(rclone_executable)) rclone_executable = "rclone"; + timeouts = TimeoutOptionsHelper.Parse(options); + #if DEBUG Console.WriteLine("Constructor {0}: {1}:{2} {3}", local_repo, remote_repo, remote_path, opt_rclone); #endif @@ -84,17 +88,11 @@ namespace Duplicati.Library.Backend #region IBackendInterface Members - public string DisplayName - { - get { return Strings.Rclone.DisplayName; } - } + public string DisplayName => Strings.Rclone.DisplayName; - public string ProtocolKey - { - get { return "rclone"; } - } + public string ProtocolKey => "rclone"; - private async Task RcloneCommandExecuter(String command, String arguments, CancellationToken cancelToken) + private async Task RcloneCommandExecuter(string command, string arguments, TimeSpan timeout, CancellationToken cancelToken) { StringBuilder outputBuilder = new StringBuilder(); StringBuilder errorBuilder = new StringBuilder(); @@ -122,11 +120,11 @@ namespace Duplicati.Library.Backend EnableRaisingEvents = true }; // attach the event handler for OutputDataReceived before starting the process - process.OutputDataReceived += new System.Diagnostics.DataReceivedEventHandler + process.OutputDataReceived += new DataReceivedEventHandler ( - delegate (object sender, System.Diagnostics.DataReceivedEventArgs e) + delegate (object sender, DataReceivedEventArgs e) { - if (!String.IsNullOrEmpty(e.Data)) + if (!string.IsNullOrEmpty(e.Data)) { #if DEBUG // Console.Error.WriteLine(String.Format("output {0}", e.Data)); @@ -137,12 +135,12 @@ namespace Duplicati.Library.Backend } ); - process.ErrorDataReceived += new System.Diagnostics.DataReceivedEventHandler + process.ErrorDataReceived += new DataReceivedEventHandler ( - delegate (object sender, System.Diagnostics.DataReceivedEventArgs e) + delegate (object sender, DataReceivedEventArgs e) { - if (!String.IsNullOrEmpty(e.Data)) + if (!string.IsNullOrEmpty(e.Data)) { #if DEBUG Console.Error.WriteLine("error {0}", e.Data); @@ -160,11 +158,17 @@ namespace Duplicati.Library.Backend process.BeginOutputReadLine(); process.BeginErrorReadLine(); + using var timer = timeout == Timeout.InfiniteTimeSpan + ? new TaskCompletionSource().Task + : Task.Delay(timeout); + + var killed = false; while (!process.HasExited) { await Task.Delay(500).ConfigureAwait(false); - if (cancelToken.IsCancellationRequested) + if (cancelToken.IsCancellationRequested || timer.IsCompleted) { + killed = true; process.Kill(); process.WaitForExit(); } @@ -174,18 +178,20 @@ namespace Duplicati.Library.Backend process.CancelErrorRead(); if (errorBuilder.ToString().Contains(RCLONE_ERROR_DIRECTORY_NOT_FOUND)) - { throw new FolderMissingException(errorBuilder.ToString()); - } if (errorBuilder.ToString().Contains(RCLONE_ERROR_CONFIG_NOT_FOUND)) - { throw new Exception($"Missing config file? {errorBuilder}"); - } if (errorBuilder.Length > 0) - { throw new Exception(errorBuilder.ToString()); + + if (killed) + { + if (timer.IsCompleted) + throw new TimeoutException(); + + throw new TaskCanceledException(); } return outputBuilder.ToString(); @@ -200,16 +206,15 @@ namespace Duplicati.Library.Backend try { - str_result = await RcloneCommandExecuter(rclone_executable, $"lsjson {remote_repo}:{remote_path}", cancelToken).ConfigureAwait(false); + str_result = await RcloneCommandExecuter(rclone_executable, $"lsjson {remote_repo}:{remote_path}", timeouts.ListTimeout, cancelToken).ConfigureAwait(false); // this will give an error if the executable does not exist. } - catch (FolderMissingException ex) { throw new FolderMissingException(ex); } - using (JsonReader jsonReader = new JsonTextReader(new StringReader(str_result))) + using (var jsonReader = new JsonTextReader(new StringReader(str_result))) { //no date parsing by JArray needed, will be parsed later jsonReader.DateParseHandling = DateParseHandling.None; @@ -220,14 +225,19 @@ namespace Duplicati.Library.Backend #if DEBUG Console.Error.WriteLine(item); #endif - FileEntry fe = new FileEntry( - item.GetValue("Name").Value(), - item.GetValue("Size").Value(), - DateTime.Parse(item.GetValue("ModTime").Value()), - DateTime.Parse(item.GetValue("ModTime").Value()) + var modTimeString = item.GetValue("ModTime")?.Value(); + var modTime = string.IsNullOrWhiteSpace(modTimeString) + ? new DateTime(0) + : DateTime.Parse(modTimeString); + + var fe = new FileEntry( + item.GetValue("Name")?.Value(), + item.GetValue("Size")?.Value() ?? -1, + modTime, + modTime ) { - IsFolder = item.GetValue("IsDir").Value() + IsFolder = item.GetValue("IsDir")?.Value() ?? false }; yield return fe; } @@ -238,7 +248,7 @@ namespace Duplicati.Library.Backend { try { - await RcloneCommandExecuter(rclone_executable, $"copyto {local_repo}:{filename} {remote_repo}:{remote_path}/{remotename}", cancelToken).ConfigureAwait(false); + await RcloneCommandExecuter(rclone_executable, $"copyto {local_repo}:{filename} {remote_repo}:{remote_path}/{remotename}", Timeout.InfiniteTimeSpan, cancelToken).ConfigureAwait(false); } catch (FolderMissingException ex) { @@ -250,7 +260,7 @@ namespace Duplicati.Library.Backend { try { - await RcloneCommandExecuter(rclone_executable, $"copyto {remote_repo}:{Path.Combine(this.remote_path, remotename)} {local_repo}:{filename}", cancelToken).ConfigureAwait(false); + await RcloneCommandExecuter(rclone_executable, $"copyto {remote_repo}:{Path.Combine(this.remote_path, remotename)} {local_repo}:{filename}", Timeout.InfiniteTimeSpan, cancelToken).ConfigureAwait(false); } catch (FolderMissingException ex) { @@ -264,7 +274,7 @@ namespace Duplicati.Library.Backend // Will give a "directory not found" error if the file does not exist, need to change that to a missing file exception try { - await RcloneCommandExecuter(rclone_executable, $"delete {remote_repo}:{Path.Combine(remote_path, remotename)}", cancelToken).ConfigureAwait(false); + await RcloneCommandExecuter(rclone_executable, $"delete {remote_repo}:{Path.Combine(remote_path, remotename)}", timeouts.ShortTimeout, cancelToken).ConfigureAwait(false); } catch (FolderMissingException ex) { @@ -272,28 +282,17 @@ namespace Duplicati.Library.Backend } } - public IList SupportedCommands - { - get - { - return new List([ - new CommandLineArgument(OPTION_LOCAL_REPO, CommandLineArgument.ArgumentType.String, Strings.Rclone.RcloneLocalRepoShort, Strings.Rclone.RcloneLocalRepoLong, "local"), - new CommandLineArgument(OPTION_REMOTE_REPO, CommandLineArgument.ArgumentType.String, Strings.Rclone.RcloneRemoteRepoShort, Strings.Rclone.RcloneRemoteRepoLong, "remote"), - new CommandLineArgument(OPTION_REMOTE_PATH, CommandLineArgument.ArgumentType.String, Strings.Rclone.RcloneRemotePathShort, Strings.Rclone.RcloneRemotePathLong, "backup"), - new CommandLineArgument(OPTION_RCLONE, CommandLineArgument.ArgumentType.String, Strings.Rclone.RcloneOptionRcloneShort, Strings.Rclone.RcloneOptionRcloneLong, ""), - new CommandLineArgument(OPTION_RCLONE_EXECUTABLE, CommandLineArgument.ArgumentType.String, Strings.Rclone.RcloneExecutableShort, Strings.Rclone.RcloneExecutableLong, "rclone") - ]); + public IList SupportedCommands => [ + new CommandLineArgument(OPTION_LOCAL_REPO, CommandLineArgument.ArgumentType.String, Strings.Rclone.RcloneLocalRepoShort, Strings.Rclone.RcloneLocalRepoLong, "local"), + new CommandLineArgument(OPTION_REMOTE_REPO, CommandLineArgument.ArgumentType.String, Strings.Rclone.RcloneRemoteRepoShort, Strings.Rclone.RcloneRemoteRepoLong, "remote"), + new CommandLineArgument(OPTION_REMOTE_PATH, CommandLineArgument.ArgumentType.String, Strings.Rclone.RcloneRemotePathShort, Strings.Rclone.RcloneRemotePathLong, "backup"), + new CommandLineArgument(OPTION_RCLONE, CommandLineArgument.ArgumentType.String, Strings.Rclone.RcloneOptionRcloneShort, Strings.Rclone.RcloneOptionRcloneLong, ""), + new CommandLineArgument(OPTION_RCLONE_EXECUTABLE, CommandLineArgument.ArgumentType.String, Strings.Rclone.RcloneExecutableShort, Strings.Rclone.RcloneExecutableLong, "rclone"), + .. TimeoutOptionsHelper.GetOptions() + .Where(x => x.Name != TimeoutOptionsHelper.READ_WRITE_TIMEOUT_OPTION) + ]; - } - } - - public string Description - { - get - { - return Strings.Rclone.Description; - } - } + public string Description => Strings.Rclone.Description; public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new[] { remote_repo }); @@ -302,7 +301,7 @@ namespace Duplicati.Library.Backend public Task CreateFolderAsync(CancellationToken cancelToken) { - return RcloneCommandExecuter(rclone_executable, $"mkdir {remote_repo}:{remote_path}", cancelToken); + return RcloneCommandExecuter(rclone_executable, $"mkdir {remote_repo}:{remote_path}", timeouts.ShortTimeout, cancelToken); } #endregion diff --git a/Duplicati/Library/Utility/Options/TimeoutOptionsHelper.cs b/Duplicati/Library/Utility/Options/TimeoutOptionsHelper.cs index 9bcb4b05d..cd93d862c 100644 --- a/Duplicati/Library/Utility/Options/TimeoutOptionsHelper.cs +++ b/Duplicati/Library/Utility/Options/TimeoutOptionsHelper.cs @@ -48,6 +48,19 @@ public static class TimeoutOptionsHelper /// public const string DEFAULT_READ_WRITE_TIMEOUT = "30s"; + /// + /// The name of the short timeout option, without a prefix + /// + public const string SHORT_TIMEOUT_OPTION = "short-timeout"; + /// + /// The name of the list timeout option, without a prefix + /// + public const string LIST_TIMEOUT_OPTION = "list-timeout"; + /// + /// The name of the read-write timeout option, without a prefix + /// + public const string READ_WRITE_TIMEOUT_OPTION = "read-write-timeout"; + /// /// Gets the timeout options /// @@ -55,9 +68,9 @@ public static class TimeoutOptionsHelper /// The timeout options public static CommandLineArgument[] GetOptions(string? prefix = null) => [ - new CommandLineArgument($"{prefix}short-timeout", CommandLineArgument.ArgumentType.Timespan, Strings.TimeoutSettingsHelper.DescriptionShortTimeoutShort, Strings.TimeoutSettingsHelper.DescriptionShortTimeoutLong, DEFAULT_SHORT_OPERATION_TIMEOUT), - new CommandLineArgument($"{prefix}list-timeout", CommandLineArgument.ArgumentType.Timespan, Strings.TimeoutSettingsHelper.DescriptionListTimeoutShort, Strings.TimeoutSettingsHelper.DescriptionListTimeoutLong, DEFAULT_LIST_OPERATION_TIMEOUT), - new CommandLineArgument($"{prefix}read-write-timeout", CommandLineArgument.ArgumentType.Timespan, Strings.TimeoutSettingsHelper.DescriptionReadWriteTimeoutShort, Strings.TimeoutSettingsHelper.DescriptionReadWriteTimeoutLong, DEFAULT_READ_WRITE_TIMEOUT) + new CommandLineArgument($"{prefix}{SHORT_TIMEOUT_OPTION}", CommandLineArgument.ArgumentType.Timespan, Strings.TimeoutSettingsHelper.DescriptionShortTimeoutShort, Strings.TimeoutSettingsHelper.DescriptionShortTimeoutLong, DEFAULT_SHORT_OPERATION_TIMEOUT), + new CommandLineArgument($"{prefix}{LIST_TIMEOUT_OPTION}", CommandLineArgument.ArgumentType.Timespan, Strings.TimeoutSettingsHelper.DescriptionListTimeoutShort, Strings.TimeoutSettingsHelper.DescriptionListTimeoutLong, DEFAULT_LIST_OPERATION_TIMEOUT), + new CommandLineArgument($"{prefix}{READ_WRITE_TIMEOUT_OPTION}", CommandLineArgument.ArgumentType.Timespan, Strings.TimeoutSettingsHelper.DescriptionReadWriteTimeoutShort, Strings.TimeoutSettingsHelper.DescriptionReadWriteTimeoutLong, DEFAULT_READ_WRITE_TIMEOUT) ]; /// @@ -68,9 +81,9 @@ public static class TimeoutOptionsHelper /// The timeout configuration public static Timeouts Parse(IReadOnlyDictionary options, string? prefix = null) => new Timeouts( - Utility.ParseTimespanOption(options, $"{prefix}short-timeout", DEFAULT_SHORT_OPERATION_TIMEOUT), - Utility.ParseTimespanOption(options, $"{prefix}list-timeout", DEFAULT_LIST_OPERATION_TIMEOUT), - Utility.ParseTimespanOption(options, $"{prefix}read-write-timeout", DEFAULT_READ_WRITE_TIMEOUT) + Utility.ParseTimespanOption(options, $"{prefix}{SHORT_TIMEOUT_OPTION}", DEFAULT_SHORT_OPERATION_TIMEOUT), + Utility.ParseTimespanOption(options, $"{prefix}{LIST_TIMEOUT_OPTION}", DEFAULT_LIST_OPERATION_TIMEOUT), + Utility.ParseTimespanOption(options, $"{prefix}{READ_WRITE_TIMEOUT_OPTION}", DEFAULT_READ_WRITE_TIMEOUT) ); /// From ea5537e40c601054a0457be465956fe4d7e4a5b1 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sat, 22 Mar 2025 09:03:34 +0100 Subject: [PATCH 29/48] Updated SharePoint --- ...uplicati.Library.Backend.SharePoint.csproj | 2 + .../SharePoint/OneDriveForBusinessBackend.cs | 29 +- .../Backend/SharePoint/SharePointBackend.cs | 374 ++++++++---------- .../Library/Backend/SharePoint/Strings.cs | 76 ++-- 4 files changed, 214 insertions(+), 267 deletions(-) diff --git a/Duplicati/Library/Backend/SharePoint/Duplicati.Library.Backend.SharePoint.csproj b/Duplicati/Library/Backend/SharePoint/Duplicati.Library.Backend.SharePoint.csproj index 2b4b2b418..0e0a72e66 100644 --- a/Duplicati/Library/Backend/SharePoint/Duplicati.Library.Backend.SharePoint.csproj +++ b/Duplicati/Library/Backend/SharePoint/Duplicati.Library.Backend.SharePoint.csproj @@ -5,6 +5,8 @@ Library Duplicati.Library.Backend Copyright © 2025 Team Duplicati, MIT license + enable + enable diff --git a/Duplicati/Library/Backend/SharePoint/OneDriveForBusinessBackend.cs b/Duplicati/Library/Backend/SharePoint/OneDriveForBusinessBackend.cs index b813e9e40..408ac7afe 100644 --- a/Duplicati/Library/Backend/SharePoint/OneDriveForBusinessBackend.cs +++ b/Duplicati/Library/Backend/SharePoint/OneDriveForBusinessBackend.cs @@ -19,14 +19,6 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; - -using Duplicati.Library.Interface; - namespace Duplicati.Library.Backend { /// @@ -39,29 +31,18 @@ namespace Duplicati.Library.Backend // This constructor is needed by the BackendLoader. public class OneDriveForBusinessBackend : SharePointBackend { - public override string ProtocolKey - { - get { return "od4b"; } - } + public override string ProtocolKey => "od4b"; - public override string DisplayName - { - get { return Strings.OneDriveForBusiness.DisplayName; } - } + public override string DisplayName => Strings.OneDriveForBusiness.DisplayName; - public override string Description - { - get { return Strings.OneDriveForBusiness.Description; } - } + public override string Description => Strings.OneDriveForBusiness.Description; public OneDriveForBusinessBackend() - :base () + : base() { } - public OneDriveForBusinessBackend(string url, Dictionary options) + public OneDriveForBusinessBackend(string url, Dictionary options) : base(url, options) { } - - } } diff --git a/Duplicati/Library/Backend/SharePoint/SharePointBackend.cs b/Duplicati/Library/Backend/SharePoint/SharePointBackend.cs index e3f553736..d56a81104 100644 --- a/Duplicati/Library/Backend/SharePoint/SharePointBackend.cs +++ b/Duplicati/Library/Backend/SharePoint/SharePointBackend.cs @@ -23,15 +23,10 @@ using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using Duplicati.Library.Utility; +using Duplicati.Library.Utility.Options; using Microsoft.SharePoint.Client; // Plain 'using' for extension methods -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; using System.Runtime.CompilerServices; using System.Text; -using System.Threading; -using System.Threading.Tasks; using SP = Microsoft.SharePoint.Client; namespace Duplicati.Library.Backend @@ -69,9 +64,9 @@ namespace Duplicati.Library.Backend private readonly bool m_useBinaryDirectMode = false; /// URL to SharePoint web. Will be determined from m_orgUri on first use. - private string m_spWebUrl; + private string? m_spWebUrl; /// Current context to SharePoint web. - private SP.ClientContext m_spContext; + private ClientContext? m_spContext; /// The chunk size for uploading files. private readonly int m_fileChunkSize = 4 << 20; // Default: 4MB @@ -79,79 +74,69 @@ namespace Duplicati.Library.Backend /// The chunk size for uploading files. private readonly int m_useContextTimeoutMs = -1; // default: do not touch original setting + /// The timeout values to use. + private readonly TimeoutOptionsHelper.Timeouts m_timeouts; + #endregion #region [Public Properties] - public virtual string ProtocolKey - { - get { return "mssp"; } - } + public virtual string ProtocolKey => "mssp"; - public virtual string DisplayName - { - get { return Strings.SharePoint.DisplayName; } - } + public virtual string DisplayName => Strings.SharePoint.DisplayName; - public virtual string Description - { - get { return Strings.SharePoint.Description; } - } + public virtual string Description => Strings.SharePoint.Description; - public virtual IList SupportedCommands - { - get - { - return new List(new ICommandLineArgument[] { - new CommandLineArgument("auth-password", CommandLineArgument.ArgumentType.Password, Strings.SharePoint.DescriptionAuthPasswordShort, Strings.SharePoint.DescriptionAuthPasswordLong), - new CommandLineArgument("auth-username", CommandLineArgument.ArgumentType.String, Strings.SharePoint.DescriptionAuthUsernameShort, Strings.SharePoint.DescriptionAuthUsernameLong), - new CommandLineArgument("integrated-authentication", CommandLineArgument.ArgumentType.Boolean, Strings.SharePoint.DescriptionIntegratedAuthenticationShort, Strings.SharePoint.DescriptionIntegratedAuthenticationLong), - new CommandLineArgument("delete-to-recycler", CommandLineArgument.ArgumentType.Boolean, Strings.SharePoint.DescriptionUseRecyclerShort, Strings.SharePoint.DescriptionUseRecyclerLong), - new CommandLineArgument("binary-direct-mode", CommandLineArgument.ArgumentType.Boolean, Strings.SharePoint.DescriptionBinaryDirectModeShort, Strings.SharePoint.DescriptionBinaryDirectModeLong, "false"), - new CommandLineArgument("web-timeout", CommandLineArgument.ArgumentType.Timespan, Strings.SharePoint.DescriptionWebTimeoutShort, Strings.SharePoint.DescriptionWebTimeoutLong), - new CommandLineArgument("chunk-size", CommandLineArgument.ArgumentType.Size, Strings.SharePoint.DescriptionChunkSizeShort, Strings.SharePoint.DescriptionChunkSizeLong, "4mb"), - }); - } - } + public virtual IList SupportedCommands => [ + .. AuthOptionsHelper.GetOptions(), + new CommandLineArgument("integrated-authentication", CommandLineArgument.ArgumentType.Boolean, Strings.SharePoint.DescriptionIntegratedAuthenticationShort, Strings.SharePoint.DescriptionIntegratedAuthenticationLong), + new CommandLineArgument("delete-to-recycler", CommandLineArgument.ArgumentType.Boolean, Strings.SharePoint.DescriptionUseRecyclerShort, Strings.SharePoint.DescriptionUseRecyclerLong), + new CommandLineArgument("binary-direct-mode", CommandLineArgument.ArgumentType.Boolean, Strings.SharePoint.DescriptionBinaryDirectModeShort, Strings.SharePoint.DescriptionBinaryDirectModeLong, "false"), + new CommandLineArgument("web-timeout", CommandLineArgument.ArgumentType.Timespan, Strings.SharePoint.DescriptionWebTimeoutShort, Strings.SharePoint.DescriptionWebTimeoutLong), + new CommandLineArgument("chunk-size", CommandLineArgument.ArgumentType.Size, Strings.SharePoint.DescriptionChunkSizeShort, Strings.SharePoint.DescriptionChunkSizeLong, "4mb"), + .. TimeoutOptionsHelper.GetOptions() + ]; - public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new string[] { + public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new string?[] { m_orgUrl.Host, string.IsNullOrWhiteSpace(m_spWebUrl) ? null : new Utility.Uri(m_spWebUrl).Host - }.Where(s => !string.IsNullOrWhiteSpace(s)) - .ToArray()); + }.WhereNotNullOrWhiteSpace().ToArray()); #endregion #region [Constructors] public SharePointBackend() - { } + { + m_serverRelPath = null!; + m_userInfo = null!; + m_timeouts = null!; + } - public SharePointBackend(string url, Dictionary options) + public SharePointBackend(string url, Dictionary options) { m_deleteToRecycler = Utility.Utility.ParseBoolOption(options, "delete-to-recycler"); m_useBinaryDirectMode = Utility.Utility.ParseBoolOption(options, "binary-direct-mode"); + m_timeouts = TimeoutOptionsHelper.Parse(options); try { - string strSpan; - if (options.TryGetValue("web-timeout", out strSpan)) + if (options.TryGetValue("web-timeout", out var strSpan)) { - TimeSpan ts = Timeparser.ParseTimeSpan(strSpan); + var ts = Timeparser.ParseTimeSpan(strSpan); if (ts.TotalMilliseconds > 30000 && ts.TotalMilliseconds < int.MaxValue) - this.m_useContextTimeoutMs = (int)ts.TotalMilliseconds; + m_useContextTimeoutMs = (int)ts.TotalMilliseconds; } } catch { } try { - string strChunkSize; - if (options.TryGetValue("chunk-size", out strChunkSize)) + if (options.TryGetValue("chunk-size", out var strChunkSize)) { - long pSize = Utility.Sizeparser.ParseSize(strChunkSize, "MB"); + var pSize = Sizeparser.ParseSize(strChunkSize, "MB"); if (pSize >= (1 << 14) && pSize <= (1 << 30)) // [16kb .. 1GB] - this.m_fileChunkSize = (int)pSize; + m_fileChunkSize = (int)pSize; } } catch { } @@ -178,31 +163,18 @@ namespace Duplicati.Library.Backend // Otherwise: Use settings from URL(precedence) or from command line options. bool useIntegratedAuthentication = Utility.Utility.ParseBoolOption(options, "integrated-authentication"); - string useUsername = null; - string usePassword = null; + string? useUsername = null; + string? usePassword = null; if (!useIntegratedAuthentication) { - if (!string.IsNullOrEmpty(u.Username)) - { - useUsername = u.Username; - if (!string.IsNullOrEmpty(u.Password)) - usePassword = u.Password; - else if (options.ContainsKey("auth-password")) - usePassword = options["auth-password"]; - } - else - { - if (options.ContainsKey("auth-username")) - { - useUsername = options["auth-username"]; - if (options.ContainsKey("auth-password")) - usePassword = options["auth-password"]; - } - } + var auth = AuthOptionsHelper.Parse(options, u); + // No validation here, maybe at least username should be set? + useUsername = auth.Username; + usePassword = auth.Password; } - if (useIntegratedAuthentication || (useUsername == null || usePassword == null)) + if (useIntegratedAuthentication || useUsername == null || usePassword == null) { // This might or might not work for on-premises SP. Maybe support if someone complains... m_userInfo = System.Net.CredentialCache.DefaultNetworkCredentials; @@ -210,7 +182,7 @@ namespace Duplicati.Library.Backend else { #pragma warning disable DE0001 - System.Security.SecureString securePwd = new System.Security.SecureString(); + var securePwd = new System.Security.SecureString(); #pragma warning restore DE0001 usePassword.ToList().ForEach(c => securePwd.AppendChar(c)); m_userInfo = new SharePointOnlineCredentials(useUsername, securePwd); @@ -232,18 +204,21 @@ namespace Duplicati.Library.Backend /// Tries a simple query to test the passed context. /// Returns 0 on success, negative if completely invalid, positive if SharePoint error (wrong creds are negative). /// - private static async Task testContextForWebAsync(SP.ClientContext ctx, bool rethrow, CancellationToken cancelToken) + private static async Task TestContextForWebAsync(ClientContext ctx, bool rethrow, TimeSpan timeout, CancellationToken cancelToken) { try { - ctx.Load(ctx.Web, w => w.Title); - await ctx.ExecuteQueryAsync().ConfigureAwait(false); // should fail and throw if anything wrong. - string webTitle = ctx.Web.Title; - if (webTitle == null) - throw new UnauthorizedAccessException(Strings.SharePoint.WebTitleReadFailedError); - return 0; + return await Utility.Utility.WithTimeout(timeout, cancelToken, async _ => + { + ctx.Load(ctx.Web, w => w.Title); + await ctx.ExecuteQueryAsync().ConfigureAwait(false); // should fail and throw if anything wrong. + string webTitle = ctx.Web.Title; + if (webTitle == null) + throw new UnauthorizedAccessException(Strings.SharePoint.WebTitleReadFailedError); + return 0; + }).ConfigureAwait(false); } - catch (Microsoft.SharePoint.Client.ServerException) + catch (ServerException) { if (rethrow) throw; else return 1; @@ -259,15 +234,15 @@ namespace Duplicati.Library.Backend /// Builds a client context and tries a simple query to test if there's a web. /// Returns 0 on success, negative if completely invalid, positive if SharePoint error (likely wrong creds). /// - private static async Task<(int status, SP.ClientContext retCtx)> testUrlForWebAsync(string url, System.Net.ICredentials userInfo, bool rethrow, CancellationToken cancelToken) + private static async Task<(int status, ClientContext? retCtx)> TestUrlForWebAsync(string url, System.Net.ICredentials userInfo, bool rethrow, TimeSpan timeout, CancellationToken cancelToken) { int result = -1; - SP.ClientContext retCtx = null; + ClientContext? retCtx = null; var ctx = CreateNewContext(url); try { ctx.Credentials = userInfo; - result = await testContextForWebAsync(ctx, rethrow, cancelToken).ConfigureAwait(false); + result = await TestContextForWebAsync(ctx, rethrow, timeout, cancelToken).ConfigureAwait(false); if (result >= 0) { retCtx = ctx; @@ -291,9 +266,9 @@ namespace Duplicati.Library.Backend /// If that won't help, we will try all possible paths from longest /// to shortest... /// - private static async Task<(string testUrl, SP.ClientContext retCtx)> findCorrectWebPathAsync(Utility.Uri orgUrl, System.Net.ICredentials userInfo, CancellationToken cancelToken) + private static async Task<(string? testUrl, ClientContext? retCtx)> FindCorrectWebPathAsync(Utility.Uri orgUrl, System.Net.ICredentials userInfo, TimeSpan timeout, CancellationToken cancelToken) { - SP.ClientContext retCtx = null; + ClientContext? retCtx = null; int status; var path = orgUrl.Path; @@ -303,34 +278,34 @@ namespace Duplicati.Library.Backend if (webIndicatorPos >= 0) { var testUrl = new Utility.Uri(orgUrl.Scheme, orgUrl.Host, path.Substring(0, webIndicatorPos), null, null, null, orgUrl.Port).ToString(); - (status, retCtx) = await testUrlForWebAsync(testUrl, userInfo, false, cancelToken).ConfigureAwait(false); + (status, retCtx) = await TestUrlForWebAsync(testUrl, userInfo, false, timeout, cancelToken).ConfigureAwait(false); if (status >= 0) return (testUrl, retCtx); } // Now go through path and see where we land a success. - string[] pathParts = path.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries); + var pathParts = path.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries); // first we look for the doc library - int docLibrary = Array.FindIndex(pathParts, p => StringComparer.OrdinalIgnoreCase.Equals(p, "documents")); + var docLibrary = Array.FindIndex(pathParts, p => StringComparer.OrdinalIgnoreCase.Equals(p, "documents")); if (docLibrary >= 0) { - string testUrl = new Utility.Uri(orgUrl.Scheme, orgUrl.Host, + var testUrl = new Utility.Uri(orgUrl.Scheme, orgUrl.Host, string.Join("/", pathParts, 0, docLibrary), null, null, null, orgUrl.Port).ToString(); - (status, retCtx) = await testUrlForWebAsync(testUrl, userInfo, false, cancelToken).ConfigureAwait(false); + (status, retCtx) = await TestUrlForWebAsync(testUrl, userInfo, false, timeout, cancelToken).ConfigureAwait(false); if (status >= 0) return (testUrl, retCtx); } // last but not least: try one after the other. - for (int pi = pathParts.Length - 1; pi >= 0; pi--) + for (var pi = pathParts.Length - 1; pi >= 0; pi--) { if (pi == docLibrary) continue; // already tested - string testUrl = new Utility.Uri(orgUrl.Scheme, orgUrl.Host, + var testUrl = new Utility.Uri(orgUrl.Scheme, orgUrl.Host, string.Join("/", pathParts, 0, pi), null, null, null, orgUrl.Port).ToString(); - (status, retCtx) = await testUrlForWebAsync(testUrl, userInfo, false, cancelToken).ConfigureAwait(false); + (status, retCtx) = await TestUrlForWebAsync(testUrl, userInfo, false, timeout, cancelToken).ConfigureAwait(false); if (status >= 0) return (testUrl, retCtx); } @@ -340,7 +315,7 @@ namespace Duplicati.Library.Backend } /// Return the preconfigured SP.ClientContext to use. - private async Task getSpClientContextAsync(bool forceNewContext, CancellationToken cancelToken) + private async Task GetSpClientContextAsync(bool forceNewContext, CancellationToken cancelToken) { if (forceNewContext) { @@ -352,8 +327,8 @@ namespace Duplicati.Library.Backend { if (m_spWebUrl == null) { - (m_spWebUrl, m_spContext) = await findCorrectWebPathAsync(m_orgUrl, m_userInfo, cancelToken).ConfigureAwait(false); - if (m_spWebUrl == null) + (m_spWebUrl, m_spContext) = await FindCorrectWebPathAsync(m_orgUrl, m_userInfo, m_timeouts.ShortTimeout, cancelToken).ConfigureAwait(false); + if (m_spWebUrl == null || m_spContext == null) throw new System.Net.WebException(Strings.SharePoint.NoSharePointWebFoundError(m_orgUrl.ToString())); } else @@ -362,52 +337,29 @@ namespace Duplicati.Library.Backend m_spContext = CreateNewContext(m_spWebUrl); m_spContext.Credentials = m_userInfo; } - if (m_spContext != null && m_useContextTimeoutMs > 0) + if (m_useContextTimeoutMs > 0) m_spContext.RequestTimeout = m_useContextTimeoutMs; } return m_spContext; } - /// /// Dedicated code to wrap ExecuteQuery on file ops and check for errors. /// We have to check for the exceptions thrown to know about file /folder existence. /// Why the funny guys at MS provided an .Exists field stays a mystery... /// - private void wrappedExecuteQueryOnContext(SP.ClientContext ctx, string serverRelPathInfo, bool isFolder) + private async Task WrappedExecuteQueryOnContextAsync(ClientContext ctx, string serverRelPathInfo, bool isFolder, TimeSpan timeout, CancellationToken cancelToken) { - try { ctx.ExecuteQuery(); } + try { await Utility.Utility.WithTimeout(timeout, cancelToken, _ => ctx.ExecuteQueryAsync()).ConfigureAwait(false); } catch (ServerException ex) { // funny: If a folder is not found, we still get a FileNotFoundException from Server...?!? // Thus, we help ourselves by just passing the info if we wanted to query a folder. if (ex.ServerErrorTypeName == "System.IO.DirectoryNotFoundException" || (ex.ServerErrorTypeName == "System.IO.FileNotFoundException" && isFolder)) - throw new Interface.FolderMissingException(Strings.SharePoint.MissingElementError(serverRelPathInfo, m_spWebUrl)); + throw new FolderMissingException(Strings.SharePoint.MissingElementError(serverRelPathInfo, m_spWebUrl)); if (ex.ServerErrorTypeName == "System.IO.FileNotFoundException") - throw new Interface.FileMissingException(Strings.SharePoint.MissingElementError(serverRelPathInfo, m_spWebUrl)); - else - throw; - } - } - - /// - /// Dedicated code to wrap ExecuteQuery on file ops and check for errors. - /// We have to check for the exceptions thrown to know about file /folder existence. - /// Why the funny guys at MS provided an .Exists field stays a mystery... - /// - private async Task wrappedExecuteQueryOnContextAsync(SP.ClientContext ctx, string serverRelPathInfo, bool isFolder, CancellationToken cancelToken) - { - try { await ctx.ExecuteQueryAsync().ConfigureAwait(false); } - catch (ServerException ex) - { - // funny: If a folder is not found, we still get a FileNotFoundException from Server...?!? - // Thus, we help ourselves by just passing the info if we wanted to query a folder. - if (ex.ServerErrorTypeName == "System.IO.DirectoryNotFoundException" - || (ex.ServerErrorTypeName == "System.IO.FileNotFoundException" && isFolder)) - throw new Interface.FolderMissingException(Strings.SharePoint.MissingElementError(serverRelPathInfo, m_spWebUrl)); - if (ex.ServerErrorTypeName == "System.IO.FileNotFoundException") - throw new Interface.FileMissingException(Strings.SharePoint.MissingElementError(serverRelPathInfo, m_spWebUrl)); + throw new FileMissingException(Strings.SharePoint.MissingElementError(serverRelPathInfo, m_spWebUrl)); else throw; } @@ -418,9 +370,9 @@ namespace Duplicati.Library.Backend /// /// The new context. /// The url to create the context for. - private static SP.ClientContext CreateNewContext(string url) + private static ClientContext CreateNewContext(string url) { - var ctx = new SP.ClientContext(url); + var ctx = new ClientContext(url); ctx.WebRequestExecutorFactory = new CustomWebRequestExecutorFactory(ctx.WebRequestExecutorFactory); return ctx; } @@ -470,18 +422,18 @@ namespace Duplicati.Library.Backend public async Task TestAsync(CancellationToken cancelToken) { - var ctx = await getSpClientContextAsync(true, cancelToken).ConfigureAwait(false); - await testContextForWebAsync(ctx, true, cancelToken).ConfigureAwait(false); + var ctx = await GetSpClientContextAsync(true, cancelToken).ConfigureAwait(false); + await TestContextForWebAsync(ctx, true, m_timeouts.ShortTimeout, cancelToken).ConfigureAwait(false); } /// public IAsyncEnumerable ListAsync(CancellationToken cancelToken) - => doListAsync(false, cancelToken); + => DoListAsync(false, cancelToken); - private async IAsyncEnumerable doListAsync(bool useNewContext, [EnumeratorCancellation] CancellationToken cancelToken) + private async IAsyncEnumerable DoListAsync(bool useNewContext, [EnumeratorCancellation] CancellationToken cancelToken) { - var ctx = await getSpClientContextAsync(useNewContext, cancelToken).ConfigureAwait(false); - SP.Folder remoteFolder = null; + var ctx = await GetSpClientContextAsync(useNewContext, cancelToken).ConfigureAwait(false); + Folder? remoteFolder = null; bool retry = false; try { @@ -489,13 +441,13 @@ namespace Duplicati.Library.Backend ctx.Load(remoteFolder, f => f.Exists); ctx.Load(remoteFolder, f => f.Files, f => f.Folders); - await wrappedExecuteQueryOnContextAsync(ctx, m_serverRelPath, true, cancelToken).ConfigureAwait(false); + await WrappedExecuteQueryOnContextAsync(ctx, m_serverRelPath, true, m_timeouts.ListTimeout, cancelToken).ConfigureAwait(false); if (!remoteFolder.Exists) - throw new Interface.FolderMissingException(Strings.SharePoint.MissingElementError(m_serverRelPath, m_spWebUrl)); + throw new FolderMissingException(Strings.SharePoint.MissingElementError(m_serverRelPath, m_spWebUrl)); } catch (ServerException) { throw; /* rethrow if Server answered */ } - catch (Interface.FileMissingException) { throw; } - catch (Interface.FolderMissingException) { throw; } + catch (FileMissingException) { throw; } + catch (FolderMissingException) { throw; } catch { if (useNewContext) @@ -506,11 +458,13 @@ namespace Duplicati.Library.Backend if (retry) { // An exception was caught, and List() should be retried. - await foreach (var f in doListAsync(true, cancelToken).ConfigureAwait(false)) + await foreach (var f in DoListAsync(true, cancelToken).ConfigureAwait(false)) yield return f; } else { + if (remoteFolder == null) + throw new FolderMissingException(Strings.SharePoint.MissingElementError(m_serverRelPath, m_spWebUrl)); var list = remoteFolder.Folders.Where(ff => ff.Exists) .Select(f => new FileEntry(f.Name, -1, f.TimeLastModified, f.TimeLastModified) { IsFolder = true }) @@ -524,39 +478,40 @@ namespace Duplicati.Library.Backend public async Task GetAsync(string remotename, string filename, CancellationToken cancelToken) { - using (System.IO.FileStream fs = System.IO.File.Create(filename)) + using (var fs = System.IO.File.Create(filename)) await GetAsync(remotename, fs, cancelToken); } - public Task GetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) - => doGetAsync(remotename, stream, false, cancelToken); + public Task GetAsync(string remotename, Stream stream, CancellationToken cancelToken) + => DoGetAsync(remotename, stream, false, cancelToken); - private async Task doGetAsync(string remotename, System.IO.Stream stream, bool useNewContext, CancellationToken cancelToken) + private async Task DoGetAsync(string remotename, Stream stream, bool useNewContext, CancellationToken cancelToken) { string fileurl = m_serverRelPath + System.Web.HttpUtility.UrlPathEncode(remotename); - var ctx = await getSpClientContextAsync(useNewContext, cancelToken).ConfigureAwait(false); + var ctx = await GetSpClientContextAsync(useNewContext, cancelToken).ConfigureAwait(false); try { - SP.File remoteFile = ctx.Web.GetFileByServerRelativeUrl(fileurl); + var remoteFile = ctx.Web.GetFileByServerRelativeUrl(fileurl); ctx.Load(remoteFile, f => f.Exists); - await wrappedExecuteQueryOnContextAsync(ctx, fileurl, false, cancelToken).ConfigureAwait(false); + await WrappedExecuteQueryOnContextAsync(ctx, fileurl, false, m_timeouts.ShortTimeout, cancelToken).ConfigureAwait(false); if (!remoteFile.Exists) - throw new Interface.FileMissingException(Strings.SharePoint.MissingElementError(fileurl, m_spWebUrl)); + throw new FileMissingException(Strings.SharePoint.MissingElementError(fileurl, m_spWebUrl)); } catch (ServerException) { throw; /* rethrow if Server answered */ } - catch (Interface.FileMissingException) { throw; } - catch (Interface.FolderMissingException) { throw; } + catch (FileMissingException) { throw; } + catch (FolderMissingException) { throw; } catch { if (useNewContext) throw; - await doGetAsync(remotename, stream, true, cancelToken); + await DoGetAsync(remotename, stream, true, cancelToken); } using (var fileInfo = SP.File.OpenBinaryDirect(ctx, fileurl)) using (var s = fileInfo.Stream) - await Utility.Utility.CopyStreamAsync(s, stream, true, cancelToken).ConfigureAwait(false); + using (var t = s.ObserveReadTimeout(m_timeouts.ReadWriteTimeout)) + await Utility.Utility.CopyStreamAsync(t, stream, true, cancelToken).ConfigureAwait(false); } public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) @@ -565,85 +520,85 @@ namespace Duplicati.Library.Backend await PutAsync(remotename, fs, cancelToken).ConfigureAwait(false); } - public Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken) { return doPutAsync(remotename, stream, false, cancelToken); } - private async Task doPutAsync(string remotename, Stream stream, bool useNewContext, CancellationToken cancelToken) + public Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken) { return DoPutAsync(remotename, stream, false, cancelToken); } + private async Task DoPutAsync(string remotename, Stream stream, bool useNewContext, CancellationToken cancelToken) { - string fileurl = m_serverRelPath + System.Web.HttpUtility.UrlPathEncode(remotename); - var ctx = await getSpClientContextAsync(useNewContext, cancelToken).ConfigureAwait(false); + var fileurl = m_serverRelPath + System.Web.HttpUtility.UrlPathEncode(remotename); + var ctx = await GetSpClientContextAsync(useNewContext, cancelToken).ConfigureAwait(false); try { - SP.Folder remoteFolder = ctx.Web.GetFolderByServerRelativeUrl(m_serverRelPath); + var remoteFolder = ctx.Web.GetFolderByServerRelativeUrl(m_serverRelPath); ctx.Load(remoteFolder, f => f.Exists, f => f.ServerRelativeUrl); - await wrappedExecuteQueryOnContextAsync(ctx, m_serverRelPath, true, cancelToken).ConfigureAwait(false); + await WrappedExecuteQueryOnContextAsync(ctx, m_serverRelPath, true, m_timeouts.ShortTimeout, cancelToken).ConfigureAwait(false); if (!remoteFolder.Exists) - throw new Interface.FolderMissingException(Strings.SharePoint.MissingElementError(m_serverRelPath, m_spWebUrl)); + throw new FolderMissingException(Strings.SharePoint.MissingElementError(m_serverRelPath, m_spWebUrl)); useNewContext = true; // disable retry if (!m_useBinaryDirectMode) - await uploadFileSlicePerSlice(ctx, remoteFolder, stream, fileurl, cancelToken).ConfigureAwait(false); + await UploadFileSlicePerSlice(ctx, remoteFolder, stream, fileurl, cancelToken).ConfigureAwait(false); } catch (ServerException) { throw; /* rethrow if Server answered */ } - catch (Interface.FileMissingException) { throw; } - catch (Interface.FolderMissingException) { throw; } + catch (FileMissingException) { throw; } + catch (FolderMissingException) { throw; } catch { if (useNewContext) throw; - await doPutAsync(remotename, stream, true, cancelToken).ConfigureAwait(false); + await DoPutAsync(remotename, stream, true, cancelToken).ConfigureAwait(false); } if (m_useBinaryDirectMode) - SP.File.SaveBinaryDirect(ctx, fileurl, stream, true); + using (var ts = stream.ObserveWriteTimeout(m_timeouts.ReadWriteTimeout, false)) + SP.File.SaveBinaryDirect(ctx, fileurl, ts, true); } /// /// Upload in chunks to bypass filesize limit. /// https://msdn.microsoft.com/en-us/library/office/dn904536.aspx /// - private async Task uploadFileSlicePerSlice(ClientContext ctx, Folder folder, Stream sourceFileStream, string fileName, CancellationToken cancelToken) + private async Task UploadFileSlicePerSlice(ClientContext ctx, Folder folder, Stream sourceFileStream, string fileName, CancellationToken cancelToken) { // Each sliced upload requires a unique ID. - Guid uploadId = Guid.NewGuid(); + var uploadId = Guid.NewGuid(); // Get the name of the file. - string uniqueFileName = Path.GetFileName(fileName); + var uniqueFileName = Path.GetFileName(fileName); // File object. - SP.File uploadFile = null; + SP.File? uploadFile = null; // Calculate block size in bytes. - int blockSize = m_fileChunkSize; - byte[] buf = new byte[blockSize]; + var blockSize = m_fileChunkSize; + var buf = new byte[blockSize]; - bool first = true; - bool needsFinalize = true; - long fileoffset = 0; + var needsFinalize = true; + var fileoffset = 0L; - int lastreadsize = -1; + var lastreadsize = -1; while (lastreadsize != 0) { - int bufCnt = 0; + var bufCnt = 0; // read chunk to array (necessary because chunk uploads fail if size unknown) while (bufCnt < blockSize && (lastreadsize = await sourceFileStream.ReadAsync(buf, bufCnt, blockSize - bufCnt, cancelToken).ConfigureAwait(false)) > 0) bufCnt += lastreadsize; using (var contentChunk = new MemoryStream(buf, 0, bufCnt, false)) { - ClientResult bytesUploaded = null; - if (first) + ClientResult? bytesUploaded = null; + if (uploadFile == null) { // Add an empty / single chunk file. - FileCreationInformation fileInfo = new FileCreationInformation(); + var fileInfo = new FileCreationInformation(); fileInfo.Url = uniqueFileName; fileInfo.Overwrite = true; fileInfo.ContentStream = (bufCnt < blockSize) ? contentChunk : new MemoryStream(0); uploadFile = folder.Files.Add(fileInfo); - if (bufCnt < blockSize) needsFinalize = false; - else bytesUploaded = uploadFile.StartUpload(uploadId, contentChunk); // new OverrideableStream(chunkStream)); - - first = false; + if (bufCnt < blockSize) + needsFinalize = false; + else + bytesUploaded = uploadFile.StartUpload(uploadId, contentChunk); // new OverrideableStream(chunkStream)); } else { @@ -663,7 +618,7 @@ namespace Duplicati.Library.Backend if (bytesUploaded == null) ctx.Load(uploadFile, f => f.Length); - ctx.ExecuteQuery(); + await Utility.Utility.WithTimeout(m_timeouts.ReadWriteTimeout, cancelToken, _ => ctx.ExecuteQueryAsync()).ConfigureAwait(false); // Check consistency and update fileoffset for the next slice. if (bytesUploaded != null) { @@ -674,31 +629,35 @@ namespace Duplicati.Library.Backend else fileoffset += bufCnt; } } - if (needsFinalize) // finalize file (should only occur if filesize is exactly a multiple of chunksize) + + if (needsFinalize && uploadFile != null) // finalize file (should only occur if filesize is exactly a multiple of chunksize) { // End sliced upload by calling FinishUpload. uploadFile = uploadFile.FinishUpload(uploadId, fileoffset, new MemoryStream(0)); ctx.Load(uploadFile, f => f.Length); - ctx.ExecuteQuery(); + await Utility.Utility.WithTimeout(m_timeouts.ReadWriteTimeout, cancelToken, _ => ctx.ExecuteQueryAsync()).ConfigureAwait(false); } + if (uploadFile == null) + throw new FileMissingException(Strings.SharePoint.MissingElementError(fileName, m_spWebUrl)); + if (uploadFile.Length != fileoffset) throw new InvalidDataException(string.Format("Reported final file size ({0:N0}) does not match internal recording ({1:N0}) for '{2}'.", uploadFile.Length, fileoffset, uniqueFileName)); return uploadFile; } - public Task CreateFolderAsync(CancellationToken cancelToken) { return doCreateFolderAsync(false, cancelToken); } - private async Task doCreateFolderAsync(bool useNewContext, CancellationToken cancelToken) + public Task CreateFolderAsync(CancellationToken cancelToken) { return DoCreateFolderAsync(false, cancelToken); } + private async Task DoCreateFolderAsync(bool useNewContext, CancellationToken cancelToken) { - var ctx = await getSpClientContextAsync(useNewContext, cancelToken).ConfigureAwait(false); + var ctx = await GetSpClientContextAsync(useNewContext, cancelToken).ConfigureAwait(false); try { - int pathLengthToWeb = new Utility.Uri(m_spWebUrl).Path.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries).Length; + var pathLengthToWeb = new Utility.Uri(m_spWebUrl).Path.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries).Length; - string[] folderNames = m_serverRelPath.Substring(0, m_serverRelPath.Length - 1).Split('/'); + var folderNames = m_serverRelPath.Substring(0, m_serverRelPath.Length - 1).Split('/'); folderNames = Array.ConvertAll(folderNames, fold => System.Net.WebUtility.UrlDecode(fold)); - var spfolders = new SP.Folder[folderNames.Length]; + var spfolders = new Folder[folderNames.Length]; var relativePathBuilder = new StringBuilder(); int fi = 0; for (; fi < folderNames.Length; fi++) @@ -710,7 +669,7 @@ namespace Duplicati.Library.Backend var folder = ctx.Web.GetFolderByServerRelativeUrl(folderRelPath); spfolders[fi] = folder; ctx.Load(folder, f => f.Exists); - try { await wrappedExecuteQueryOnContextAsync(ctx, folderRelPath, true, cancelToken); } + try { await WrappedExecuteQueryOnContextAsync(ctx, folderRelPath, true, m_timeouts.ShortTimeout, cancelToken); } catch (FolderMissingException) { break; } if (!folder.Exists) break; @@ -720,51 +679,56 @@ namespace Duplicati.Library.Backend spfolders[fi] = spfolders[fi - 1].Folders.Add(folderNames[fi]); ctx.Load(spfolders[folderNames.Length - 1], f => f.Exists); - await wrappedExecuteQueryOnContextAsync(ctx, m_serverRelPath, true, cancelToken); + await WrappedExecuteQueryOnContextAsync(ctx, m_serverRelPath, true, m_timeouts.ShortTimeout, cancelToken).ConfigureAwait(false); if (!spfolders[folderNames.Length - 1].Exists) - throw new Interface.FolderMissingException(Strings.SharePoint.MissingElementError(m_serverRelPath, m_spWebUrl)); + throw new FolderMissingException(Strings.SharePoint.MissingElementError(m_serverRelPath, m_spWebUrl)); } catch (ServerException) { throw; /* rethrow if Server answered */ } - catch (Interface.FileMissingException) { throw; } - catch (Interface.FolderMissingException) { throw; } + catch (FileMissingException) { throw; } + catch (FolderMissingException) { throw; } catch { if (useNewContext) throw; - await doCreateFolderAsync(true, cancelToken).ConfigureAwait(false); + await DoCreateFolderAsync(true, cancelToken).ConfigureAwait(false); } } public Task DeleteAsync(string remotename, CancellationToken cancellationToken) - => doDeleteAsync(remotename, false, cancellationToken); - private async Task doDeleteAsync(string remotename, bool useNewContext, CancellationToken cancellationToken) + => DoDeleteAsync(remotename, false, cancellationToken); + private async Task DoDeleteAsync(string remotename, bool useNewContext, CancellationToken cancellationToken) { - var ctx = await getSpClientContextAsync(useNewContext, cancellationToken).ConfigureAwait(false); + var ctx = await GetSpClientContextAsync(useNewContext, cancellationToken).ConfigureAwait(false); try { string fileurl = m_serverRelPath + System.Web.HttpUtility.UrlPathEncode(remotename); SP.File remoteFile = ctx.Web.GetFileByServerRelativeUrl(fileurl); ctx.Load(remoteFile); - await wrappedExecuteQueryOnContextAsync(ctx, fileurl, false, cancellationToken).ConfigureAwait(false); + await WrappedExecuteQueryOnContextAsync(ctx, fileurl, false, m_timeouts.ShortTimeout, cancellationToken).ConfigureAwait(false); if (!remoteFile.Exists) - throw new Interface.FileMissingException(Strings.SharePoint.MissingElementError(fileurl, m_spWebUrl)); + throw new FileMissingException(Strings.SharePoint.MissingElementError(fileurl, m_spWebUrl)); - if (m_deleteToRecycler) remoteFile.Recycle(); - else remoteFile.DeleteObject(); + await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancellationToken, _ => + { + if (m_deleteToRecycler) + remoteFile.Recycle(); + else + remoteFile.DeleteObject(); - await ctx.ExecuteQueryAsync().ConfigureAwait(false); + return ctx.ExecuteQueryAsync(); + }).ConfigureAwait(false); } catch (ServerException) { throw; /* rethrow if Server answered */ } - catch (Interface.FileMissingException) { throw; } - catch (Interface.FolderMissingException) { throw; } + catch (FileMissingException) { throw; } + catch (FolderMissingException) { throw; } catch { if (useNewContext) throw; - await doDeleteAsync(remotename, true, cancellationToken).ConfigureAwait(false); + await DoDeleteAsync(remotename, true, cancellationToken).ConfigureAwait(false); } } @@ -781,7 +745,11 @@ namespace Duplicati.Library.Backend m_spContext.Dispose(); } catch { } - m_userInfo = null; + finally + { + m_spContext = null; + } + m_userInfo = null!; } #endregion diff --git a/Duplicati/Library/Backend/SharePoint/Strings.cs b/Duplicati/Library/Backend/SharePoint/Strings.cs index 542620149..cd674308a 100644 --- a/Duplicati/Library/Backend/SharePoint/Strings.cs +++ b/Duplicati/Library/Backend/SharePoint/Strings.cs @@ -1,56 +1,52 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. using Duplicati.Library.Localization.Short; namespace Duplicati.Library.Backend.Strings { internal static class SharePoint { - public static string Description { get { return LC.L(@"This backend can read and write data to a SharePoint server (including OneDrive for Business). Allowed formats are ""mssp://tennant.sharepoint.com/PathToWeb//BaseDocLibrary/subfolder"" and ""mssp://username:password@tennant.sharepoint.com/PathToWeb//BaseDocLibrary/subfolder"". Use a double slash '//' in the path to denote the web from the documents library."); } } - public static string DisplayName { get { return LC.L(@"Microsoft SharePoint"); } } - public static string DescriptionAuthPasswordLong { get { return LC.L(@"The password used to connect to the server. This may also be supplied as the environment variable ""AUTH_PASSWORD""."); } } - public static string DescriptionAuthPasswordShort { get { return LC.L(@"Supply the password used to connect to the server"); } } - public static string DescriptionAuthUsernameLong { get { return LC.L(@"The username used to connect to the server. This may also be supplied as the environment variable ""AUTH_USERNAME""."); } } - public static string DescriptionAuthUsernameShort { get { return LC.L(@"Supply the username used to connect to the server"); } } - public static string DescriptionIntegratedAuthenticationLong { get { return LC.L(@"If the server and client both supports integrated authentication, this option enables that authentication method. This is likely only available with windows servers and clients."); } } - public static string DescriptionIntegratedAuthenticationShort { get { return LC.L(@"Use windows integrated authentication to connect to the server"); } } - public static string DescriptionUseRecyclerLong { get { return LC.L(@"Use this option to have files moved to the recycle bin folder instead of removing them permanently when compacting or deleting backups."); } } - public static string DescriptionUseRecyclerShort { get { return LC.L(@"Move deleted files to the recycle bin"); } } + public static string Description => LC.L(@"This backend can read and write data to a SharePoint server (including OneDrive for Business). Allowed formats are ""mssp://tennant.sharepoint.com/PathToWeb//BaseDocLibrary/subfolder"" and ""mssp://username:password@tennant.sharepoint.com/PathToWeb//BaseDocLibrary/subfolder"". Use a double slash '//' in the path to denote the web from the documents library."); + public static string DisplayName => LC.L(@"Microsoft SharePoint"); + public static string DescriptionIntegratedAuthenticationLong => LC.L(@"If the server and client both supports integrated authentication, this option enables that authentication method. This is likely only available with windows servers and clients."); + public static string DescriptionIntegratedAuthenticationShort => LC.L(@"Use windows integrated authentication to connect to the server"); + public static string DescriptionUseRecyclerLong => LC.L(@"Use this option to have files moved to the recycle bin folder instead of removing them permanently when compacting or deleting backups."); + public static string DescriptionUseRecyclerShort => LC.L(@"Move deleted files to the recycle bin"); - public static string DescriptionBinaryDirectModeLong { get { return LC.L(@"Use this option to upload files to SharePoint as a whole with BinaryDirect mode. This is the most efficient way of uploading, but can cause non-recoverable timeouts under certain conditions. Use this option only with very fast and stable internet connections."); } } - public static string DescriptionBinaryDirectModeShort { get { return LC.L(@"Upload files using binary direct mode"); } } + public static string DescriptionBinaryDirectModeLong => LC.L(@"Use this option to upload files to SharePoint as a whole with BinaryDirect mode. This is the most efficient way of uploading, but can cause non-recoverable timeouts under certain conditions. Use this option only with very fast and stable internet connections."); + public static string DescriptionBinaryDirectModeShort => LC.L(@"Upload files using binary direct mode"); - public static string DescriptionWebTimeoutLong { get { return LC.L(@"Use this option to specify a custom value for timeouts of web operation when communicating with SharePoint Server. Recommended value is 180s."); } } - public static string DescriptionWebTimeoutShort { get { return LC.L(@"Set timeout for SharePoint web operations"); } } + public static string DescriptionWebTimeoutLong => LC.L(@"Use this option to specify a custom value for timeouts of web operation when communicating with SharePoint Server. Recommended value is 180s."); + public static string DescriptionWebTimeoutShort => LC.L(@"Set timeout for SharePoint web operations"); - public static string DescriptionChunkSizeLong { get { return LC.L(@"Use this option to specify the size of each chunk when uploading to SharePoint Server. Recommended value is 4MB."); } } - public static string DescriptionChunkSizeShort { get { return LC.L(@"Set block size for chunked uploads to SharePoint"); } } + public static string DescriptionChunkSizeLong => LC.L(@"Use this option to specify the size of each chunk when uploading to SharePoint Server. Recommended value is 4MB."); + public static string DescriptionChunkSizeShort => LC.L(@"Set block size for chunked uploads to SharePoint"); - public static string MissingElementError(string serverrelpath, string hosturl) { return LC.L(@"Element with path '{0}' not found on host '{1}'.", serverrelpath, hosturl); } + public static string MissingElementError(string serverrelpath, string? hosturl) { return LC.L(@"Element with path '{0}' not found on host '{1}'.", serverrelpath, hosturl); } public static string NoSharePointWebFoundError(string url) { return LC.L(@"No SharePoint web could be logged in to at path '{0}'. Maybe wrong credentials. Or try using '//' in path to separate web from folder path.", url); } - public static string WebTitleReadFailedError { get { return LC.L(@"Everything seemed alright, but then web title could not be read to test connection. Something's wrong."); } } + public static string WebTitleReadFailedError => LC.L(@"Everything seemed alright, but then web title could not be read to test connection. Something's wrong."); } internal static class OneDriveForBusiness { - public static string Description { get { return LC.L(@"This backend can read and write data to Microsoft OneDrive for Business. Allowed formats are ""od4b://tennant.sharepoint.com/personal/username_domain/Documents/subfolder"" and ""od4b://username:password@tennant.sharepoint.com/personal/username_domain/Documents/folder"". You can use a double slash '//' in the path to denote the base path from the documents folder."); } } - public static string DisplayName { get { return LC.L(@"Microsoft OneDrive for Business"); } } + public static string Description => LC.L(@"This backend can read and write data to Microsoft OneDrive for Business. Allowed formats are ""od4b://tennant.sharepoint.com/personal/username_domain/Documents/subfolder"" and ""od4b://username:password@tennant.sharepoint.com/personal/username_domain/Documents/folder"". You can use a double slash '//' in the path to denote the base path from the documents folder."); + public static string DisplayName => LC.L(@"Microsoft OneDrive for Business"); } } From 5833434a6e64bcbb6013beea29cae11f2e11f344 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sat, 22 Mar 2025 09:23:38 +0100 Subject: [PATCH 30/48] Updated Sia --- .../Sia/Duplicati.Library.Backend.Sia.csproj | 2 + Duplicati/Library/Backend/Sia/Sia.cs | 217 ++++++++---------- Duplicati/Library/Backend/Sia/Strings.cs | 16 +- 3 files changed, 110 insertions(+), 125 deletions(-) diff --git a/Duplicati/Library/Backend/Sia/Duplicati.Library.Backend.Sia.csproj b/Duplicati/Library/Backend/Sia/Duplicati.Library.Backend.Sia.csproj index b9c7aa57c..7de58c33f 100644 --- a/Duplicati/Library/Backend/Sia/Duplicati.Library.Backend.Sia.csproj +++ b/Duplicati/Library/Backend/Sia/Duplicati.Library.Backend.Sia.csproj @@ -4,6 +4,8 @@ net8.0 Library Copyright © 2025 Team Duplicati, MIT license + enable + enable diff --git a/Duplicati/Library/Backend/Sia/Sia.cs b/Duplicati/Library/Backend/Sia/Sia.cs index e99b91c1a..56144e239 100644 --- a/Duplicati/Library/Backend/Sia/Sia.cs +++ b/Duplicati/Library/Backend/Sia/Sia.cs @@ -21,14 +21,11 @@ using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using Duplicati.Library.Utility; +using Duplicati.Library.Utility.Options; using Newtonsoft.Json; -using System; -using System.Collections.Generic; using System.Globalization; using System.Net; using System.Runtime.CompilerServices; -using System.Threading; -using System.Threading.Tasks; namespace Duplicati.Library.Backend.Sia { @@ -42,35 +39,41 @@ namespace Duplicati.Library.Backend.Sia private readonly int m_apiport; private readonly string m_targetpath; private readonly float m_redundancy; - private readonly string m_authorization; + private readonly string? m_authorization; + private readonly TimeoutOptionsHelper.Timeouts m_timeouts; // ReSharper disable once UnusedMember.Global // This constructor is needed by the BackendLoader. public Sia() { + m_apihost = null!; + m_targetpath = null!; + m_timeouts = null!; } // ReSharper disable once UnusedMember.Global // This constructor is needed by the BackendLoader. - public Sia(string url, Dictionary options) + public Sia(string url, Dictionary options) { var uri = new Utility.Uri(url); m_apihost = uri.Host; m_apiport = uri.Port; m_targetpath = uri.Path; + m_timeouts = TimeoutOptionsHelper.Parse(options); m_redundancy = 1.5F; - if (options.ContainsKey(SIA_REDUNDANCY)) - m_redundancy = (float)decimal.Parse(options[SIA_REDUNDANCY], CultureInfo.InvariantCulture); + var redundancyString = options.GetValueOrDefault(SIA_REDUNDANCY); + if (!string.IsNullOrWhiteSpace(redundancyString)) + m_redundancy = (float)decimal.Parse(redundancyString, CultureInfo.InvariantCulture); if (m_apiport <= 0) m_apiport = 9980; - if (options.ContainsKey(SIA_TARGETPATH)) - { - m_targetpath = options[SIA_TARGETPATH]; - } + var targetPathString = options.GetValueOrDefault(SIA_TARGETPATH); + if (!string.IsNullOrWhiteSpace(targetPathString)) + m_targetpath = targetPathString; + while (m_targetpath.Contains("//")) m_targetpath = m_targetpath.Replace("//", "/"); while (m_targetpath.StartsWith("/", StringComparison.Ordinal)) @@ -82,14 +85,14 @@ namespace Duplicati.Library.Backend.Sia m_targetpath = "backup"; m_authorization = options.ContainsKey(SIA_PASSWORD) && !string.IsNullOrEmpty(options[SIA_PASSWORD]) - ? "Basic " + System.Convert.ToBase64String(System.Text.Encoding.GetEncoding("ISO-8859-1").GetBytes(":" + options[SIA_PASSWORD])) + ? "Basic " + Convert.ToBase64String(System.Text.Encoding.GetEncoding("ISO-8859-1").GetBytes(":" + options[SIA_PASSWORD])) : null; } - private System.Net.HttpWebRequest CreateRequest(string endpoint) + private HttpWebRequest CreateRequest(string endpoint) { string baseurl = "http://" + m_apihost + ":" + m_apiport; - System.Net.HttpWebRequest req = (System.Net.HttpWebRequest)System.Net.HttpWebRequest.Create(baseurl + endpoint); + var req = (HttpWebRequest)WebRequest.Create(baseurl + endpoint); if (m_authorization != null) { @@ -103,27 +106,25 @@ namespace Duplicati.Library.Backend.Sia return req; } - private string getResponseBodyOnError(string context, System.Net.WebException wex) + private async Task GetResponseBodyOnError(string context, WebException wex, CancellationToken cancellationToken) { - HttpWebResponse response = wex.Response as HttpWebResponse; + var response = wex.Response as HttpWebResponse; if (response is null) - { return $"{context} failed with error: {wex.Message}"; - } - string body = ""; - using (System.IO.Stream data = response.GetResponseStream()) - using (var reader = new System.IO.StreamReader(data)) + var body = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancellationToken, _ => { - body = reader.ReadToEnd(); - } + using (var data = response.GetResponseStream()) + using (var reader = new StreamReader(data)) + return reader.ReadToEnd(); + }).ConfigureAwait(false); return string.Format("{0} failed, response: {1}", context, body); } public class SiaFile { [JsonProperty("siapath")] - public string Siapath { get; set; } + public string? Siapath { get; set; } [JsonProperty("available")] public bool Available { get; set; } [JsonProperty("filesize")] @@ -137,29 +138,29 @@ namespace Duplicati.Library.Backend.Sia public class SiaFileList { [JsonProperty("files")] - public SiaFile[] Files { get; set; } + public SiaFile[]? Files { get; set; } } public class SiaDownloadFile { [JsonProperty("siapath")] - public string Siapath { get; set; } + public string? Siapath { get; set; } [JsonProperty("destination")] - public string Destination { get; set; } + public string? Destination { get; set; } [JsonProperty("filesize")] public long Filesize { get; set; } [JsonProperty("received")] public long Received { get; set; } [JsonProperty("starttime")] - public string Starttime { get; set; } + public string? Starttime { get; set; } [JsonProperty("error")] - public string Error { get; set; } + public string? Error { get; set; } } public class SiaDownloadList { [JsonProperty("downloads")] - public SiaDownloadFile[] Files { get; set; } + public SiaDownloadFile[]? Files { get; set; } } private async Task GetFiles(CancellationToken cancelToken) @@ -173,29 +174,28 @@ namespace Duplicati.Library.Backend.Sia try { var req = CreateRequest(endpoint); - req.Method = System.Net.WebRequestMethods.Http.Get; + req.Method = WebRequestMethods.Http.Get; - var areq = new Utility.AsyncHttpRequest(req); + var areq = new AsyncHttpRequest(req); - using (var resp = (System.Net.HttpWebResponse)areq.GetResponse()) + using (var resp = await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, _ => (HttpWebResponse)areq.GetResponse()).ConfigureAwait(false)) { int code = (int)resp.StatusCode; if (code < 200 || code >= 300) - throw new System.Net.WebException(resp.StatusDescription, null, System.Net.WebExceptionStatus.ProtocolError, resp); + throw new WebException(resp.StatusDescription, null, WebExceptionStatus.ProtocolError, resp); var serializer = new JsonSerializer(); - using (var rs = areq.GetResponseStream()) - using (var sr = new System.IO.StreamReader(rs)) - using (var jr = new Newtonsoft.Json.JsonTextReader(sr)) - { - fl = (SiaFileList)serializer.Deserialize(jr, typeof(SiaFileList)); - } + using (var rs = await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, _ => areq.GetResponseStream()).ConfigureAwait(false)) + using (var sr = new StreamReader(rs)) + using (var jr = new JsonTextReader(sr)) + fl = serializer.Deserialize(jr) + ?? throw new Exception("Failed to deserialize response"); } } - catch (System.Net.WebException wex) + catch (WebException wex) { - throw new Exception(getResponseBodyOnError(endpoint, wex)); + throw new Exception(await GetResponseBodyOnError(endpoint, wex, cancelToken).ConfigureAwait(false)); } return fl; } @@ -219,44 +219,43 @@ namespace Duplicati.Library.Backend.Sia return false; } - private SiaDownloadList GetDownloads() + private async Task GetDownloads(CancellationToken cancelToken) { var fl = new SiaDownloadList(); string endpoint = "/renter/downloads"; try { - System.Net.HttpWebRequest req = CreateRequest(endpoint); - req.Method = System.Net.WebRequestMethods.Http.Get; + var req = CreateRequest(endpoint); + req.Method = WebRequestMethods.Http.Get; - Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req); + var areq = new AsyncHttpRequest(req); - using (System.Net.HttpWebResponse resp = (System.Net.HttpWebResponse)areq.GetResponse()) + using (var resp = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => (HttpWebResponse)areq.GetResponse()).ConfigureAwait(false)) { int code = (int)resp.StatusCode; if (code < 200 || code >= 300) - throw new System.Net.WebException(resp.StatusDescription, null, System.Net.WebExceptionStatus.ProtocolError, resp); + throw new WebException(resp.StatusDescription, null, WebExceptionStatus.ProtocolError, resp); var serializer = new JsonSerializer(); - using (var rs = areq.GetResponseStream()) - using (var sr = new System.IO.StreamReader(rs)) - using (var jr = new Newtonsoft.Json.JsonTextReader(sr)) - { - fl = (SiaDownloadList)serializer.Deserialize(jr, typeof(SiaDownloadList)); - } + using (var rs = await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, _ => areq.GetResponseStream())) + using (var sr = new StreamReader(rs)) + using (var jr = new JsonTextReader(sr)) + fl = serializer.Deserialize(jr) + ?? throw new Exception("Failed to deserialize response"); } } - catch (System.Net.WebException wex) + catch (WebException wex) { - throw new Exception(getResponseBodyOnError(endpoint, wex)); + throw new Exception(await GetResponseBodyOnError(endpoint, wex, cancelToken).ConfigureAwait(false)); } return fl; } - private bool IsDownloadComplete(string siafilename, string localname) + private async Task IsDownloadComplete(string siafilename, string localname, CancellationToken cancelToken) { - SiaDownloadList fl = GetDownloads(); + var fl = await GetDownloads(cancelToken).ConfigureAwait(false); if (fl.Files == null) return false; @@ -273,10 +272,10 @@ namespace Duplicati.Library.Backend.Sia try { // Sia seems to keep the file open/locked for a while, make sure we can open it - System.IO.FileStream fs = new System.IO.FileStream(localname, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.ReadWrite); - fs.Close(); + using (var fs = new FileStream(localname, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) + fs.Close(); } - catch (System.IO.IOException) + catch (IOException) { return false; } @@ -298,15 +297,9 @@ namespace Duplicati.Library.Backend.Sia return Task.CompletedTask; } - public string DisplayName - { - get { return Strings.Sia.DisplayName; } - } + public string DisplayName => Strings.Sia.DisplayName; - public string ProtocolKey - { - get { return "sia"; } - } + public string ProtocolKey => "sia"; /// public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) @@ -316,7 +309,7 @@ namespace Duplicati.Library.Backend.Sia { fl = await GetFiles(cancelToken).ConfigureAwait(false); } - catch (System.Net.WebException wex) + catch (WebException wex) { throw new Exception("failed to call /renter/files " + wex.Message); } @@ -327,9 +320,9 @@ namespace Duplicati.Library.Backend.Sia { // Sia returns a complete file list, but we're only interested in files that are // in our target path - if (f.Siapath.StartsWith(m_targetpath, StringComparison.Ordinal)) + if (f.Siapath != null && f.Siapath.StartsWith(m_targetpath, StringComparison.Ordinal)) { - FileEntry fe = new FileEntry(f.Siapath.Substring(m_targetpath.Length + 1)) + var fe = new FileEntry(f.Siapath.Substring(m_targetpath.Length + 1)) { Size = f.Filesize, IsFolder = false @@ -370,7 +363,7 @@ namespace Duplicati.Library.Backend.Sia } catch (WebException wex) { - throw new Exception(getResponseBodyOnError(endpoint, wex)); + throw new Exception(await GetResponseBodyOnError(endpoint, wex, cancelToken).ConfigureAwait(false)); } } @@ -384,27 +377,27 @@ namespace Duplicati.Library.Backend.Sia { endpoint = string.Format("/renter/download/{0}/{1}?destination={2}", m_targetpath, - Library.Utility.Uri.UrlEncode(remotename).Replace("+", "%20"), - Library.Utility.Uri.UrlEncode(tmpfilename).Replace("+", "%20") + Utility.Uri.UrlEncode(remotename).Replace("+", "%20"), + Utility.Uri.UrlEncode(tmpfilename).Replace("+", "%20") ); - System.Net.HttpWebRequest req = CreateRequest(endpoint); - req.Method = System.Net.WebRequestMethods.Http.Get; + var req = CreateRequest(endpoint); + req.Method = WebRequestMethods.Http.Get; - Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req); + var areq = new AsyncHttpRequest(req); - using (System.Net.HttpWebResponse resp = (System.Net.HttpWebResponse)areq.GetResponse()) + using (var resp = (HttpWebResponse)areq.GetResponse()) { int code = (int)resp.StatusCode; if (code < 200 || code >= 300) - throw new System.Net.WebException(resp.StatusDescription, null, System.Net.WebExceptionStatus.ProtocolError, resp); + throw new WebException(resp.StatusDescription, null, WebExceptionStatus.ProtocolError, resp); - while (!IsDownloadComplete(siafile, localname)) + while (!await IsDownloadComplete(siafile, localname, cancelToken).ConfigureAwait(false)) await Task.Delay(5000, cancelToken).ConfigureAwait(false); - System.IO.File.Copy(tmpfilename, localname, true); + File.Copy(tmpfilename, localname, true); try { - System.IO.File.Delete(tmpfilename); + File.Delete(tmpfilename); } catch (Exception) { @@ -412,16 +405,16 @@ namespace Duplicati.Library.Backend.Sia } } } - catch (System.Net.WebException wex) + catch (WebException wex) { - if (wex.Response is HttpWebResponse response && response.StatusCode == System.Net.HttpStatusCode.NotFound) + if (wex.Response is HttpWebResponse response && response.StatusCode == HttpStatusCode.NotFound) throw new FileMissingException(wex); else - throw new Exception(getResponseBodyOnError(endpoint, wex)); + throw new Exception(await GetResponseBodyOnError(endpoint, wex, cancelToken).ConfigureAwait(false)); } } - public Task DeleteAsync(string remotename, CancellationToken cancellationToken) + public async Task DeleteAsync(string remotename, CancellationToken cancellationToken) { string endpoint = ""; @@ -429,51 +422,41 @@ namespace Duplicati.Library.Backend.Sia { endpoint = string.Format("/renter/delete/{0}/{1}", m_targetpath, - Library.Utility.Uri.UrlEncode(remotename).Replace("+", "%20") + Utility.Uri.UrlEncode(remotename).Replace("+", "%20") ); - System.Net.HttpWebRequest req = CreateRequest(endpoint); - req.Method = System.Net.WebRequestMethods.Http.Post; + var req = CreateRequest(endpoint); + req.Method = WebRequestMethods.Http.Post; - Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req); - - using (System.Net.HttpWebResponse resp = (System.Net.HttpWebResponse)areq.GetResponse()) + await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancellationToken, _ => { - int code = (int)resp.StatusCode; - if (code < 200 || code >= 300) - throw new System.Net.WebException(resp.StatusDescription, null, System.Net.WebExceptionStatus.ProtocolError, resp); - } + var areq = new AsyncHttpRequest(req); + using (var resp = (HttpWebResponse)areq.GetResponse()) + { + int code = (int)resp.StatusCode; + if (code < 200 || code >= 300) + throw new WebException(resp.StatusDescription, null, WebExceptionStatus.ProtocolError, resp); + } + }).ConfigureAwait(false); } - catch (System.Net.WebException wex) + catch (WebException wex) { - if (wex.Response is HttpWebResponse response && response.StatusCode == System.Net.HttpStatusCode.NotFound) + if (wex.Response is HttpWebResponse response && response.StatusCode == HttpStatusCode.NotFound) throw new FileMissingException(wex); else - throw new Exception(getResponseBodyOnError(endpoint, wex)); + throw new Exception(await GetResponseBodyOnError(endpoint, wex, cancellationToken).ConfigureAwait(false)); } - - return Task.CompletedTask; } - public IList SupportedCommands - { - get - { - return new List([ + public IList SupportedCommands => new List([ new CommandLineArgument(SIA_TARGETPATH, CommandLineArgument.ArgumentType.String, Strings.Sia.SiaPathDescriptionShort, Strings.Sia.SiaPathDescriptionLong, "/backup"), new CommandLineArgument(SIA_PASSWORD, CommandLineArgument.ArgumentType.Password, Strings.Sia.SiaPasswordShort, Strings.Sia.SiaPasswordLong, null), new CommandLineArgument(SIA_REDUNDANCY, CommandLineArgument.ArgumentType.Decimal, Strings.Sia.SiaRedundancyDescriptionShort, Strings.Sia.SiaRedundancyDescriptionLong, "1.5"), + .. TimeoutOptionsHelper.GetOptions() + .Where(x => x.Name != TimeoutOptionsHelper.READ_WRITE_TIMEOUT_OPTION) ]); - } - } - public string Description - { - get - { - return Strings.Sia.Description; - } - } + public string Description => Strings.Sia.Description; public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new[] { new System.Uri(m_apihost).Host }); diff --git a/Duplicati/Library/Backend/Sia/Strings.cs b/Duplicati/Library/Backend/Sia/Strings.cs index 191a8b43a..268bde58c 100644 --- a/Duplicati/Library/Backend/Sia/Strings.cs +++ b/Duplicati/Library/Backend/Sia/Strings.cs @@ -23,13 +23,13 @@ namespace Duplicati.Library.Backend.Strings { internal static class Sia { - public static string Description { get { return LC.L(@"This backend can read and write data to Sia."); } } - public static string DisplayName { get { return LC.L(@"Sia Decentralized Cloud"); } } - public static string SiaPathDescriptionLong { get { return LC.L(@"Set the target path. Example: /backup"); } } - public static string SiaPathDescriptionShort { get { return LC.L(@"Backup path"); } } - public static string SiaPasswordLong { get { return LC.L(@"Supply a password for Sia server."); } } - public static string SiaPasswordShort { get { return LC.L(@"Sia password"); } } - public static string SiaRedundancyDescriptionLong { get { return LC.L(@"The minimum value for redundancy is 1.0."); } } - public static string SiaRedundancyDescriptionShort { get { return LC.L(@"Set the minimum redundancy"); } } + public static string Description => LC.L(@"This backend can read and write data to Sia."); + public static string DisplayName => LC.L(@"Sia Decentralized Cloud"); + public static string SiaPathDescriptionLong => LC.L(@"Set the target path. Example: /backup"); + public static string SiaPathDescriptionShort => LC.L(@"Backup path"); + public static string SiaPasswordLong => LC.L(@"Supply a password for Sia server."); + public static string SiaPasswordShort => LC.L(@"Sia password"); + public static string SiaRedundancyDescriptionLong => LC.L(@"The minimum value for redundancy is 1.0."); + public static string SiaRedundancyDescriptionShort => LC.L(@"Set the minimum redundancy"); } } From a15b123d4b4e690adc4846266c6895d86ffa9c73 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sat, 22 Mar 2025 11:19:58 +0100 Subject: [PATCH 31/48] Udated the SSH backend --- .../Duplicati.Library.Backend.SSHv2.csproj | 2 + .../Library/Backend/SSHv2/KeyGenerator.cs | 159 +++++------- .../Library/Backend/SSHv2/KeyUploader.cs | 143 +++++----- .../Library/Backend/SSHv2/SSHv2Backend.cs | 245 ++++++++---------- Duplicati/Library/Backend/SSHv2/Strings.cs | 65 +++-- 5 files changed, 282 insertions(+), 332 deletions(-) diff --git a/Duplicati/Library/Backend/SSHv2/Duplicati.Library.Backend.SSHv2.csproj b/Duplicati/Library/Backend/SSHv2/Duplicati.Library.Backend.SSHv2.csproj index 7c7bbd948..a4427a708 100644 --- a/Duplicati/Library/Backend/SSHv2/Duplicati.Library.Backend.SSHv2.csproj +++ b/Duplicati/Library/Backend/SSHv2/Duplicati.Library.Backend.SSHv2.csproj @@ -5,6 +5,8 @@ Library Duplicati.Library.Backend Copyright © 2025 Team Duplicati, MIT license + enable + enable diff --git a/Duplicati/Library/Backend/SSHv2/KeyGenerator.cs b/Duplicati/Library/Backend/SSHv2/KeyGenerator.cs index 600e57ebd..05ac23ca4 100644 --- a/Duplicati/Library/Backend/SSHv2/KeyGenerator.cs +++ b/Duplicati/Library/Backend/SSHv2/KeyGenerator.cs @@ -1,31 +1,27 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System; -using System.Collections.Generic; using System.Security.Cryptography; using Duplicati.Library.Interface; -using Duplicati.Library.Utility; using System.Text; -using System.Linq; namespace Duplicati.Library.Backend { @@ -34,19 +30,19 @@ namespace Duplicati.Library.Backend private const string KEY_TYPE_NAME = "key-type"; private const string KEY_USERNAME = "key-username"; private const string KEY_KEYLEN = "key-bits"; - + private const string KEY_TEMPLATE_DSA = "-----BEGIN DSA PRIVATE KEY-----\n{0}\n-----END DSA PRIVATE KEY----- \n"; private const string KEY_TEMPLATE_RSA = "-----BEGIN RSA PRIVATE KEY-----\n{0}\n-----END RSA PRIVATE KEY----- \n"; private const string PUB_KEY_FORMAT_DSA = "ssh-dss"; private const string PUB_KEY_FORMAT_RSA = "ssh-rsa"; - + private const string KEYTYPE_DSA = "dsa"; private const string KEYTYPE_RSA = "rsa"; - + private readonly string DEFAULT_USERNAME = "backup-user@" + System.Environment.MachineName; - private const string DEFAULT_KEYTYPE = KEYTYPE_DSA; + private const string DEFAULT_KEYTYPE = KEYTYPE_DSA; private const int DEFAULT_KEYLEN = 1024; - + private static void EncodePEMLength(System.IO.Stream s, uint length) { s.WriteByte((byte)((length >> 24) & 0xff)); @@ -54,7 +50,7 @@ namespace Duplicati.Library.Backend s.WriteByte((byte)((length >> 8) & 0xff)); s.WriteByte((byte)(length & 0xff)); } - + private static void EncodeDERLength(System.IO.Stream s, uint length) { if (length < 0x7f) @@ -76,27 +72,27 @@ namespace Duplicati.Library.Backend s.WriteByte((byte)(length & 0xff)); } } - + private static byte[] EncodeDER(byte[][] data) { byte[] payload; - using(var ms = new System.IO.MemoryStream()) + using (var ms = new MemoryStream()) { - foreach(var b in data) + foreach (var b in data) { ms.WriteByte(0x02); var isNegative = (b[0] & 0x80) != 0; - + EncodeDERLength(ms, (uint)(b.Length + (isNegative ? 1 : 0))); if (isNegative) ms.WriteByte(0); - ms.Write(b, 0, b.Length); + ms.Write(b, 0, b.Length); } - + payload = ms.ToArray(); } - - using(var ms = new System.IO.MemoryStream()) + + using (var ms = new MemoryStream()) { ms.WriteByte(0x30); EncodeDERLength(ms, (uint)payload.Length); @@ -104,12 +100,12 @@ namespace Duplicati.Library.Backend return ms.ToArray(); } } - + private static byte[] EncodePEM(byte[][] data) { - using(var ms = new System.IO.MemoryStream()) + using (var ms = new System.IO.MemoryStream()) { - foreach(var n in data) + foreach (var n in data) { var isNegative = (n[0] & 0x80) != 0; EncodePEMLength(ms, (uint)(n.Length + (isNegative ? 1 : 0))); @@ -120,20 +116,20 @@ namespace Duplicati.Library.Backend return ms.ToArray(); } } - + private static IDictionary OutputKey(byte[] private_key, byte[] public_key, string pem_template, string key_name, string username) { var res = new Dictionary(); - + var b64_raw = Convert.ToBase64String(private_key); var sb = new StringBuilder(); var lw = 64; - for(var i = 0; i < b64_raw.Length; i += lw) + for (var i = 0; i < b64_raw.Length; i += lw) { sb.Append(b64_raw.Substring(i, Math.Min(lw, b64_raw.Length - i))); sb.Append("\n"); } - + var b64 = sb.ToString().Trim(); var pem = string.Format(pem_template, b64); var uri = SSHv2.KEYFILE_URI + Duplicati.Library.Utility.Uri.UrlEncode(pem); @@ -145,24 +141,20 @@ namespace Duplicati.Library.Backend res["pubkey"] = pub; return res; } - + #region IWebModule implementation public IDictionary Execute(IDictionary options) - { - string keytype; - if (!options.TryGetValue(KEY_TYPE_NAME, out keytype)) + { + if (!options.TryGetValue(KEY_TYPE_NAME, out var keytype)) keytype = DEFAULT_KEYTYPE; - string username; - if (!options.TryGetValue(KEY_USERNAME, out username)) + if (!options.TryGetValue(KEY_USERNAME, out var username)) username = DEFAULT_USERNAME; - string keylen_s; - if (!options.TryGetValue(KEY_KEYLEN, out keylen_s)) + if (!options.TryGetValue(KEY_KEYLEN, out var keylen_s)) keylen_s = "0"; - int keylen; - if (!int.TryParse(keylen_s, out keylen)) + if (!int.TryParse(keylen_s, out var keylen)) keylen = DEFAULT_KEYLEN; if (KEYTYPE_RSA.Equals(keytype, StringComparison.OrdinalIgnoreCase)) @@ -172,38 +164,37 @@ namespace Duplicati.Library.Backend rsa.KeySize = keylen; else rsa.KeySize = DEFAULT_KEYLEN; - + var key = rsa.ExportParameters(true); - - var privateEntries = new byte[][] { new byte[]{ 0x0 }, key.Modulus, key.Exponent, key.D, key.P, key.Q, key.DP, key.DQ, key.InverseQ }; - var publicEntries = new byte[][] { - System.Text.Encoding.ASCII.GetBytes(PUB_KEY_FORMAT_RSA), - key.Exponent, - key.Modulus + + var privateEntries = new byte[][] { new byte[] { 0x0 }, key.Modulus ?? [], key.Exponent ?? [], key.D ?? [], key.P ?? [], key.Q ?? [], key.DP ?? [], key.DQ ?? [], key.InverseQ ?? [] }; + var publicEntries = new byte[][] { + Encoding.ASCII.GetBytes(PUB_KEY_FORMAT_RSA), + key.Exponent ?? [], + key.Modulus ?? [] }; - + return OutputKey(EncodeDER(privateEntries), EncodePEM(publicEntries), KEY_TEMPLATE_RSA, PUB_KEY_FORMAT_RSA, username); } else if (KEYTYPE_DSA.Equals(keytype, StringComparison.OrdinalIgnoreCase)) { - var dsa = DSACryptoServiceProvider.Create(); if (keylen > 0) dsa.KeySize = keylen; else dsa.KeySize = DEFAULT_KEYLEN; - + var key = dsa.ExportParameters(true); - - var privateEntries = new byte[][] { new byte[] { 0x0 }, key.P, key.Q, key.G, key.Y, key.X }; + + var privateEntries = new byte[][] { new byte[] { 0x0 }, key.P ?? [], key.Q ?? [], key.G ?? [], key.Y ?? [], key.X ?? [] }; var publicEntries = new byte[][] { - System.Text.Encoding.ASCII.GetBytes(PUB_KEY_FORMAT_DSA), - key.P, - key.Q, - key.G, - key.Y + Encoding.ASCII.GetBytes(PUB_KEY_FORMAT_DSA), + key.P ?? [], + key.Q ?? [], + key.G ?? [], + key.Y ?? [] }; - + return OutputKey(EncodeDER(privateEntries), EncodePEM(publicEntries), KEY_TEMPLATE_DSA, PUB_KEY_FORMAT_DSA, username); } else @@ -211,21 +202,15 @@ namespace Duplicati.Library.Backend throw new UserInformationException(string.Format("Unsupported key type: {0}", keytype), "SSHUnsupportedKey"); } } - public string Key { get { return "ssh-keygen"; } } - - public string DisplayName { get { return Strings.KeyGenerator.DisplayName; } } - public string Description { get { return Strings.KeyGenerator.Description; } } - public IList SupportedCommands - { - get - { - return new List(new ICommandLineArgument[] { - new CommandLineArgument(KEY_KEYLEN, CommandLineArgument.ArgumentType.Integer, Strings.KeyGenerator.KeyLenShort, Strings.KeyGenerator.KeyLenLong, DEFAULT_KEYLEN.ToString(), null, new string[] {"1024", "2048"}), - new CommandLineArgument(KEY_TYPE_NAME, CommandLineArgument.ArgumentType.Enumeration, Strings.KeyGenerator.KeyTypeShort, Strings.KeyGenerator.KeyTypeLong, DEFAULT_KEYTYPE, null, new string[] {KEYTYPE_DSA, KEYTYPE_RSA}), - new CommandLineArgument(KEY_USERNAME, CommandLineArgument.ArgumentType.Integer, Strings.KeyGenerator.KeyUsernameShort, Strings.KeyGenerator.KeyUsernameLong, DEFAULT_USERNAME), - }); - } - } + public string Key => "ssh-keygen"; + + public string DisplayName => Strings.KeyGenerator.DisplayName; + public string Description => Strings.KeyGenerator.Description; + public IList SupportedCommands => [ + new CommandLineArgument(KEY_KEYLEN, CommandLineArgument.ArgumentType.Integer, Strings.KeyGenerator.KeyLenShort, Strings.KeyGenerator.KeyLenLong, DEFAULT_KEYLEN.ToString(), null, new string[] {"1024", "2048"}), + new CommandLineArgument(KEY_TYPE_NAME, CommandLineArgument.ArgumentType.Enumeration, Strings.KeyGenerator.KeyTypeShort, Strings.KeyGenerator.KeyTypeLong, DEFAULT_KEYTYPE, null, new string[] {KEYTYPE_DSA, KEYTYPE_RSA}), + new CommandLineArgument(KEY_USERNAME, CommandLineArgument.ArgumentType.Integer, Strings.KeyGenerator.KeyUsernameShort, Strings.KeyGenerator.KeyUsernameLong, DEFAULT_USERNAME), + ]; #endregion } } diff --git a/Duplicati/Library/Backend/SSHv2/KeyUploader.cs b/Duplicati/Library/Backend/SSHv2/KeyUploader.cs index a595bc9d4..24a00af5b 100644 --- a/Duplicati/Library/Backend/SSHv2/KeyUploader.cs +++ b/Duplicati/Library/Backend/SSHv2/KeyUploader.cs @@ -1,29 +1,26 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System; -using System.Collections.Generic; using Duplicati.Library.Interface; using Duplicati.Library.Utility; -using System.Linq; namespace Duplicati.Library.Backend { @@ -31,46 +28,44 @@ namespace Duplicati.Library.Backend { private const string OPTION_URL = "ssh-url"; private const string OPTION_KEY = "ssh-pubkey"; - + private const string AUTHORIZED_KEYS_FILE = "authorized_keys"; private const string SSH_FOLDER = ".ssh"; private const string AUTHORIZED_KEYS_PATH = SSH_FOLDER + "/" + AUTHORIZED_KEYS_FILE; - + private const short SSH_FOLDER_PERMISSIONS = 755;// 0x1ED /*755*/; private const short AUTHORIZED_KEYS_PERMISSIONS = 644; //0x1A4 /*644*/; private const short AUTHORIZED_KEYS_BACKUP_PERMISSIONS = 600; //0x180 /*600*/; - + #region IWebModule implementation public IDictionary Execute(IDictionary options) - { + { var res = new Dictionary(); - - string url; - string pubkey_s; - - options.TryGetValue(OPTION_URL, out url); - options.TryGetValue(OPTION_KEY, out pubkey_s); - + + options.TryGetValue(OPTION_URL, out var url); + options.TryGetValue(OPTION_KEY, out var pubkey_s); + if (string.IsNullOrWhiteSpace(url)) throw new ArgumentException(OPTION_URL); if (string.IsNullOrWhiteSpace(pubkey_s)) throw new ArgumentException(OPTION_KEY); - - var uri = new Utility.Uri(url); - foreach(var key in uri.QueryParameters.AllKeys) - options[key] = uri.QueryParameters[key]; - - + + var uri = new Utility.Uri(url); + foreach (var key in uri.QueryParameters.AllKeys) + if (!string.IsNullOrWhiteSpace(key)) + options[key] = uri.QueryParameters[key] ?? ""; + + pubkey_s = pubkey_s.Trim(); var pubkey = pubkey_s.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries).Select(x => x.Trim()).Where(x => x.Length > 0).ToArray(); - + if (pubkey.Length != 3) throw new ArgumentException(OPTION_KEY); - - using(var connection = new SSHv2(url, (Dictionary)options)) + + using (var connection = new SSHv2(url, (Dictionary)options)) { - var client = connection.Client; + var client = connection.CreateConnection(CancellationToken.None).Await(); try { client.ChangeDirectory(SSH_FOLDER); @@ -81,30 +76,30 @@ namespace Duplicati.Library.Backend client.ChangePermissions(SSH_FOLDER, SSH_FOLDER_PERMISSIONS); client.ChangeDirectory(SSH_FOLDER); } - - var sshfolder = client.ListDirectory(".").First(x => x.Name == "."); + + var sshfolder = client.ListDirectory(".").First(x => x.Name == "."); client.ChangeDirectory(".."); - + if (!sshfolder.OwnerCanRead || !sshfolder.OwnerCanWrite) client.ChangePermissions(SSH_FOLDER, SSH_FOLDER_PERMISSIONS); - - string authorized_keys = ""; - byte[] authorized_keys_bytes = null; - + + var authorized_keys = ""; + byte[]? authorized_keys_bytes = null; + var existing_authorized_keys = client.ListDirectory(SSH_FOLDER).Any(x => x.Name == AUTHORIZED_KEYS_FILE); if (existing_authorized_keys) { - using(var ms = new System.IO.MemoryStream()) + using (var ms = new MemoryStream()) { client.DownloadFile(AUTHORIZED_KEYS_PATH, ms); authorized_keys_bytes = ms.ToArray(); authorized_keys = System.Text.Encoding.ASCII.GetString(authorized_keys_bytes); } } - - var keys = authorized_keys == null ? new string[0] : authorized_keys.Split(new char[] { '\n' }, StringSplitOptions.RemoveEmptyEntries); + + var keys = authorized_keys.Split(new char[] { '\n' }, StringSplitOptions.RemoveEmptyEntries); var cleaned_keys = keys.Select(x => x.Trim()).Where(x => x.Length > 0 && !x.StartsWith("#", StringComparison.Ordinal)); - + // Does the key already exist? if (cleaned_keys.Any(x => { @@ -120,43 +115,37 @@ namespace Duplicati.Library.Backend if (new_file.Trim().Length > 0) new_file = authorized_keys.Trim() + "\n"; new_file += string.Join(" ", pubkey) + "\n"; - - if (existing_authorized_keys) + + if (existing_authorized_keys && authorized_keys_bytes != null) { - var filename = AUTHORIZED_KEYS_PATH + ".backup-" + Library.Utility.Utility.SerializeDateTime(DateTime.UtcNow); - using(var ms = new System.IO.MemoryStream(authorized_keys_bytes)) + var filename = AUTHORIZED_KEYS_PATH + ".backup-" + Utility.Utility.SerializeDateTime(DateTime.UtcNow); + using (var ms = new MemoryStream(authorized_keys_bytes)) client.UploadFile(ms, filename); client.ChangePermissions(filename, AUTHORIZED_KEYS_BACKUP_PERMISSIONS); } - - using(var ms = new System.IO.MemoryStream(System.Text.Encoding.ASCII.GetBytes(new_file))) + + using (var ms = new MemoryStream(System.Text.Encoding.ASCII.GetBytes(new_file))) client.UploadFile(ms, AUTHORIZED_KEYS_PATH); - + if (!existing_authorized_keys) client.ChangePermissions(AUTHORIZED_KEYS_PATH, AUTHORIZED_KEYS_PERMISSIONS); - + res["status"] = "Key updated"; } - + } - + return res; } - public string Key { get { return "ssh-keyupload"; } } - public string DisplayName { get { return Strings.KeyUploader.DisplayName; } } - public string Description { get { return Strings.KeyUploader.Description; } } + public string Key => "ssh-keyupload"; + public string DisplayName => Strings.KeyUploader.DisplayName; + public string Description => Strings.KeyUploader.Description; - public IList SupportedCommands - { - get - { - return new List(new ICommandLineArgument[] { - new CommandLineArgument(OPTION_URL, CommandLineArgument.ArgumentType.String, Strings.KeyUploader.UrlShort, Strings.KeyUploader.UrlLong), - new CommandLineArgument(OPTION_KEY, CommandLineArgument.ArgumentType.String, Strings.KeyUploader.PubkeyShort, Strings.KeyUploader.PubkeyLong), - }.Union(new SSHv2().SupportedCommands).ToList()); - } - } + public IList SupportedCommands => new ICommandLineArgument[] { + new CommandLineArgument(OPTION_URL, CommandLineArgument.ArgumentType.String, Strings.KeyUploader.UrlShort, Strings.KeyUploader.UrlLong), + new CommandLineArgument(OPTION_KEY, CommandLineArgument.ArgumentType.String, Strings.KeyUploader.PubkeyShort, Strings.KeyUploader.PubkeyLong), + }.Union(new SSHv2().SupportedCommands).ToList(); #endregion } diff --git a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs index 518273565..e368ba2d6 100644 --- a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs +++ b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs @@ -23,15 +23,12 @@ using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using Duplicati.Library.SourceProvider; +using Duplicati.Library.Utility; +using Duplicati.Library.Utility.Options; using Renci.SshNet; using Renci.SshNet.Common; -using System; -using System.Collections.Generic; using System.Globalization; -using System.Linq; using System.Runtime.CompilerServices; -using System.Threading; -using System.Threading.Tasks; namespace Duplicati.Library.Backend { @@ -46,54 +43,56 @@ namespace Duplicati.Library.Backend private const string SSH_TIMEOUT_OPTION = "ssh-operation-timeout"; private const string SSH_KEEPALIVE_OPTION = "ssh-keepalive"; - readonly Dictionary m_options; + readonly Dictionary m_options; private readonly string m_server; private readonly string m_path; private readonly string m_username; - private readonly string m_password; - private readonly string m_fingerprint; + private readonly string? m_password; + private readonly string? m_fingerprint; private readonly bool m_fingerprintallowall; private readonly TimeSpan m_operationtimeout; private readonly TimeSpan m_keepaliveinterval; private readonly int m_port = 22; private readonly bool m_useRelativePath; + private readonly TimeoutOptionsHelper.Timeouts m_timeouts; private string m_initialDirectory; - private SftpClient m_con; + private SftpClient? m_con; public SSHv2() { + m_server = null!; + m_path = null!; + m_username = null!; + m_options = null!; + m_timeouts = null!; + m_con = null!; + m_initialDirectory = null!; } - public SSHv2(string url, Dictionary options) + public SSHv2(string url, Dictionary options) : this() { m_options = options; var uri = new Utility.Uri(url); uri.RequireHost(); - if (options.ContainsKey("auth-username")) - m_username = options["auth-username"]; - if (options.ContainsKey("auth-password")) - m_password = options["auth-password"]; - if (options.ContainsKey(SSH_FINGERPRINT_OPTION)) - m_fingerprint = options[SSH_FINGERPRINT_OPTION]; - if (!string.IsNullOrEmpty(uri.Username)) - m_username = uri.Username; - if (!string.IsNullOrEmpty(uri.Password)) - m_password = uri.Password; + var auth = AuthOptionsHelper.Parse(options, uri); + if (!auth.HasUsername) + throw new UserInformationException(Strings.SSHv2Backend.UsernameRequired, "UsernameNotSpecified"); + m_username = auth.Username!; + m_password = auth.Password; + m_fingerprint = options.GetValueOrDefault(SSH_FINGERPRINT_OPTION); m_fingerprintallowall = Utility.Utility.ParseBoolOption(options, SSH_FINGERPRINT_ACCEPT_ANY_OPTION); m_useRelativePath = Utility.Utility.ParseBoolOption(options, SSH_RELATIVE_PATH_OPTION); m_path = uri.Path; if (!string.IsNullOrWhiteSpace(m_path)) - { m_path = Util.AppendDirSeparator(m_path, "/"); - } if (m_useRelativePath) { @@ -110,16 +109,8 @@ namespace Duplicati.Library.Backend if (uri.Port > 0) m_port = uri.Port; - string timeoutstr; - options.TryGetValue(SSH_TIMEOUT_OPTION, out timeoutstr); - - if (!string.IsNullOrWhiteSpace(timeoutstr)) - m_operationtimeout = Library.Utility.Timeparser.ParseTimeSpan(timeoutstr); - - options.TryGetValue(SSH_KEEPALIVE_OPTION, out timeoutstr); - - if (!string.IsNullOrWhiteSpace(timeoutstr)) - m_keepaliveinterval = Library.Utility.Timeparser.ParseTimeSpan(timeoutstr); + m_keepaliveinterval = Utility.Utility.ParseTimespanOption(options, SSH_KEEPALIVE_OPTION, "0s"); + m_operationtimeout = Utility.Utility.ParseTimespanOption(options, SSH_TIMEOUT_OPTION, "0s"); } #region IBackend Members @@ -129,7 +120,7 @@ namespace Duplicati.Library.Backend public async Task CreateFolderAsync(CancellationToken cancelToken) { - await CreateConnection(cancelToken).ConfigureAwait(false); + var con = await CreateConnection(cancelToken).ConfigureAwait(false); // Since the SftpClient.CreateDirectory method does not create all the parent directories // as needed, this has to be done manually. @@ -137,16 +128,16 @@ namespace Duplicati.Library.Backend foreach (string part in m_path.Split('/').Where(x => !String.IsNullOrEmpty(x))) { partialPath += $"/{part}"; - if (await this.m_con.ExistsAsync(partialPath, cancelToken)) + if (await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => con.ExistsAsync(partialPath, ct)).ConfigureAwait(false)) { - if (!this.m_con.GetAttributes(partialPath).IsDirectory) + if (!await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => con.GetAttributes(partialPath).IsDirectory).ConfigureAwait(false)) { throw new ArgumentException($"The path {partialPath} already exists and is not a directory."); } } else { - await this.m_con.CreateDirectoryAsync(partialPath, cancelToken).ConfigureAwait(false); + await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => con.CreateDirectoryAsync(partialPath, ct)).ConfigureAwait(false); } } @@ -158,15 +149,15 @@ namespace Duplicati.Library.Backend public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) { - using (System.IO.FileStream fs = System.IO.File.Open(filename, System.IO.FileMode.Open, - System.IO.FileAccess.Read, System.IO.FileShare.Read)) + using (var fs = File.Open(filename, FileMode.Open, + FileAccess.Read, FileShare.Read)) await PutAsync(remotename, fs, cancelToken).ConfigureAwait(false); } public async Task GetAsync(string remotename, string filename, CancellationToken cancelToken) { - using (var fs = System.IO.File.Open(filename, System.IO.FileMode.Create, - System.IO.FileAccess.Write, System.IO.FileShare.None)) + using (var fs = File.Open(filename, FileMode.Create, + FileAccess.Write, FileShare.None)) await GetAsync(remotename, fs, cancelToken).ConfigureAwait(false); } @@ -174,9 +165,9 @@ namespace Duplicati.Library.Backend { try { - await CreateConnection(cancelToken).ConfigureAwait(false); - await SetWorkingDirectory(cancelToken).ConfigureAwait(false); - await m_con.DeleteFileAsync(remotename, cancelToken).ConfigureAwait(false); + var con = await CreateConnection(cancelToken).ConfigureAwait(false); + await SetWorkingDirectory(con, cancelToken).ConfigureAwait(false); + await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => con.DeleteFileAsync(remotename, ct)).ConfigureAwait(false); } catch (SftpPathNotFoundException ex) { @@ -184,42 +175,31 @@ namespace Duplicati.Library.Backend } } - public IList SupportedCommands - { - get - { - return new List(new ICommandLineArgument[] - { - new CommandLineArgument("auth-password", CommandLineArgument.ArgumentType.Password, - Strings.SSHv2Backend.DescriptionAuthPasswordShort, - Strings.SSHv2Backend.DescriptionAuthPasswordLong), - new CommandLineArgument("auth-username", CommandLineArgument.ArgumentType.String, - Strings.SSHv2Backend.DescriptionAuthUsernameShort, - Strings.SSHv2Backend.DescriptionAuthUsernameLong), - new CommandLineArgument(SSH_FINGERPRINT_OPTION, CommandLineArgument.ArgumentType.String, - Strings.SSHv2Backend.DescriptionFingerprintShort, - Strings.SSHv2Backend.DescriptionFingerprintLong), - new CommandLineArgument(SSH_FINGERPRINT_ACCEPT_ANY_OPTION, CommandLineArgument.ArgumentType.Boolean, - Strings.SSHv2Backend.DescriptionAnyFingerprintShort, - Strings.SSHv2Backend.DescriptionAnyFingerprintLong), - new CommandLineArgument(SSH_KEYFILE_OPTION, CommandLineArgument.ArgumentType.Path, - Strings.SSHv2Backend.DescriptionSshkeyfileShort, - Strings.SSHv2Backend.DescriptionSshkeyfileLong), - new CommandLineArgument(SSH_KEYFILE_INLINE, CommandLineArgument.ArgumentType.Password, - Strings.SSHv2Backend.DescriptionSshkeyShort, - Strings.SSHv2Backend.DescriptionSshkeyLong(KEYFILE_URI)), - new CommandLineArgument(SSH_TIMEOUT_OPTION, CommandLineArgument.ArgumentType.Timespan, - Strings.SSHv2Backend.DescriptionSshtimeoutShort, Strings.SSHv2Backend.DescriptionSshtimeoutLong, - "0"), - new CommandLineArgument(SSH_KEEPALIVE_OPTION, CommandLineArgument.ArgumentType.Timespan, - Strings.SSHv2Backend.DescriptionSshkeepaliveShort, - Strings.SSHv2Backend.DescriptionSshkeepaliveLong, "0"), - new CommandLineArgument(SSH_RELATIVE_PATH_OPTION, CommandLineArgument.ArgumentType.Boolean, - Strings.SSHv2Backend.DescriptionRelativePathShort, - Strings.SSHv2Backend.DescriptionRelativePathLong) - }); - } - } + public IList SupportedCommands => [ + .. AuthOptionsHelper.GetOptions(), + new CommandLineArgument(SSH_FINGERPRINT_OPTION, CommandLineArgument.ArgumentType.String, + Strings.SSHv2Backend.DescriptionFingerprintShort, + Strings.SSHv2Backend.DescriptionFingerprintLong), + new CommandLineArgument(SSH_FINGERPRINT_ACCEPT_ANY_OPTION, CommandLineArgument.ArgumentType.Boolean, + Strings.SSHv2Backend.DescriptionAnyFingerprintShort, + Strings.SSHv2Backend.DescriptionAnyFingerprintLong), + new CommandLineArgument(SSH_KEYFILE_OPTION, CommandLineArgument.ArgumentType.Path, + Strings.SSHv2Backend.DescriptionSshkeyfileShort, + Strings.SSHv2Backend.DescriptionSshkeyfileLong), + new CommandLineArgument(SSH_KEYFILE_INLINE, CommandLineArgument.ArgumentType.Password, + Strings.SSHv2Backend.DescriptionSshkeyShort, + Strings.SSHv2Backend.DescriptionSshkeyLong(KEYFILE_URI)), + new CommandLineArgument(SSH_TIMEOUT_OPTION, CommandLineArgument.ArgumentType.Timespan, + Strings.SSHv2Backend.DescriptionSshtimeoutShort, Strings.SSHv2Backend.DescriptionSshtimeoutLong, + "0", null,null, Strings.SSHv2Backend.TimeoutDeprecated(SSH_TIMEOUT_OPTION, TimeoutOptionsHelper.SHORT_TIMEOUT_OPTION, TimeoutOptionsHelper.LIST_TIMEOUT_OPTION, TimeoutOptionsHelper.READ_WRITE_TIMEOUT_OPTION)), + new CommandLineArgument(SSH_KEEPALIVE_OPTION, CommandLineArgument.ArgumentType.Timespan, + Strings.SSHv2Backend.DescriptionSshkeepaliveShort, + Strings.SSHv2Backend.DescriptionSshkeepaliveLong, "0"), + new CommandLineArgument(SSH_RELATIVE_PATH_OPTION, CommandLineArgument.ArgumentType.Boolean, + Strings.SSHv2Backend.DescriptionRelativePathShort, + Strings.SSHv2Backend.DescriptionRelativePathLong), + .. TimeoutOptionsHelper.GetOptions(), + ]; public string Description => Strings.SSHv2Backend.Description; @@ -251,26 +231,35 @@ namespace Duplicati.Library.Backend #region IStreamingBackend Implementation - public async Task PutAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) + public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken) { - await CreateConnection(cancelToken).ConfigureAwait(false); - await SetWorkingDirectory(cancelToken).ConfigureAwait(false); - await Task.Factory.FromAsync( - (cb, state) => m_con.BeginUploadFile(stream, remotename, cb, state, _ => cancelToken.ThrowIfCancellationRequested()), - m_con.EndUploadFile, - null); + var con = await CreateConnection(cancelToken).ConfigureAwait(false); + await SetWorkingDirectory(con, cancelToken).ConfigureAwait(false); + try + { + using var ts = stream.ObserveReadTimeout(m_timeouts.ReadWriteTimeout, false); + await Task.Factory.FromAsync( + (cb, state) => con.BeginUploadFile(ts, remotename, cb, state, _ => cancelToken.ThrowIfCancellationRequested()), + con.EndUploadFile, + null); + } + catch (SftpPathNotFoundException ex) + { + throw new FolderMissingException(ex); + } } - public async Task GetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) + public async Task GetAsync(string remotename, Stream stream, CancellationToken cancelToken) { - await CreateConnection(cancelToken).ConfigureAwait(false); - await SetWorkingDirectory(cancelToken).ConfigureAwait(false); + var con = await CreateConnection(cancelToken).ConfigureAwait(false); + await SetWorkingDirectory(con, cancelToken).ConfigureAwait(false); try { + using var ts = stream.ObserveWriteTimeout(m_timeouts.ReadWriteTimeout, false); await Task.Factory.FromAsync( - (cb, state) => m_con.BeginDownloadFile(remotename, stream, cb, state, _ => cancelToken.ThrowIfCancellationRequested()), - m_con.EndDownloadFile, + (cb, state) => con.BeginDownloadFile(remotename, ts, cb, state, _ => cancelToken.ThrowIfCancellationRequested()), + con.EndDownloadFile, null).ConfigureAwait(false); } catch (SftpPathNotFoundException ex) @@ -285,42 +274,36 @@ namespace Duplicati.Library.Backend public async Task RenameAsync(string source, string target, CancellationToken cancelToken) { - await CreateConnection(cancelToken).ConfigureAwait(false); - await SetWorkingDirectory(cancelToken).ConfigureAwait(false); - await m_con.RenameFileAsync(source, target, cancelToken).ConfigureAwait(false); + var con = await CreateConnection(cancelToken).ConfigureAwait(false); + await SetWorkingDirectory(con, cancelToken).ConfigureAwait(false); + await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => con.RenameFileAsync(source, target, ct).ConfigureAwait(false)); } #endregion #region Implementation - private async Task CreateConnection(CancellationToken cancelToken) + internal async Task CreateConnection(CancellationToken cancelToken) { if (m_con != null && m_con.IsConnected) - return; + return m_con; if (m_con != null && !m_con.IsConnected) { - await this.TryConnect(m_con, cancelToken).ConfigureAwait(false); - return; + await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => m_con.ConnectAsync(ct)).ConfigureAwait(false); + return m_con; } SftpClient con; m_options.TryGetValue(SSH_KEYFILE_OPTION, out var keyFile); if (string.IsNullOrWhiteSpace(keyFile)) - { m_options.TryGetValue(SSH_KEYFILE_INLINE, out keyFile); - } if (!string.IsNullOrWhiteSpace(keyFile)) - { con = new SftpClient(m_server, m_port, m_username, ValidateKeyFile(keyFile, m_password)); - } else - { con = new SftpClient(m_server, m_port, m_username, m_password ?? string.Empty); - } con.HostKeyReceived += (sender, e) => { @@ -333,17 +316,17 @@ namespace Duplicati.Library.Backend } var hostFingerprint = - $"{e.HostKeyName} {e.KeyLength.ToString()} {BitConverter.ToString(e.FingerPrint).Replace('-', ':')}"; + $"{e.HostKeyName} {e.KeyLength} {BitConverter.ToString(e.FingerPrint).Replace('-', ':')}"; if (string.IsNullOrEmpty(m_fingerprint)) - throw new Library.Utility.HostKeyException( + throw new HostKeyException( Strings.SSHv2Backend.FingerprintNotSpecifiedManagedError( hostFingerprint.ToLower(CultureInfo.InvariantCulture), SSH_FINGERPRINT_OPTION, SSH_FINGERPRINT_ACCEPT_ANY_OPTION), hostFingerprint, m_fingerprint); if (!string.Equals(hostFingerprint, m_fingerprint, StringComparison.OrdinalIgnoreCase)) - throw new Library.Utility.HostKeyException( + throw new HostKeyException( Strings.SSHv2Backend.FingerprintNotMatchManagedError( hostFingerprint.ToLower(CultureInfo.InvariantCulture)), hostFingerprint, m_fingerprint); @@ -351,23 +334,20 @@ namespace Duplicati.Library.Backend e.CanTrust = true; }; - if (m_operationtimeout.Ticks != 0) + if (m_operationtimeout.TotalSeconds > 0) con.OperationTimeout = m_operationtimeout; - if (m_keepaliveinterval.Ticks != 0) + if (m_keepaliveinterval.TotalSeconds > 0) con.KeepAliveInterval = m_keepaliveinterval; - await this.TryConnect(con, cancelToken).ConfigureAwait(false); + await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => con.ConnectAsync(ct)).ConfigureAwait(false); if (m_useRelativePath && (string.IsNullOrWhiteSpace(con.WorkingDirectory) || !con.WorkingDirectory.StartsWith("/"))) throw new UserInformationException("Server does not report absolute initial directory, please switch to absolute paths", "RelativePathNotSupported"); m_initialDirectory = con.WorkingDirectory; - m_con = con; + return m_con = con; } - private Task TryConnect(SftpClient client, CancellationToken cancelToken) - => client.ConnectAsync(cancelToken); - - private async Task SetWorkingDirectory(CancellationToken cancelToken) + private async Task SetWorkingDirectory(SftpClient connection, CancellationToken cancelToken) { if (string.IsNullOrEmpty(m_path)) return; @@ -376,17 +356,17 @@ namespace Duplicati.Library.Backend ? Util.AppendDirSeparator(m_initialDirectory, "/") + m_path : m_path; - var workingDir = Util.AppendDirSeparator(m_con.WorkingDirectory, "/"); + var workingDir = Util.AppendDirSeparator(connection.WorkingDirectory, "/"); if (workingDir == targetPath) return; try { - await m_con.ChangeDirectoryAsync(targetPath, cancelToken).ConfigureAwait(false); + await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => connection.ChangeDirectoryAsync(targetPath, ct)).ConfigureAwait(false); } catch (Exception ex) { - throw new Interface.FolderMissingException( + throw new FolderMissingException( Strings.SSHv2Backend.FolderNotFoundManagedError(m_path, ex.Message), ex); } } @@ -396,10 +376,10 @@ namespace Duplicati.Library.Backend { var path = "."; - await CreateConnection(cancelToken).ConfigureAwait(false); - await SetWorkingDirectory(cancelToken).ConfigureAwait(false); + var con = await CreateConnection(cancelToken).ConfigureAwait(false); + await SetWorkingDirectory(con, cancelToken).ConfigureAwait(false); - await foreach (var ls in m_con.ListDirectoryAsync(path, cancelToken).ConfigureAwait(false)) + await foreach (var ls in await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, ct => con.ListDirectoryAsync(path, ct)).ConfigureAwait(false)) { if (ls.Name.ToString() == "." || ls.Name.ToString() == "..") continue; yield return new FileEntry(ls.Name, ls.Length, @@ -408,7 +388,7 @@ namespace Duplicati.Library.Backend } } - private static Renci.SshNet.PrivateKeyFile ValidateKeyFile(string filename, string password) + private static PrivateKeyFile ValidateKeyFile(string filename, string? password) { try { @@ -419,8 +399,8 @@ namespace Duplicati.Library.Backend : new PrivateKeyFile(filename, password); } - using (var ms = new System.IO.MemoryStream()) - using (var sr = new System.IO.StreamWriter(ms)) + using (var ms = new MemoryStream()) + using (var sr = new StreamWriter(ms)) { sr.Write(Utility.Uri.UrlDecode(filename.Substring(KEYFILE_URI.Length))); sr.Flush(); @@ -441,18 +421,13 @@ namespace Duplicati.Library.Backend #endregion - internal SftpClient Client - { - get { return m_con; } - } - public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new[] { m_server }); /// - public async IAsyncEnumerable ListAsync(string path, [EnumeratorCancellation] CancellationToken cancellationToken) + public async IAsyncEnumerable ListAsync(string? path, [EnumeratorCancellation] CancellationToken cancellationToken) { - await CreateConnection(cancellationToken).ConfigureAwait(false); - await SetWorkingDirectory(cancellationToken).ConfigureAwait(false); + var con = await CreateConnection(cancellationToken).ConfigureAwait(false); + await SetWorkingDirectory(con, cancellationToken).ConfigureAwait(false); var prefixPath = m_useRelativePath ? "" : m_path; if (!string.IsNullOrWhiteSpace(prefixPath)) @@ -465,7 +440,7 @@ namespace Duplicati.Library.Backend if (string.IsNullOrWhiteSpace(filterPath)) filterPath = "."; - await foreach (var x in Client.ListDirectoryAsync(filterPath, cancellationToken).ConfigureAwait(false)) + await foreach (var x in await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancellationToken, ct => con.ListDirectoryAsync(filterPath, ct)).ConfigureAwait(false)) yield return new FileEntry( x.Name, x.Attributes.Size, @@ -477,7 +452,7 @@ namespace Duplicati.Library.Backend } /// - public Task GetEntryAsync(string path, CancellationToken cancellationToken) - => Task.FromResult(null); + public Task GetEntryAsync(string path, CancellationToken cancellationToken) + => Task.FromResult(null); } } \ No newline at end of file diff --git a/Duplicati/Library/Backend/SSHv2/Strings.cs b/Duplicati/Library/Backend/SSHv2/Strings.cs index a182b771a..0f539c76f 100644 --- a/Duplicati/Library/Backend/SSHv2/Strings.cs +++ b/Duplicati/Library/Backend/SSHv2/Strings.cs @@ -23,48 +23,47 @@ namespace Duplicati.Library.Backend.Strings { internal static class KeyGenerator { - public static string Description { get { return LC.L(@"Module for generating SSH private/public keys"); } } - public static string DisplayName { get { return LC.L(@"SSH Key Generator"); } } - public static string KeyUsernameLong { get { return LC.L(@"A username to append to the public key."); } } - public static string KeyUsernameShort { get { return LC.L(@"Public key username"); } } - public static string KeyTypeLong { get { return LC.L(@"Determines the type of key to generate."); } } - public static string KeyTypeShort { get { return LC.L(@"The key type"); } } - public static string KeyLenLong { get { return LC.L(@"The length of the key in bits."); } } - public static string KeyLenShort { get { return LC.L(@"The key length"); } } + public static string Description => LC.L(@"Module for generating SSH private/public keys"); + public static string DisplayName => LC.L(@"SSH Key Generator"); + public static string KeyUsernameLong => LC.L(@"A username to append to the public key."); + public static string KeyUsernameShort => LC.L(@"Public key username"); + public static string KeyTypeLong => LC.L(@"Determines the type of key to generate."); + public static string KeyTypeShort => LC.L(@"The key type"); + public static string KeyLenLong => LC.L(@"The length of the key in bits."); + public static string KeyLenShort => LC.L(@"The key length"); } internal static class KeyUploader { - public static string Description { get { return LC.L(@"Module for uploading SSH public keys"); } } - public static string DisplayName { get { return LC.L(@"SSH Key Uploader"); } } - public static string UrlLong { get { return LC.L(@"The SSH connection URL used to establish the connection."); } } - public static string UrlShort { get { return LC.L(@"The SSH connection URL"); } } - public static string PubkeyLong { get { return LC.L(@"The SSH public key must be a valid SSH string, which is appended to the .ssh/authorized_keys file."); } } - public static string PubkeyShort { get { return LC.L(@"The SSH public key to append"); } } + public static string Description => LC.L(@"Module for uploading SSH public keys"); + public static string DisplayName => LC.L(@"SSH Key Uploader"); + public static string UrlLong => LC.L(@"The SSH connection URL used to establish the connection."); + public static string UrlShort => LC.L(@"The SSH connection URL"); + public static string PubkeyLong => LC.L(@"The SSH public key must be a valid SSH string, which is appended to the .ssh/authorized_keys file."); + public static string PubkeyShort => LC.L(@"The SSH public key to append"); } internal static class SSHv2Backend { - public static string Description { get { return LC.L(@"This backend can read and write data to an SSH based backend, using SFTP. Allowed formats are ""ssh://hostname/folder"" and ""ssh://username:password@hostname/folder""."); } } - public static string DisplayName { get { return LC.L(@"SFTP (SSH)"); } } - public static string DescriptionAuthPasswordLong { get { return LC.L(@"The password used to connect to the server. This may also be supplied as the environment variable ""AUTH_PASSWORD""."); } } - public static string DescriptionAuthPasswordShort { get { return LC.L(@"Supply the password used to connect to the server"); } } - public static string DescriptionAuthUsernameLong { get { return LC.L(@"The username used to connect to the server. This may also be supplied as the environment variable ""AUTH_USERNAME""."); } } - public static string DescriptionAuthUsernameShort { get { return LC.L(@"Supply the username used to connect to the server"); } } - public static string DescriptionFingerprintLong { get { return LC.L(@"The server fingerprint used for validation of server identity. Format is e.g. ""ssh-rsa 4096 11:22:33:44:55:66:77:88:99:00:11:22:33:44:55:66""."); } } - public static string DescriptionFingerprintShort { get { return LC.L(@"Supply server fingerprint used for validation of server identity"); } } - public static string DescriptionAnyFingerprintLong { get { return LC.L(@"To guard against man-in-the-middle attacks, the server fingerprint is verified on connection. Use this option to disable host-key fingerprint verification. You should only use this option for testing."); } } - public static string DescriptionAnyFingerprintShort { get { return LC.L(@"Disable fingerprint validation"); } } - public static string DescriptionSshkeyfileLong { get { return LC.L(@"Point to a valid OpenSSH keyfile. If the file is encrypted, the password supplied is used to decrypt it. If the keyfile is specified, the password is not used to authenticate."); } } - public static string DescriptionSshkeyfileShort { get { return LC.L(@"Use a SSH private key to authenticate"); } } + public static string Description => LC.L(@"This backend can read and write data to an SSH based backend, using SFTP. Allowed formats are ""ssh://hostname/folder"" and ""ssh://username:password@hostname/folder""."); + public static string DisplayName => LC.L(@"SFTP (SSH)"); + public static string UsernameRequired => LC.L(@"A username is required"); + public static string PasswordRequired => LC.L(@"A password is required if not using a keyfile"); + public static string DescriptionFingerprintLong => LC.L(@"The server fingerprint used for validation of server identity. Format is e.g. ""ssh-rsa 4096 11:22:33:44:55:66:77:88:99:00:11:22:33:44:55:66""."); + public static string DescriptionFingerprintShort => LC.L(@"Supply server fingerprint used for validation of server identity"); + public static string DescriptionAnyFingerprintLong => LC.L(@"To guard against man-in-the-middle attacks, the server fingerprint is verified on connection. Use this option to disable host-key fingerprint verification. You should only use this option for testing."); + public static string DescriptionAnyFingerprintShort => LC.L(@"Disable fingerprint validation"); + public static string DescriptionSshkeyfileLong => LC.L(@"Point to a valid OpenSSH keyfile. If the file is encrypted, the password supplied is used to decrypt it. If the keyfile is specified, the password is not used to authenticate."); + public static string DescriptionSshkeyfileShort => LC.L(@"Use a SSH private key to authenticate"); public static string DescriptionSshkeyLong(string urlprefix) { return LC.L(@"An url-encoded SSH private key. The private key must be prefixed with {0}. If the key is encrypted, the password supplied is used to decrypt it. If the private key is specified, the password is not used to authenticate.", urlprefix); } - public static string DescriptionSshkeyShort { get { return LC.L(@"Use a SSH private key to authenticate"); } } - public static string DescriptionSshtimeoutLong { get { return LC.L(@"Use this option to manage the internal timeout for SSH operations. If the value is set to zero, the operations will not time out."); } } - public static string DescriptionSshtimeoutShort { get { return LC.L(@"Set the operation timeout value"); } } - public static string DescriptionSshkeepaliveLong { get { return LC.L(@"Use this option to enable the keep-alive interval for the SSH connection. If the connection is idle, aggressive firewalls might close the connection. Using keep-alive will keep the connection open in this scenario. If this value is set to zero, the keep-alive is disabled."); } } - public static string DescriptionSshkeepaliveShort { get { return LC.L(@"Set a keepalive value"); } } - public static string DescriptionRelativePathShort { get { return LC.L(@"Treat source path as relative to the initial path"); } } - public static string DescriptionRelativePathLong { get { return LC.L(@"Use this option to treat the source path as relative to the initial path. This is useful when the full path of the system is not known."); } } + public static string DescriptionSshkeyShort => LC.L(@"Use a SSH private key to authenticate"); + public static string DescriptionSshtimeoutLong => LC.L(@"Use this option to manage the internal timeout for SSH operations. If the value is set to zero, the operations will not time out."); + public static string DescriptionSshtimeoutShort => LC.L(@"Set the operation timeout value"); + public static string DescriptionSshkeepaliveLong => LC.L(@"Use this option to enable the keep-alive interval for the SSH connection. If the connection is idle, aggressive firewalls might close the connection. Using keep-alive will keep the connection open in this scenario. If this value is set to zero, the keep-alive is disabled."); + public static string DescriptionSshkeepaliveShort => LC.L(@"Set a keepalive value"); + public static string DescriptionRelativePathShort => LC.L(@"Treat source path as relative to the initial path"); + public static string DescriptionRelativePathLong => LC.L(@"Use this option to treat the source path as relative to the initial path. This is useful when the full path of the system is not known."); public static string FolderNotFoundManagedError(string foldername, string message) { return LC.L(@"Unable to set folder to {0}, error message: {1}", foldername, message); } public static string FingerprintNotMatchManagedError(string fingerprint) { return LC.L(@"Validation of server fingerprint failed. Server returned fingerprint ""{0}"". Cause of this message is either not correct configuration or Man-in-the-middle attack!", fingerprint); } public static string FingerprintNotSpecifiedManagedError(string fingerprint, string hostkeyoption, string allkeysoptions) { return LC.L(@"Please add --{1}=""{0}"" to trust this host. Optionally you can use --{2} (NOT SECURE) for testing!", fingerprint, hostkeyoption, allkeysoptions); } + public static string TimeoutDeprecated(string self, string opt1, string opt2, string opt3) { return LC.L(@"The option {0} is deprecated. Please use --{1}, --{2} or --{3} instead.", self, opt1, opt2, opt3); } } } From 834db514a1cf00e9eca0b8a7b8b5ca01952d4e14 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sat, 22 Mar 2025 15:33:51 +0100 Subject: [PATCH 32/48] Fixed parsing error --- Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs index e368ba2d6..02c6a2395 100644 --- a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs +++ b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs @@ -57,7 +57,7 @@ namespace Duplicati.Library.Backend private readonly int m_port = 22; private readonly bool m_useRelativePath; private readonly TimeoutOptionsHelper.Timeouts m_timeouts; - private string m_initialDirectory; + private string? m_initialDirectory; private SftpClient? m_con; @@ -69,11 +69,9 @@ namespace Duplicati.Library.Backend m_options = null!; m_timeouts = null!; m_con = null!; - m_initialDirectory = null!; } public SSHv2(string url, Dictionary options) - : this() { m_options = options; var uri = new Utility.Uri(url); @@ -88,6 +86,7 @@ namespace Duplicati.Library.Backend m_fingerprint = options.GetValueOrDefault(SSH_FINGERPRINT_OPTION); m_fingerprintallowall = Utility.Utility.ParseBoolOption(options, SSH_FINGERPRINT_ACCEPT_ANY_OPTION); m_useRelativePath = Utility.Utility.ParseBoolOption(options, SSH_RELATIVE_PATH_OPTION); + m_timeouts = TimeoutOptionsHelper.Parse(options); m_path = uri.Path; @@ -353,7 +352,7 @@ namespace Duplicati.Library.Backend return; var targetPath = m_useRelativePath - ? Util.AppendDirSeparator(m_initialDirectory, "/") + m_path + ? Util.AppendDirSeparator(m_initialDirectory ?? "", "/") + m_path : m_path; var workingDir = Util.AppendDirSeparator(connection.WorkingDirectory, "/"); From 63cae93ac74fd0ca45a4f6c6938f0901e0e30011 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 24 Mar 2025 08:53:56 +0100 Subject: [PATCH 33/48] Updated Storj backend --- .../Duplicati.Library.Backend.Storj.csproj | 6 +- .../Library/Backend/Storj/StorjBackend.cs | 194 +++++++----------- .../Library/Backend/Storj/StorjConfig.cs | 47 ++--- Duplicati/Library/Backend/Storj/Strings.cs | 41 ++-- 4 files changed, 119 insertions(+), 169 deletions(-) diff --git a/Duplicati/Library/Backend/Storj/Duplicati.Library.Backend.Storj.csproj b/Duplicati/Library/Backend/Storj/Duplicati.Library.Backend.Storj.csproj index 84d35a141..64cc4d4b0 100644 --- a/Duplicati/Library/Backend/Storj/Duplicati.Library.Backend.Storj.csproj +++ b/Duplicati/Library/Backend/Storj/Duplicati.Library.Backend.Storj.csproj @@ -3,8 +3,10 @@ net8.0 Library - Storj backend for Duplicati - Copyright © 2025 Team Duplicati, MIT license + Storj backend for Duplicati + Copyright © 2025 Team Duplicati, MIT license + enable + enable diff --git a/Duplicati/Library/Backend/Storj/StorjBackend.cs b/Duplicati/Library/Backend/Storj/StorjBackend.cs index 5de882aea..e08ae37f0 100644 --- a/Duplicati/Library/Backend/Storj/StorjBackend.cs +++ b/Duplicati/Library/Backend/Storj/StorjBackend.cs @@ -18,13 +18,11 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; -using System; -using System.Collections.Generic; -using System.IO; +using Duplicati.Library.Utility; +using Duplicati.Library.Utility.Options; using System.Runtime.CompilerServices; -using System.Threading; -using System.Threading.Tasks; using uplink.NET.Interfaces; using uplink.NET.Models; using uplink.NET.Services; @@ -51,14 +49,15 @@ namespace Duplicati.Library.Backend.Storj private const string STORJ_DEFAULT_SATELLITE = "us1.storj.io:7777"; private const string STORJ_DEFAULT_AUTH_METHOD = STORJ_AUTH_METHOD_API_KEY; - private readonly string _satellite; - private readonly string _api_key; - private readonly string _secret; + private readonly string? _satellite; + private readonly string? _api_key; + private readonly string? _secret; private readonly string _bucket; - private readonly string _folder; - private Access _access; - private IBucketService _bucketService; - private IObjectService _objectService; + private readonly string? _folder; + private readonly Access _access; + private readonly IBucketService _bucketService; + private readonly IObjectService _objectService; + private readonly TimeoutOptionsHelper.Timeouts _timeouts; public static readonly Dictionary KNOWN_STORJ_SATELLITES = new Dictionary(StringComparer.OrdinalIgnoreCase){ { "US Central", "us1.storj.io:7777" }, @@ -77,7 +76,7 @@ namespace Duplicati.Library.Backend.Storj if (_libraryLoaded) return; - Access.SetTempDirectory(Library.Utility.TempFolder.SystemTempPath); + Access.SetTempDirectory(Utility.TempFolder.SystemTempPath); _libraryLoaded = true; } @@ -85,11 +84,16 @@ namespace Duplicati.Library.Backend.Storj // This constructor is needed by the BackendLoader. public Storj() { + _bucket = null!; + _access = null!; + _bucketService = null!; + _objectService = null!; + _timeouts = null!; } // ReSharper disable once UnusedMember.Global // This constructor is needed by the BackendLoader. - public Storj(string url, Dictionary options) + public Storj(string url, Dictionary options) { InitStorjLibrary(); @@ -102,17 +106,13 @@ namespace Duplicati.Library.Backend.Storj } else { - //Create an access for a satellite, API key and encryption passphrase + //Create an access for a satellite, API key and encryption passphrase _satellite = options.GetValueOrDefault(STORJ_SATELLITE, STORJ_DEFAULT_SATELLITE); + if (string.IsNullOrWhiteSpace(_satellite)) + _satellite = STORJ_DEFAULT_SATELLITE; - if (options.ContainsKey(STORJ_API_KEY)) - { - _api_key = options[STORJ_API_KEY]; - } - if (options.ContainsKey(STORJ_SECRET)) - { - _secret = options[STORJ_SECRET]; - } + _api_key = options.GetValueOrDefault(STORJ_API_KEY); + _secret = options.GetValueOrDefault(STORJ_SECRET); _access = new Access(_satellite, _api_key, _secret, new Config() { UserAgent = STORJ_PARTNER_ID }); } @@ -121,53 +121,36 @@ namespace Duplicati.Library.Backend.Storj _objectService = new ObjectService(_access); //If no bucket was provided use the default "duplicati"-bucket - if (options.ContainsKey(STORJ_BUCKET)) - { - _bucket = options[STORJ_BUCKET]; - } - else - { - _bucket = "duplicati"; - } + var bucket = options.GetValueOrDefault(STORJ_BUCKET); + _bucket = string.IsNullOrWhiteSpace(bucket) + ? STORJ_DEFAULT_BUCKET + : bucket; - if (options.ContainsKey(STORJ_FOLDER)) - { - _folder = options[STORJ_FOLDER]; - } + _folder = options.GetValueOrDefault(STORJ_FOLDER); + _timeouts = TimeoutOptionsHelper.Parse(options); } - public string DisplayName - { - get { return Strings.Storj.DisplayName; } - } + public string DisplayName => Strings.Storj.DisplayName; public string ProtocolKey => PROTOCOL_KEY; - public IList SupportedCommands - { - get - { - return new List(new ICommandLineArgument[] { - new CommandLineArgument(STORJ_AUTH_METHOD, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjAuthMethodDescriptionShort, Strings.Storj.StorjAuthMethodDescriptionLong, STORJ_DEFAULT_AUTH_METHOD, null, [STORJ_AUTH_METHOD_API_KEY, STORJ_AUTH_METHOD_ACCESS_GRANT]), - new CommandLineArgument(STORJ_SATELLITE, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjSatelliteDescriptionShort, Strings.Storj.StorjSatelliteDescriptionLong, STORJ_DEFAULT_SATELLITE), - new CommandLineArgument(STORJ_API_KEY, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjAPIKeyDescriptionShort, Strings.Storj.StorjAPIKeyDescriptionLong), - new CommandLineArgument(STORJ_SECRET, CommandLineArgument.ArgumentType.Password, Strings.Storj.StorjSecretDescriptionShort, Strings.Storj.StorjSecretDescriptionLong), - new CommandLineArgument(STORJ_SHARED_ACCESS, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjSharedAccessDescriptionShort, Strings.Storj.StorjSharedAccessDescriptionLong), - new CommandLineArgument(STORJ_BUCKET, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjBucketDescriptionShort, Strings.Storj.StorjBucketDescriptionLong, STORJ_DEFAULT_BUCKET), - new CommandLineArgument(STORJ_FOLDER, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjFolderDescriptionShort, Strings.Storj.StorjFolderDescriptionLong), - }); - } - } + public IList SupportedCommands => [ + new CommandLineArgument(STORJ_AUTH_METHOD, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjAuthMethodDescriptionShort, Strings.Storj.StorjAuthMethodDescriptionLong, STORJ_DEFAULT_AUTH_METHOD, null, [STORJ_AUTH_METHOD_API_KEY, STORJ_AUTH_METHOD_ACCESS_GRANT]), + new CommandLineArgument(STORJ_SATELLITE, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjSatelliteDescriptionShort, Strings.Storj.StorjSatelliteDescriptionLong, STORJ_DEFAULT_SATELLITE), + new CommandLineArgument(STORJ_API_KEY, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjAPIKeyDescriptionShort, Strings.Storj.StorjAPIKeyDescriptionLong), + new CommandLineArgument(STORJ_SECRET, CommandLineArgument.ArgumentType.Password, Strings.Storj.StorjSecretDescriptionShort, Strings.Storj.StorjSecretDescriptionLong), + new CommandLineArgument(STORJ_SHARED_ACCESS, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjSharedAccessDescriptionShort, Strings.Storj.StorjSharedAccessDescriptionLong), + new CommandLineArgument(STORJ_BUCKET, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjBucketDescriptionShort, Strings.Storj.StorjBucketDescriptionLong, STORJ_DEFAULT_BUCKET), + new CommandLineArgument(STORJ_FOLDER, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjFolderDescriptionShort, Strings.Storj.StorjFolderDescriptionLong), + .. TimeoutOptionsHelper.GetOptions(), + ]; - public string Description - { - get - { - return Strings.Storj.Description; - } - } + public string Description => Strings.Storj.Description; - public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(Array.Empty()); + public Task GetDNSNamesAsync(CancellationToken cancelToken) + => Task.FromResult(string.IsNullOrWhiteSpace(_satellite) + ? [] + : [_satellite.Split(':').First()]); public Task CreateFolderAsync(CancellationToken cancelToken) { @@ -175,12 +158,15 @@ namespace Duplicati.Library.Backend.Storj return Task.CompletedTask; } + private Task GetBucketAsync(CancellationToken cancelToken) + => Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancelToken, _ => _bucketService.EnsureBucketAsync(_bucket)); + public async Task DeleteAsync(string remotename, CancellationToken cancelToken) { try { - var bucket = await _bucketService.EnsureBucketAsync(_bucket).ConfigureAwait(false); - await _objectService.DeleteObjectAsync(bucket, GetBasePath() + remotename).ConfigureAwait(false); + var bucket = await GetBucketAsync(cancelToken).ConfigureAwait(false); + await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancelToken, _ => _objectService.DeleteObjectAsync(bucket, GetBasePath() + remotename)).ConfigureAwait(false); } catch (Exception root) { @@ -190,25 +176,13 @@ namespace Duplicati.Library.Backend.Storj public void Dispose() { - if (_objectService != null) - { - _objectService = null; - } - if (_bucketService != null) - { - _bucketService = null; - } - if (_access != null) - { - _access.Dispose(); - _access = null; - } + _access?.Dispose(); } public async Task GetAsync(string remotename, string filename, CancellationToken cancelToken) { - var bucket = await _bucketService.EnsureBucketAsync(_bucket).ConfigureAwait(false); - var download = await _objectService.DownloadObjectAsync(bucket, GetBasePath() + remotename, new DownloadOptions(), false).ConfigureAwait(false); + var bucket = await GetBucketAsync(cancelToken).ConfigureAwait(false); + var download = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancelToken, _ => _objectService.DownloadObjectAsync(bucket, GetBasePath() + remotename, new DownloadOptions(), false)).ConfigureAwait(false); await download.StartDownloadAsync().ConfigureAwait(false); if (download.Completed) @@ -223,16 +197,19 @@ namespace Duplicati.Library.Backend.Storj public async Task GetAsync(string remotename, Stream stream, CancellationToken cancelToken) { - int index = 0; - var bucket = await _bucketService.EnsureBucketAsync(_bucket).ConfigureAwait(false); - var download = await _objectService.DownloadObjectAsync(bucket, GetBasePath() + remotename, new DownloadOptions(), false).ConfigureAwait(false); + var index = 0; + var bucket = await GetBucketAsync(cancelToken).ConfigureAwait(false); + var download = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancelToken, _ => _objectService.DownloadObjectAsync(bucket, GetBasePath() + remotename, new DownloadOptions(), false)).ConfigureAwait(false); + using var ts = stream.ObserveWriteTimeout(_timeouts.ReadWriteTimeout, false); download.DownloadOperationProgressChanged += (op) => { cancelToken.ThrowIfCancellationRequested(); - int newPartLength = (int)op.BytesReceived - index; - byte[] newPart = new byte[newPartLength]; + var newPartLength = (int)op.BytesReceived - index; + // TODO: Copy here should not be needed? + // ts.Write(op.DownloadedBytes, index, newPartLength); + var newPart = new byte[newPartLength]; Array.Copy(op.DownloadedBytes, index, newPart, 0, newPartLength); - stream.Write(newPart, 0, newPartLength); + ts.Write(newPart, 0, newPartLength); index = index + newPartLength; }; await download.StartDownloadAsync().ConfigureAwait(false); @@ -241,9 +218,9 @@ namespace Duplicati.Library.Backend.Storj /// public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { - var bucket = await _bucketService.EnsureBucketAsync(_bucket).ConfigureAwait(false); + var bucket = await GetBucketAsync(cancelToken).ConfigureAwait(false); var prefix = GetBasePath(); - var objects = await _objectService.ListObjectsAsync(bucket, new ListObjectsOptions { Recursive = true, System = true, Custom = true, Prefix = prefix }).ConfigureAwait(false); + var objects = await Utility.Utility.WithTimeout(_timeouts.ListTimeout, cancelToken, _ => _objectService.ListObjectsAsync(bucket, new ListObjectsOptions { Recursive = true, System = true, Custom = true, Prefix = prefix })).ConfigureAwait(false); foreach (var obj in objects.Items) { @@ -264,29 +241,19 @@ namespace Duplicati.Library.Backend.Storj public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken) { - var bucket = await _bucketService.EnsureBucketAsync(_bucket).ConfigureAwait(false); - CustomMetadata custom = new CustomMetadata(); + var bucket = await GetBucketAsync(cancelToken).ConfigureAwait(false); + var custom = new CustomMetadata(); custom.Entries.Add(new CustomMetadataEntry { Key = StorjFile.STORJ_LAST_ACCESS, Value = DateTime.Now.ToUniversalTime().ToString("O") }); custom.Entries.Add(new CustomMetadataEntry { Key = StorjFile.STORJ_LAST_MODIFICATION, Value = DateTime.Now.ToUniversalTime().ToString("O") }); - var upload = await _objectService.UploadObjectAsync(bucket, GetBasePath() + remotename, new UploadOptions(), stream, custom, false).ConfigureAwait(false); + using var ts = stream.ObserveReadTimeout(_timeouts.ReadWriteTimeout, false); + var upload = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancelToken, _ => _objectService.UploadObjectAsync(bucket, GetBasePath() + remotename, new UploadOptions(), ts, custom, false)).ConfigureAwait(false); await upload.StartUploadAsync().ConfigureAwait(false); if (upload.Failed) - { throw new Exception(upload.ErrorMessage); - } } - public async Task TestAsync(CancellationToken cancelToken) - { - var ct = CancellationTokenSource.CreateLinkedTokenSource(cancelToken); - var testTask = TestImplAsync(ct.Token); - ct.CancelAfter(10000); - await Task.WhenAny(testTask, Task.Delay(Timeout.Infinite, ct.Token)).ConfigureAwait(false); - if (testTask.IsCompleted) - await testTask.ConfigureAwait(false); - else if (!testTask.IsCompletedSuccessfully) - throw new Exception(Strings.Storj.TestConnectionFailed); - } + public Task TestAsync(CancellationToken cancelToken) + => Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancelToken, ct => TestImplAsync(ct)); /// /// Test the connection by: @@ -297,9 +264,9 @@ namespace Duplicati.Library.Backend.Storj /// true, if the test was successfull or and exception private async Task TestImplAsync(CancellationToken cancelToken) { - string testFileName = GetBasePath() + "duplicati_test.dat"; + var testFileName = GetBasePath() + "duplicati_test.dat"; - var bucket = await _bucketService.EnsureBucketAsync(_bucket).ConfigureAwait(false); + var bucket = await GetBucketAsync(cancelToken).ConfigureAwait(false); var upload = await _objectService.UploadObjectAsync(bucket, testFileName, new UploadOptions(), GetRandomBytes(256), false).ConfigureAwait(false); await upload.StartUploadAsync().ConfigureAwait(false); @@ -309,9 +276,7 @@ namespace Duplicati.Library.Backend.Storj await _objectService.DeleteObjectAsync(bucket, testFileName).ConfigureAwait(false); if (download.Failed || download.BytesReceived != 256) - { throw new Exception(download.ErrorMessage); - } return true; } @@ -321,12 +286,9 @@ namespace Duplicati.Library.Backend.Storj /// /// The base path within a bucket where the backup shall be placed private string GetBasePath() - { - if (!string.IsNullOrEmpty(_folder)) - return _folder + "/"; - else - return ""; - } + => string.IsNullOrWhiteSpace(_folder) + ? "" + : Util.AppendDirSeparator(_folder, "/"); /// /// Creates some random bytes with the given length - just for testing the connection @@ -335,10 +297,8 @@ namespace Duplicati.Library.Backend.Storj /// A byte-array with the given length private static byte[] GetRandomBytes(long length) { - byte[] bytes = new byte[length]; - Random rand = new Random(); - rand.NextBytes(bytes); - + var bytes = new byte[length]; + Random.Shared.NextBytes(bytes); return bytes; } } diff --git a/Duplicati/Library/Backend/Storj/StorjConfig.cs b/Duplicati/Library/Backend/Storj/StorjConfig.cs index b4bd5fe37..99a3da856 100644 --- a/Duplicati/Library/Backend/Storj/StorjConfig.cs +++ b/Duplicati/Library/Backend/Storj/StorjConfig.cs @@ -18,10 +18,9 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. + using Duplicati.Library.Interface; using Duplicati.Library.Localization.Short; -using System; -using System.Collections.Generic; namespace Duplicati.Library.Backend.Storj { @@ -29,7 +28,7 @@ namespace Duplicati.Library.Backend.Storj { private const ConfigType DEFAULT_CONFIG_TYPE = ConfigType.Satellites; private const string KEY_CONFIGTYPE = "storj-config"; - private static readonly string DEFAULT_CONFIG_TYPE_STR = Enum.GetName(typeof(ConfigType), DEFAULT_CONFIG_TYPE); + private static readonly string DEFAULT_CONFIG_TYPE_STR = DEFAULT_CONFIG_TYPE.ToString(); public enum ConfigType { @@ -39,47 +38,31 @@ namespace Duplicati.Library.Backend.Storj #region IWebModule implementation - public string Key { get { return "storj-getconfig"; } } + public string Key => "storj-getconfig"; - public string DisplayName { get { return LC.L("Storj DCS configuration module"); } } + public string DisplayName => Strings.StorjConfig.DisplayName; - public string Description { get { return LC.L("Expose Storj DCS configuration as a web module"); } } + public string Description => Strings.StorjConfig.Description; - public IList SupportedCommands - { - get - { - return new List([ - new CommandLineArgument(KEY_CONFIGTYPE, CommandLineArgument.ArgumentType.Enumeration, LC.L("The config to get"), LC.L("Provide different config values"), DEFAULT_CONFIG_TYPE_STR, Enum.GetNames(typeof(ConfigType))) - - ]); - } - } + public IList SupportedCommands => [ + new CommandLineArgument(KEY_CONFIGTYPE, CommandLineArgument.ArgumentType.Enumeration, LC.L("The config to get"), LC.L("Provide different config values"), DEFAULT_CONFIG_TYPE_STR, Enum.GetNames(typeof(ConfigType))) + ]; public IDictionary Execute(IDictionary options) { - string k; - options.TryGetValue(KEY_CONFIGTYPE, out k); + options.TryGetValue(KEY_CONFIGTYPE, out var k); if (string.IsNullOrWhiteSpace(k)) - { k = DEFAULT_CONFIG_TYPE_STR; - } - ConfigType ct; - if (!Enum.TryParse(k, true, out ct)) - { + if (!Enum.TryParse(k, true, out var ct)) ct = DEFAULT_CONFIG_TYPE; - } - switch (ct) + return ct switch { - case ConfigType.Satellites: - return Storj.KNOWN_STORJ_SATELLITES; - case ConfigType.AuthenticationMethods: - return Storj.KNOWN_AUTHENTICATION_METHODS; - default: - return Storj.KNOWN_STORJ_SATELLITES; - } + ConfigType.Satellites => Storj.KNOWN_STORJ_SATELLITES, + ConfigType.AuthenticationMethods => Storj.KNOWN_AUTHENTICATION_METHODS, + _ => Storj.KNOWN_STORJ_SATELLITES, + }; } #endregion } diff --git a/Duplicati/Library/Backend/Storj/Strings.cs b/Duplicati/Library/Backend/Storj/Strings.cs index aded7e079..92bc070f7 100644 --- a/Duplicati/Library/Backend/Storj/Strings.cs +++ b/Duplicati/Library/Backend/Storj/Strings.cs @@ -22,24 +22,29 @@ using Duplicati.Library.Localization.Short; namespace Duplicati.Library.Backend.Strings { - public static class Storj + internal static class Storj { - public static string Description { get { return LC.L(@"This backend can read and write data to the Storj DCS."); } } - public static string DisplayName { get { return LC.L(@"Storj DCS (Decentralized Cloud Storage)"); } } - public static string TestConnectionFailed { get { return LC.L(@"Connection-test failed."); } } - public static string StorjAuthMethodDescriptionLong { get { return LC.L(@"Specify the authentication method which describes which way to use to connect to the network - either via API key or via an access grant."); } } - public static string StorjAuthMethodDescriptionShort { get { return LC.L(@"Authentication method"); } } - public static string StorjSatelliteDescriptionLong { get { return LC.L(@"Specify the satellite that keeps track of all metadata. Use a Storj DCS server for high-performance SLA-backed connectivity or use a community server. Or even host your own."); } } - public static string StorjSatelliteDescriptionShort { get { return LC.L(@"Satellite"); } } - public static string StorjAPIKeyDescriptionLong { get { return LC.L(@"Supply the API key which grants access to a specific project on your chosen satellite. Head over to the dashboard of your satellite to create one if you do not already have an API key."); } } - public static string StorjAPIKeyDescriptionShort { get { return LC.L(@"API key"); } } - public static string StorjSecretDescriptionLong { get { return LC.L(@"Supply the encryption passphrase used to encrypt your data before sending it to the Storj network. This passphrase can be the only secret to provide - for Storj you do not necessary need any additional encryption (from Duplicati) in place."); } } - public static string StorjSecretDescriptionShort { get { return LC.L(@"Encryption passphrase"); } } - public static string StorjSharedAccessDescriptionLong { get { return LC.L(@"Supply the access grant which contains all information in one encrypted string. You may use it instead of a satellite, API key and secret."); } } - public static string StorjSharedAccessDescriptionShort { get { return LC.L(@"Access grant"); } } - public static string StorjBucketDescriptionLong { get { return LC.L(@"Specify the bucket for storing the backup."); } } - public static string StorjBucketDescriptionShort { get { return LC.L(@"Bucket"); } } - public static string StorjFolderDescriptionLong { get { return LC.L(@"Specify the folder in the bucket for storing the backup."); } } - public static string StorjFolderDescriptionShort { get { return LC.L(@"Folder"); } } + public static string Description => LC.L(@"This backend can read and write data to the Storj DCS."); + public static string DisplayName => LC.L(@"Storj DCS (Decentralized Cloud Storage)"); + public static string StorjAuthMethodDescriptionLong => LC.L(@"Specify the authentication method which describes which way to use to connect to the network - either via API key or via an access grant."); + public static string StorjAuthMethodDescriptionShort => LC.L(@"Authentication method"); + public static string StorjSatelliteDescriptionLong => LC.L(@"Specify the satellite that keeps track of all metadata. Use a Storj DCS server for high-performance SLA-backed connectivity or use a community server. Or even host your own."); + public static string StorjSatelliteDescriptionShort => LC.L(@"Satellite"); + public static string StorjAPIKeyDescriptionLong => LC.L(@"Supply the API key which grants access to a specific project on your chosen satellite. Head over to the dashboard of your satellite to create one if you do not already have an API key."); + public static string StorjAPIKeyDescriptionShort => LC.L(@"API key"); + public static string StorjSecretDescriptionLong => LC.L(@"Supply the encryption passphrase used to encrypt your data before sending it to the Storj network. This passphrase can be the only secret to provide - for Storj you do not necessary need any additional encryption (from Duplicati) in place."); + public static string StorjSecretDescriptionShort => LC.L(@"Encryption passphrase"); + public static string StorjSharedAccessDescriptionLong => LC.L(@"Supply the access grant which contains all information in one encrypted string. You may use it instead of a satellite, API key and secret."); + public static string StorjSharedAccessDescriptionShort => LC.L(@"Access grant"); + public static string StorjBucketDescriptionLong => LC.L(@"Specify the bucket for storing the backup."); + public static string StorjBucketDescriptionShort => LC.L(@"Bucket"); + public static string StorjFolderDescriptionLong => LC.L(@"Specify the folder in the bucket for storing the backup."); + public static string StorjFolderDescriptionShort => LC.L(@"Folder"); + } + + internal static class StorjConfig + { + public static string DisplayName => LC.L("Storj DCS configuration module"); + public static string Description => LC.L("Expose Storj DCS configuration as a web module"); } } From a54e37ff96eee84e4674492eed9529cd6e233308 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 24 Mar 2025 08:58:02 +0100 Subject: [PATCH 34/48] Changed Sharepoint to work better with large chunks --- .../Library/Backend/SharePoint/SharePointBackend.cs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Duplicati/Library/Backend/SharePoint/SharePointBackend.cs b/Duplicati/Library/Backend/SharePoint/SharePointBackend.cs index d56a81104..8b977bbb1 100644 --- a/Duplicati/Library/Backend/SharePoint/SharePointBackend.cs +++ b/Duplicati/Library/Backend/SharePoint/SharePointBackend.cs @@ -584,6 +584,7 @@ namespace Duplicati.Library.Backend bufCnt += lastreadsize; using (var contentChunk = new MemoryStream(buf, 0, bufCnt, false)) + using (var timeoutStream = contentChunk.ObserveReadTimeout(m_timeouts.ReadWriteTimeout)) { ClientResult? bytesUploaded = null; if (uploadFile == null) @@ -592,13 +593,13 @@ namespace Duplicati.Library.Backend var fileInfo = new FileCreationInformation(); fileInfo.Url = uniqueFileName; fileInfo.Overwrite = true; - fileInfo.ContentStream = (bufCnt < blockSize) ? contentChunk : new MemoryStream(0); + fileInfo.ContentStream = (bufCnt < blockSize) ? timeoutStream : new MemoryStream(0); uploadFile = folder.Files.Add(fileInfo); if (bufCnt < blockSize) needsFinalize = false; else - bytesUploaded = uploadFile.StartUpload(uploadId, contentChunk); // new OverrideableStream(chunkStream)); + bytesUploaded = uploadFile.StartUpload(uploadId, timeoutStream); // new OverrideableStream(chunkStream)); } else { @@ -606,19 +607,19 @@ namespace Duplicati.Library.Backend //uploadFile = ctx.Web.GetFileByServerRelativeUrl(folder.ServerRelativeUrl + System.IO.Path.AltDirectorySeparatorChar + uniqueFileName); if (bufCnt < blockSize) // Last block: end sliced upload by calling FinishUpload. { - uploadFile = uploadFile.FinishUpload(uploadId, fileoffset, contentChunk); + uploadFile = uploadFile.FinishUpload(uploadId, fileoffset, timeoutStream); needsFinalize = false; // signal no final call necessary. } else // Continue sliced upload. { - bytesUploaded = uploadFile.ContinueUpload(uploadId, fileoffset, contentChunk); + bytesUploaded = uploadFile.ContinueUpload(uploadId, fileoffset, timeoutStream); } } if (bytesUploaded == null) ctx.Load(uploadFile, f => f.Length); - await Utility.Utility.WithTimeout(m_timeouts.ReadWriteTimeout, cancelToken, _ => ctx.ExecuteQueryAsync()).ConfigureAwait(false); + await ctx.ExecuteQueryAsync().ConfigureAwait(false); // Check consistency and update fileoffset for the next slice. if (bytesUploaded != null) { From 9d617fe08a45fd4c3e6518a30ae7de029f7c5d09 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 24 Mar 2025 09:15:49 +0100 Subject: [PATCH 35/48] Updated TahoeLAFS --- ...Duplicati.Library.Backend.TahoeLAFS.csproj | 2 + .../Library/Backend/TahoeLAFS/Strings.cs | 16 +- .../Library/Backend/TahoeLAFS/TahoeBackend.cs | 199 ++++++++---------- .../Utility/Options/SslOptionsHelper.cs | 35 ++- 4 files changed, 126 insertions(+), 126 deletions(-) diff --git a/Duplicati/Library/Backend/TahoeLAFS/Duplicati.Library.Backend.TahoeLAFS.csproj b/Duplicati/Library/Backend/TahoeLAFS/Duplicati.Library.Backend.TahoeLAFS.csproj index 808a68d75..87a3061b8 100644 --- a/Duplicati/Library/Backend/TahoeLAFS/Duplicati.Library.Backend.TahoeLAFS.csproj +++ b/Duplicati/Library/Backend/TahoeLAFS/Duplicati.Library.Backend.TahoeLAFS.csproj @@ -5,6 +5,8 @@ Library Duplicati.Library.Backend Copyright © 2025 Team Duplicati, MIT license + enable + enable diff --git a/Duplicati/Library/Backend/TahoeLAFS/Strings.cs b/Duplicati/Library/Backend/TahoeLAFS/Strings.cs index 6b25b337b..2ab06071b 100644 --- a/Duplicati/Library/Backend/TahoeLAFS/Strings.cs +++ b/Duplicati/Library/Backend/TahoeLAFS/Strings.cs @@ -19,13 +19,15 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Localization.Short; -namespace Duplicati.Library.Backend.Strings { - internal static class TahoeBackend { - public static string Description { get { return LC.L(@"This backend can read and write data to a Tahoe-LAFS based backend. Allowed format is ""tahoe://hostname:port/uri/$DIRCAP""."); } } - public static string Displayname { get { return LC.L(@"Tahoe-LAFS"); } } - public static string DescriptionUseSSLLong { get { return LC.L(@"Use this option to communicate using Secure Socket Layer (SSL) over http (https)."); } } - public static string DescriptionUseSSLShort { get { return LC.L(@"Instruct Duplicati to use an SSL (https) connection"); } } +namespace Duplicati.Library.Backend.Strings +{ + internal static class TahoeBackend + { + public static string Description => LC.L(@"This backend can read and write data to a Tahoe-LAFS based backend. Allowed format is ""tahoe://hostname:port/uri/$DIRCAP""."); + public static string Displayname => LC.L(@"Tahoe-LAFS"); + public static string DescriptionUseSSLLong => LC.L(@"Use this option to communicate using Secure Socket Layer (SSL) over http (https)."); + public static string DescriptionUseSSLShort => LC.L(@"Instruct Duplicati to use an SSL (https) connection"); public static string MissingFolderError(string foldername, string message) { return LC.L(@"The folder {0} was not found, message: {1}", foldername, message); } - public static string UnrecognizedUriError { get { return LC.L(@"Unsupported URL format, must start with ""uri/URI:DIR2:"""); } } + public static string UnrecognizedUriError => LC.L(@"Unsupported URL format, must start with ""uri/URI:DIR2:"""); } } diff --git a/Duplicati/Library/Backend/TahoeLAFS/TahoeBackend.cs b/Duplicati/Library/Backend/TahoeLAFS/TahoeBackend.cs index 22d693924..c3da4ad88 100644 --- a/Duplicati/Library/Backend/TahoeLAFS/TahoeBackend.cs +++ b/Duplicati/Library/Backend/TahoeLAFS/TahoeBackend.cs @@ -21,14 +21,12 @@ using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; +using Duplicati.Library.Utility; +using Duplicati.Library.Utility.Options; using Newtonsoft.Json; using Newtonsoft.Json.Linq; -using System; -using System.Collections.Generic; using System.Net; using System.Runtime.CompilerServices; -using System.Threading; -using System.Threading.Tasks; namespace Duplicati.Library.Backend { @@ -36,27 +34,28 @@ namespace Duplicati.Library.Backend { private readonly string m_url; private readonly bool m_useSSL = false; + private readonly TimeoutOptionsHelper.Timeouts m_timeouts; private class TahoeEl { - public string nodetype { get; set; } - public TahoeNode node { get; set; } + public string? nodetype { get; set; } + public TahoeNode? node { get; set; } } private class TahoeNode { - public string rw_uri { get; set; } - public string verify_uri { get; set; } - public string ro_uri { get; set; } - public Dictionary children { get; set; } + public string? rw_uri { get; set; } + public string? verify_uri { get; set; } + public string? ro_uri { get; set; } + public Dictionary? children { get; set; } public bool mutable { get; set; } public long size { get; set; } - public TahoeMetadata metadata { get; set; } + public TahoeMetadata? metadata { get; set; } } private class TahoeMetadata { - public TahoeStamps tahoe { get; set; } + public TahoeStamps? tahoe { get; set; } } private class TahoeStamps @@ -68,15 +67,13 @@ namespace Duplicati.Library.Backend private class TahoeElConverter : JsonConverter { public override bool CanConvert(Type objectType) - { - return objectType == typeof(TahoeEl); - } + => objectType == typeof(TahoeEl); - public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + public override object ReadJson(JsonReader reader, Type objectType, object? existingValue, JsonSerializer serializer) { var array = JArray.Load(reader); - string nodetype = null; - TahoeNode node = null; + string? nodetype = null; + TahoeNode? node = null; foreach (var token in array.Children()) if (token.Type == JTokenType.String) nodetype = token.ToString(); @@ -86,18 +83,18 @@ namespace Duplicati.Library.Backend return new TahoeEl() { nodetype = nodetype, node = node }; } - public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) - { - throw new NotImplementedException(); - } + public override void WriteJson(JsonWriter writer, object? value, JsonSerializer serializer) + => throw new NotImplementedException(); } public TahoeBackend() { + m_url = null!; + m_timeouts = null!; } - public TahoeBackend(string url, Dictionary options) + public TahoeBackend(string url, Dictionary options) { //Validate URL var u = new Utility.Uri(url); @@ -106,15 +103,18 @@ namespace Duplicati.Library.Backend if (!u.Path.StartsWith("uri/URI:DIR2:", StringComparison.Ordinal) && !u.Path.StartsWith("uri/URI%3ADIR2%3A", StringComparison.Ordinal)) throw new UserInformationException(Strings.TahoeBackend.UnrecognizedUriError, "TahoeInvalidUri"); - m_useSSL = Utility.Utility.ParseBoolOption(options, "use-ssl"); + // TODO: When upgrading to HttpClient, also support the certificate options + var certOptions = SslOptionsHelper.Parse(options); + m_useSSL = certOptions.UseSSL; m_url = u.SetScheme(m_useSSL ? "https" : "http").SetQuery(null).SetCredentials(null, null).ToString(); m_url = Util.AppendDirSeparator(m_url, "/"); + m_timeouts = TimeoutOptionsHelper.Parse(options); } - private System.Net.HttpWebRequest CreateRequest(string remotename, string queryparams) + private HttpWebRequest CreateRequest(string remotename, string queryparams) { - var req = (System.Net.HttpWebRequest)System.Net.HttpWebRequest.Create(m_url + (Library.Utility.Uri.UrlEncode(remotename).Replace("+", "%20")) + (string.IsNullOrEmpty(queryparams) || queryparams.Trim().Length == 0 ? "" : "?" + queryparams)); + var req = (HttpWebRequest)HttpWebRequest.Create($"{m_url}{Utility.Uri.UrlEncode(remotename).Replace("+", "%20")}{(string.IsNullOrEmpty(queryparams) || queryparams.Trim().Length == 0 ? "" : "?" + queryparams)}"); req.KeepAlive = false; req.UserAgent = "Duplicati Tahoe-LAFS Client v" + System.Reflection.Assembly.GetExecutingAssembly().GetName().Version; @@ -127,86 +127,73 @@ namespace Duplicati.Library.Backend public Task TestAsync(CancellationToken cancelToken) => this.TestListAsync(cancelToken); - public Task CreateFolderAsync(CancellationToken cancelToken) + public async Task CreateFolderAsync(CancellationToken cancelToken) { var req = CreateRequest("", "t=mkdir"); - req.Method = System.Net.WebRequestMethods.Http.Post; - var areq = new Utility.AsyncHttpRequest(req); - using (areq.GetResponse()) + req.Method = WebRequestMethods.Http.Post; + var areq = new AsyncHttpRequest(req); + using (await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => areq.GetResponse()).ConfigureAwait(false)) { } - - return Task.CompletedTask; } - public string DisplayName - { - get { return Strings.TahoeBackend.Displayname; } - } + public string DisplayName => Strings.TahoeBackend.Displayname; - public string ProtocolKey - { - get { return "tahoe"; } - } + public string ProtocolKey => "tahoe"; /// public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { - // Remove warning until this is rewritten to use HttpClient - await Task.CompletedTask; - TahoeEl data; try { var req = CreateRequest("", "t=json"); - req.Method = System.Net.WebRequestMethods.Http.Get; + req.Method = WebRequestMethods.Http.Get; - var areq = new Utility.AsyncHttpRequest(req); - using (var resp = (System.Net.HttpWebResponse)areq.GetResponse()) + var areq = new AsyncHttpRequest(req); + using (var resp = await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, _ => (HttpWebResponse)areq.GetResponse()).ConfigureAwait(false)) { var code = (int)resp.StatusCode; if (code < 200 || code >= 300) //For some reason Mono does not throw this automatically - throw new System.Net.WebException(resp.StatusDescription, null, System.Net.WebExceptionStatus.ProtocolError, resp); + throw new WebException(resp.StatusDescription, null, WebExceptionStatus.ProtocolError, resp); using (var rs = areq.GetResponseStream()) - using (var sr = new System.IO.StreamReader(rs)) - using (var jr = new Newtonsoft.Json.JsonTextReader(sr)) + using (var sr = new StreamReader(rs)) + using (var jr = new JsonTextReader(sr)) { - var jsr = new Newtonsoft.Json.JsonSerializer(); + var jsr = new JsonSerializer(); jsr.Converters.Add(new TahoeElConverter()); - data = jsr.Deserialize(jr); + data = jsr.Deserialize(jr) + ?? throw new Exception("Invalid folder listing response"); } } } - catch (System.Net.WebException wex) + catch (WebException wex) + when (wex.Response is HttpWebResponse response + && (response.StatusCode == HttpStatusCode.Conflict || response.StatusCode == HttpStatusCode.NotFound)) { - //Convert to better exception - if (wex.Response as System.Net.HttpWebResponse != null) - if ((wex.Response as System.Net.HttpWebResponse).StatusCode == System.Net.HttpStatusCode.Conflict || (wex.Response as System.Net.HttpWebResponse).StatusCode == System.Net.HttpStatusCode.NotFound) - throw new Interface.FolderMissingException(Strings.TahoeBackend.MissingFolderError(m_url, wex.Message), wex); - - throw; + throw new FolderMissingException(Strings.TahoeBackend.MissingFolderError(m_url, wex.Message), wex); } if (data == null || data.node == null || data.nodetype != "dirnode") throw new Exception("Invalid folder listing response"); - foreach (var e in data.node.children) + foreach (var e in data.node.children ?? []) { if (e.Value == null || e.Value.node == null) continue; - bool isDir = e.Value.nodetype == "dirnode"; - bool isFile = e.Value.nodetype == "filenode"; + var isDir = e.Value.nodetype == "dirnode"; + var isFile = e.Value.nodetype == "filenode"; if (!isDir && !isFile) continue; - FileEntry fe = new FileEntry(e.Key); + var fe = new FileEntry(e.Key); fe.IsFolder = isDir; if (e.Value.node.metadata != null && e.Value.node.metadata.tahoe != null) - fe.LastModification = Duplicati.Library.Utility.Utility.EPOCH + TimeSpan.FromSeconds(e.Value.node.metadata.tahoe.linkmotime); + fe.LastModification = Utility.Utility.EPOCH + TimeSpan.FromSeconds(e.Value.node.metadata.tahoe.linkmotime); if (isFile) fe.Size = e.Value.node.size; @@ -217,53 +204,40 @@ namespace Duplicati.Library.Backend public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) { - using (var fs = System.IO.File.OpenRead(filename)) + using (var fs = File.OpenRead(filename)) await PutAsync(remotename, fs, cancelToken).ConfigureAwait(false); } public async Task GetAsync(string remotename, string filename, CancellationToken cancelToken) { - using (var fs = System.IO.File.Create(filename)) + using (var fs = File.Create(filename)) await GetAsync(remotename, fs, cancelToken).ConfigureAwait(false); } - public Task DeleteAsync(string remotename, CancellationToken cancelToken) + public async Task DeleteAsync(string remotename, CancellationToken cancelToken) { try { - System.Net.HttpWebRequest req = CreateRequest(remotename, ""); + var req = CreateRequest(remotename, ""); req.Method = "DELETE"; - Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req); - using (areq.GetResponse()) + var areq = new AsyncHttpRequest(req); + using (await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => areq.GetResponse()).ConfigureAwait(false)) { } } - catch (System.Net.WebException wex) + catch (WebException wex) + when (wex.Response is HttpWebResponse response && response.StatusCode == HttpStatusCode.NotFound) { - if (wex.Response is HttpWebResponse response && response.StatusCode == System.Net.HttpStatusCode.NotFound) - throw new FileMissingException(wex); - else - throw; - } - - return Task.CompletedTask; - } - - public IList SupportedCommands - { - get - { - return new List(new ICommandLineArgument[] { - new CommandLineArgument("use-ssl", CommandLineArgument.ArgumentType.Boolean, Strings.TahoeBackend.DescriptionUseSSLShort, Strings.TahoeBackend.DescriptionUseSSLLong), - }); + throw new FileMissingException(wex); } } - public string Description - { - get { return Strings.TahoeBackend.Description; } - } + public IList SupportedCommands => [ + .. SslOptionsHelper.GetSslOnlyOption(), + ]; - public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new[] { new Uri(m_url).Host }); + public string Description => Strings.TahoeBackend.Description; + + public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new[] { new System.Uri(m_url).Host }); #endregion @@ -277,53 +251,52 @@ namespace Duplicati.Library.Backend #region IStreamingBackend Members - public async Task PutAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) + public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken) { try { - System.Net.HttpWebRequest req = CreateRequest(remotename, ""); - req.Method = System.Net.WebRequestMethods.Http.Put; + var req = CreateRequest(remotename, ""); + req.Method = WebRequestMethods.Http.Put; req.ContentType = "application/binary"; try { req.ContentLength = stream.Length; } catch { } - Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req); - using (System.IO.Stream s = areq.GetRequestStream()) - await Utility.Utility.CopyStreamAsync(stream, s, true, cancelToken).ConfigureAwait(false); + var areq = new AsyncHttpRequest(req); + using (var s = areq.GetRequestStream()) + using (var t = s.ObserveWriteTimeout(m_timeouts.ReadWriteTimeout)) + await Utility.Utility.CopyStreamAsync(stream, t, true, cancelToken).ConfigureAwait(false); - using (System.Net.HttpWebResponse resp = (System.Net.HttpWebResponse)areq.GetResponse()) + using (var resp = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => (HttpWebResponse)areq.GetResponse()).ConfigureAwait(false)) { int code = (int)resp.StatusCode; - if (code < 200 || code >= 300) //For some reason Mono does not throw this automatically - throw new System.Net.WebException(resp.StatusDescription, null, System.Net.WebExceptionStatus.ProtocolError, resp); + if (code < 200 || code >= 300) + throw new WebException(resp.StatusDescription, null, WebExceptionStatus.ProtocolError, resp); } } - catch (System.Net.WebException wex) + catch (WebException wex) + when (wex.Response is HttpWebResponse response + && (response.StatusCode == HttpStatusCode.Conflict || response.StatusCode == HttpStatusCode.NotFound)) { - //Convert to better exception - if (wex.Response as System.Net.HttpWebResponse != null) - if ((wex.Response as System.Net.HttpWebResponse).StatusCode == System.Net.HttpStatusCode.Conflict || (wex.Response as System.Net.HttpWebResponse).StatusCode == System.Net.HttpStatusCode.NotFound) - throw new Interface.FolderMissingException(Strings.TahoeBackend.MissingFolderError(m_url, wex.Message), wex); - - throw; + throw new FolderMissingException(Strings.TahoeBackend.MissingFolderError(m_url, wex.Message), wex); } } public async Task GetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) { var req = CreateRequest(remotename, ""); - req.Method = System.Net.WebRequestMethods.Http.Get; + req.Method = WebRequestMethods.Http.Get; - var areq = new Utility.AsyncHttpRequest(req); - using (var resp = (System.Net.HttpWebResponse)areq.GetResponse()) + var areq = new AsyncHttpRequest(req); + using (var resp = await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, _ => (HttpWebResponse)areq.GetResponse()).ConfigureAwait(false)) { int code = (int)resp.StatusCode; - if (code < 200 || code >= 300) //For some reason Mono does not throw this automatically - throw new System.Net.WebException(resp.StatusDescription, null, System.Net.WebExceptionStatus.ProtocolError, resp); + if (code < 200 || code >= 300) + throw new WebException(resp.StatusDescription, null, WebExceptionStatus.ProtocolError, resp); using (var s = areq.GetResponseStream()) - await Utility.Utility.CopyStreamAsync(s, stream, true, cancelToken).ConfigureAwait(false); + using (var t = s.ObserveReadTimeout(m_timeouts.ReadWriteTimeout)) + await Utility.Utility.CopyStreamAsync(t, stream, true, cancelToken).ConfigureAwait(false); } } diff --git a/Duplicati/Library/Utility/Options/SslOptionsHelper.cs b/Duplicati/Library/Utility/Options/SslOptionsHelper.cs index 3932121bd..e065731a7 100644 --- a/Duplicati/Library/Utility/Options/SslOptionsHelper.cs +++ b/Duplicati/Library/Utility/Options/SslOptionsHelper.cs @@ -33,6 +33,19 @@ namespace Duplicati.Library.Utility.Options; /// public static class SslOptionsHelper { + /// + /// The name of the ssl option without a prefix + /// + public const string USE_SSL_OPTION = "use-ssl"; + /// + /// The name of the accept any ssl certificate option without a prefix + /// + public const string ACCEPT_ANY_SSL_CERTIFICATE_OPTION = "accept-any-ssl-certificate"; + /// + /// The name of the accept specified ssl hash option without a prefix + /// + public const string ACCEPT_SPECIFIED_SSL_HASH_OPTION = "accept-specified-ssl-hash"; + /// /// Gets the SSL certificate options for certificate only /// @@ -40,8 +53,18 @@ public static class SslOptionsHelper /// The SSL certificate options public static CommandLineArgument[] GetCertOnlyOptions(string? prefix = null) => [ - new CommandLineArgument($"{prefix}accept-any-ssl-certificate", CommandLineArgument.ArgumentType.Boolean, Strings.SslOptionsHelper.DescriptionAcceptAnyCertificateShort, Strings.SslOptionsHelper.DescriptionAcceptAnyCertificateLong, "false"), - new CommandLineArgument($"{prefix}accept-specified-ssl-hash", CommandLineArgument.ArgumentType.String, Strings.SslOptionsHelper.DescriptionAcceptHashShort, Strings.SslOptionsHelper.DescriptionAcceptHashLong) + new CommandLineArgument($"{prefix}{ACCEPT_ANY_SSL_CERTIFICATE_OPTION}", CommandLineArgument.ArgumentType.Boolean, Strings.SslOptionsHelper.DescriptionAcceptAnyCertificateShort, Strings.SslOptionsHelper.DescriptionAcceptAnyCertificateLong, "false"), + new CommandLineArgument($"{prefix}{ACCEPT_SPECIFIED_SSL_HASH_OPTION}", CommandLineArgument.ArgumentType.String, Strings.SslOptionsHelper.DescriptionAcceptHashShort, Strings.SslOptionsHelper.DescriptionAcceptHashLong) + ]; + + /// + /// Gets the SSL certificate options for SSL only + /// + /// An optional prefix for the options + /// The SSL certificate options + public static CommandLineArgument[] GetSslOnlyOption(string? prefix = null) => + [ + new CommandLineArgument($"{prefix}{USE_SSL_OPTION}", CommandLineArgument.ArgumentType.Boolean, Strings.SslOptionsHelper.DescriptionUseSSLShort, Strings.SslOptionsHelper.DescriptionUseSSLLong, "false") ]; /// @@ -51,7 +74,7 @@ public static class SslOptionsHelper /// The SSL certificate options public static CommandLineArgument[] GetOptions(string? prefix = null) => [ - new CommandLineArgument($"{prefix}use-ssl", CommandLineArgument.ArgumentType.Timespan, Strings.SslOptionsHelper.DescriptionUseSSLShort, Strings.SslOptionsHelper.DescriptionUseSSLLong), + .. GetSslOnlyOption(prefix), .. GetCertOnlyOptions(prefix) ]; @@ -63,10 +86,10 @@ public static class SslOptionsHelper /// The parsed SSL certificate options public static SslCertificateOptions Parse(IReadOnlyDictionary options, string? prefix = null) { - var acceptSpecificCertificatesString = options.TryGetValue($"{prefix}accept-specified-ssl-hash", out var value) ? value : null; + var acceptSpecificCertificatesString = options.TryGetValue($"{prefix}{ACCEPT_SPECIFIED_SSL_HASH_OPTION}", out var value) ? value : null; return new SslCertificateOptions( - Utility.ParseBoolOption(options, $"{prefix}use-ssl"), - Utility.ParseBoolOption(options, $"{prefix}accept-any-ssl-certificate"), + Utility.ParseBoolOption(options, $"{prefix}{USE_SSL_OPTION}"), + Utility.ParseBoolOption(options, $"{prefix}{ACCEPT_ANY_SSL_CERTIFICATE_OPTION}"), string.IsNullOrWhiteSpace(acceptSpecificCertificatesString) ? [] : acceptSpecificCertificatesString.Split([",", ";"], StringSplitOptions.RemoveEmptyEntries) ); } From 68d311ada060cca714d5bb3d3326015035ab996d Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 24 Mar 2025 10:15:06 +0100 Subject: [PATCH 36/48] Updated Tencent COS backend --- .../Library/Backend/TencentCOS/COSBackend.cs | 256 +++++++----------- ...uplicati.Library.Backend.TencentCOS.csproj | 2 + .../Library/Backend/TencentCOS/Strings.cs | 28 +- 3 files changed, 121 insertions(+), 165 deletions(-) diff --git a/Duplicati/Library/Backend/TencentCOS/COSBackend.cs b/Duplicati/Library/Backend/TencentCOS/COSBackend.cs index 66f2071e7..ad62ce76e 100644 --- a/Duplicati/Library/Backend/TencentCOS/COSBackend.cs +++ b/Duplicati/Library/Backend/TencentCOS/COSBackend.cs @@ -18,6 +18,7 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. + using COSXML; using COSXML.Auth; using COSXML.Model.Bucket; @@ -26,13 +27,9 @@ using COSXML.Model.Tag; using COSXML.Utils; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; +using Duplicati.Library.Utility.Options; using Newtonsoft.Json; -using System; -using System.Collections.Generic; -using System.IO; using System.Runtime.CompilerServices; -using System.Threading; -using System.Threading.Tasks; namespace Duplicati.Library.Backend.TencentCOS { @@ -41,7 +38,7 @@ namespace Duplicati.Library.Backend.TencentCOS /// en: https://intl.cloud.tencent.com/document/product/436 /// zh: https://cloud.tencent.com/document/product/436 /// - public class COS : IBackend, IStreamingBackend, IRenameEnabledBackend + public class COS : IBackend, IRenameEnabledBackend { private static readonly string LOGTAG = Logging.Log.LogTagFromType(); @@ -69,9 +66,14 @@ namespace Duplicati.Library.Backend.TencentCOS /// private const int KEY_DURATION_SECOND = 60 * 60; + /// + /// Tencent Cloud Object Storage Configuration + /// private readonly CosOptions _cosOptions; - - CosXml cosXml; + /// + /// The timeout values + /// + private readonly TimeoutOptionsHelper.Timeouts _timeouts; /// /// Tencent Cloud Object Storage Configuration @@ -81,100 +83,80 @@ namespace Duplicati.Library.Backend.TencentCOS /// /// Tencent Cloud Account APPID /// - public string Appid { get; set; } + public required string? Appid { get; set; } /// /// Cloud API Secret ID /// - public string SecretId { get; set; } + public required string SecretId { get; set; } /// /// Cloud API Secret Key /// - public string SecretKey { get; set; } + public required string SecretKey { get; set; } /// /// Bucket region ap-guangzhou ap-hongkong /// en: https://intl.cloud.tencent.com/document/product/436/6224 /// zh: https://cloud.tencent.com/document/product/436/6224 /// - public string Region { get; set; } + public required string? Region { get; set; } /// /// Bucket, format: BucketName-APPID /// - public string Bucket { get; set; } + public required string Bucket { get; set; } /// /// A path or subfolder in a bucket /// - public string Path { get; set; } + public required string? Path { get; set; } /// /// Storage class of the object /// en: https://intl.cloud.tencent.com/document/product/436/30925 /// zh: https://cloud.tencent.com/document/product/436/33417 /// - public string StorageClass { get; set; } + public required string? StorageClass { get; set; } } - public COS() { } - - public COS(string url, Dictionary options) + public COS() { - _cosOptions = new CosOptions(); + _cosOptions = null!; + _timeouts = null!; + } + public COS(string url, Dictionary options) + { var uri = new Utility.Uri(url?.Trim()); var prefix = uri.HostAndPath?.Trim()?.Trim('/')?.Trim('\\'); + var auth = AuthOptionsHelper.ParseWithAlias(options, uri, COS_SECRET_ID, COS_SECRET_KEY) + .RequireCredentials(); - if (!string.IsNullOrEmpty(prefix)) - { - _cosOptions.Path = prefix + "/"; - } + var bucket = options.GetValueOrDefault(COS_BUCKET); + if (string.IsNullOrWhiteSpace(bucket)) + throw new ArgumentException("COS bucket name is required"); - if (options.ContainsKey(COS_APP_ID)) + _cosOptions = new CosOptions() { - _cosOptions.Appid = options[COS_APP_ID]; - } + Appid = options.GetValueOrDefault(COS_APP_ID), + Region = options.GetValueOrDefault(COS_REGION), + SecretId = auth.Username!, + SecretKey = auth.Password!, + Bucket = bucket, + StorageClass = options.GetValueOrDefault(COS_STORAGE_CLASS), + Path = string.IsNullOrEmpty(prefix) ? null : Util.AppendDirSeparator(prefix, "/") + }; - if (options.ContainsKey(COS_REGION)) - { - _cosOptions.Region = options[COS_REGION]; - } - - if (options.ContainsKey(COS_SECRET_ID)) - { - _cosOptions.SecretId = options[COS_SECRET_ID]; - } - - if (options.ContainsKey(COS_SECRET_KEY)) - { - _cosOptions.SecretKey = options[COS_SECRET_KEY]; - } - - if (options.ContainsKey(COS_BUCKET)) - { - _cosOptions.Bucket = options[COS_BUCKET]; - } - - if (options.ContainsKey(COS_STORAGE_CLASS)) - { - _cosOptions.StorageClass = options[COS_STORAGE_CLASS]; - } + _timeouts = TimeoutOptionsHelper.Parse(options); } - CosXml GetCosXml() + private CosXml GetCosXml() { - string appid = _cosOptions.Appid; - string region = _cosOptions.Region; - string secretId = _cosOptions.SecretId; - string secretKey = _cosOptions.SecretKey; - - CosXmlConfig config = new CosXmlConfig.Builder() + var config = new CosXmlConfig.Builder() .SetConnectionTimeoutMs(CONNECTION_TIMEOUT_MS) .SetReadWriteTimeoutMs(READ_WRITE_TIMEOUT_MS) .IsHttps(COS_USE_SSL) - .SetAppid(appid) - .SetRegion(region) + .SetAppid(_cosOptions.Appid) + .SetRegion(_cosOptions.Region) .Build(); // Initialization QCloudCredentialProvider, COS SDK provides three ways: the permanent temporary keys custom - QCloudCredentialProvider cosCredentialProvider = new DefaultQCloudCredentialProvider(secretId, secretKey, KEY_DURATION_SECOND); - + var cosCredentialProvider = new DefaultQCloudCredentialProvider(_cosOptions.SecretId, _cosOptions.SecretKey, KEY_DURATION_SECOND); return new CosXmlServer(config, cosCredentialProvider); } @@ -182,11 +164,11 @@ namespace Duplicati.Library.Backend.TencentCOS public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { bool isTruncated = true; - string filename = null; + string? filename = null; while (isTruncated) { - cosXml = GetCosXml(); + var cosXml = GetCosXml(); var bucket = _cosOptions.Bucket; var prefix = _cosOptions.Path; @@ -200,9 +182,15 @@ namespace Duplicati.Library.Backend.TencentCOS request.SetPrefix(prefix); var tcs = new TaskCompletionSource(); - cancelToken.Register(() => tcs.TrySetCanceled()); - cosXml.GetBucket(request, (result) => tcs.SetResult(result as GetBucketResult), (clientEx, serverEx) => tcs.SetException((Exception)clientEx ?? serverEx)); - var result = await tcs.Task.ConfigureAwait(false); + var result = await Utility.Utility.WithTimeout(_timeouts.ListTimeout, cancelToken, async ct => + { + ct.Register(() => tcs.TrySetCanceled()); + cosXml.GetBucket(request, + (result) => tcs.SetResult((GetBucketResult)result), + (clientEx, serverEx) => tcs.SetException((Exception)clientEx ?? serverEx) + ); + return await tcs.Task.ConfigureAwait(false); + }).ConfigureAwait(false); var info = result.listBucket; @@ -212,16 +200,13 @@ namespace Duplicati.Library.Backend.TencentCOS foreach (var item in info.contentsList) { var last = DateTime.Parse(item.lastModified); - var fileName = item.key; if (!string.IsNullOrWhiteSpace(prefix)) { fileName = fileName.Substring(prefix.Length); if (fileName.StartsWith("/", StringComparison.Ordinal)) - { fileName = fileName.Trim('/'); - } } yield return new FileEntry(fileName, item.size, last, last); @@ -229,32 +214,21 @@ namespace Duplicati.Library.Backend.TencentCOS } } - public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) - { - using (FileStream fs = File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.Read)) - await PutAsync(remotename, fs, cancelToken).ConfigureAwait(false); - } - - public async Task GetAsync(string remotename, string filename, CancellationToken cancelToken) - { - using (var fs = File.Open(filename, FileMode.Create, FileAccess.Write, FileShare.None)) - await GetAsync(remotename, fs, cancelToken).ConfigureAwait(false); - } - - public Task DeleteAsync(string remotename, CancellationToken cancelToken) + public async Task DeleteAsync(string remotename, CancellationToken cancelToken) { try { - cosXml = GetCosXml(); - string bucket = _cosOptions.Bucket; - string key = GetFullKey(remotename); + var cosXml = GetCosXml(); + var bucket = _cosOptions.Bucket; + var key = GetFullKey(remotename); + await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancelToken, ct => + { + var request = new DeleteObjectRequest(bucket, key); + ct.Register(() => request.Cancel()); + request.SetSign(TimeUtils.GetCurrentTime(TimeUnit.SECONDS), 600); - var request = new DeleteObjectRequest(bucket, key); - cancelToken.Register(() => request.Cancel()); - - request.SetSign(TimeUtils.GetCurrentTime(TimeUnit.SECONDS), 600); - - var result = cosXml.DeleteObject(request); + cosXml.DeleteObject(request); + }).ConfigureAwait(false); } catch (COSXML.CosException.CosClientException clientEx) { @@ -266,21 +240,22 @@ namespace Duplicati.Library.Backend.TencentCOS Logging.Log.WriteErrorMessage(LOGTAG, "Delete", serverEx, "Delete failed: {0}, {1}", remotename, serverEx.GetInfo()); throw; } - - return Task.CompletedTask; } - public Task TestAsync(CancellationToken cancelToken) + public async Task TestAsync(CancellationToken cancelToken) { var json = JsonConvert.SerializeObject(_cosOptions); try { - cosXml = GetCosXml(); + var cosXml = GetCosXml(); string bucket = _cosOptions.Bucket; - var request = new HeadBucketRequest(bucket); - request.SetSign(TimeUtils.GetCurrentTime(TimeUnit.SECONDS), 600); - cancelToken.Register(() => request.Cancel()); - var result = cosXml.HeadBucket(request); + var result = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancelToken, ct => + { + var request = new HeadBucketRequest(bucket); + request.SetSign(TimeUtils.GetCurrentTime(TimeUnit.SECONDS), 600); + cancelToken.Register(() => request.Cancel()); + return cosXml.HeadBucket(request); + }).ConfigureAwait(false); Logging.Log.WriteInformationMessage(LOGTAG, "Test", "Request complete {0}: {1}, {2}", result.httpCode, json, result.GetResultInfo()); } @@ -294,8 +269,6 @@ namespace Duplicati.Library.Backend.TencentCOS Logging.Log.WriteErrorMessage(LOGTAG, "Test", serverEx, "Request failed: {0}, {1}", json, serverEx.GetInfo()); throw; } - - return Task.CompletedTask; } public Task CreateFolderAsync(CancellationToken cancelToken) @@ -306,27 +279,17 @@ namespace Duplicati.Library.Backend.TencentCOS public void Dispose() { - if (cosXml != null) - { - cosXml = null; - } } - public Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken) + public Task PutAsync(string remotename, string filename, CancellationToken cancelToken) { try { - cosXml = GetCosXml(); - string bucket = _cosOptions.Bucket; - string key = GetFullKey(remotename); + var cosXml = GetCosXml(); + var bucket = _cosOptions.Bucket; + var key = GetFullKey(remotename); - byte[] buffer = new byte[stream.Length]; - if (Utility.Utility.ForceStreamRead(stream, buffer, buffer.Length) != stream.Length) - { - throw new Exception("Bad file read"); - } - - PutObjectRequest request = new PutObjectRequest(bucket, key, buffer); + var request = new PutObjectRequest(bucket, key, filename); request.SetSign(TimeUtils.GetCurrentTime(TimeUnit.SECONDS), KEY_DURATION_SECOND); request.SetRequestHeader("Content-Type", "application/octet-stream"); @@ -352,13 +315,13 @@ namespace Duplicati.Library.Backend.TencentCOS return Task.CompletedTask; } - public async Task GetAsync(string remotename, Stream stream, CancellationToken cancelToken) + public async Task GetAsync(string remotename, string filename, CancellationToken cancelToken) { try { - cosXml = GetCosXml(); - string bucket = _cosOptions.Bucket; - string key = GetFullKey(remotename); + var cosXml = GetCosXml(); + var bucket = _cosOptions.Bucket; + var key = GetFullKey(remotename); var request = new GetObjectBytesRequest(bucket, key); @@ -368,9 +331,7 @@ namespace Duplicati.Library.Backend.TencentCOS var result = cosXml.GetObject(request); var bytes = result.content; - - using (var ms = new MemoryStream(bytes)) - await Utility.Utility.CopyStreamAsync(ms, stream, cancelToken).ConfigureAwait(false); + await File.WriteAllBytesAsync(filename, bytes, cancelToken).ConfigureAwait(false); } catch (COSXML.CosException.CosClientException clientEx) { @@ -388,7 +349,7 @@ namespace Duplicati.Library.Backend.TencentCOS { try { - cosXml = GetCosXml(); + var cosXml = GetCosXml(); var sourceAppid = _cosOptions.Appid; var sourceBucket = _cosOptions.Bucket; var sourceRegion = _cosOptions.Region; @@ -398,14 +359,17 @@ namespace Duplicati.Library.Backend.TencentCOS var bucket = _cosOptions.Bucket; var key = GetFullKey(newname); - var request = new CopyObjectRequest(bucket, key); - request.SetSign(TimeUtils.GetCurrentTime(TimeUnit.SECONDS), KEY_DURATION_SECOND); - request.SetCopySource(copySource); - request.SetCopyMetaDataDirective(COSXML.Common.CosMetaDataDirective.COPY); - cancelToken.Register(() => request.Cancel()); + var result = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancelToken, ct => + { + var request = new CopyObjectRequest(bucket, key); + request.SetSign(TimeUtils.GetCurrentTime(TimeUnit.SECONDS), KEY_DURATION_SECOND); + request.SetCopySource(copySource); + request.SetCopyMetaDataDirective(COSXML.Common.CosMetaDataDirective.COPY); + ct.Register(() => request.Cancel()); - var result = cosXml.CopyObject(request); + return cosXml.CopyObject(request); + }).ConfigureAwait(false); //Console.WriteLine(result.GetResultInfo()); @@ -429,30 +393,20 @@ namespace Duplicati.Library.Backend.TencentCOS public string ProtocolKey => "cos"; - public IList SupportedCommands - { - get - { - return new List(new ICommandLineArgument[] { - new CommandLineArgument(COS_APP_ID, CommandLineArgument.ArgumentType.String, Strings.COSBackend.COSAccountDescriptionShort, Strings.COSBackend.COSAccountDescriptionLong), - new CommandLineArgument(COS_REGION, CommandLineArgument.ArgumentType.String, Strings.COSBackend.COSLocationDescriptionShort, Strings.COSBackend.COSLocationDescriptionLong), - new CommandLineArgument(COS_SECRET_ID, CommandLineArgument.ArgumentType.String, Strings.COSBackend.COSAPISecretIdDescriptionShort, Strings.COSBackend.COSAPISecretIdDescriptionLong), - new CommandLineArgument(COS_SECRET_KEY, CommandLineArgument.ArgumentType.Password, Strings.COSBackend.COSAPISecretKeyDescriptionShort, Strings.COSBackend.COSAPISecretKeyDescriptionLong), - new CommandLineArgument(COS_BUCKET, CommandLineArgument.ArgumentType.String, Strings.COSBackend.COSBucketDescriptionShort, Strings.COSBackend.COSBucketDescriptionLong), - new CommandLineArgument(COS_STORAGE_CLASS, CommandLineArgument.ArgumentType.String, Strings.COSBackend.COSStorageClassDescriptionShort, Strings.COSBackend.COSStorageClassDescriptionLong) - }); - } - } + public IList SupportedCommands => [ + new CommandLineArgument(COS_APP_ID, CommandLineArgument.ArgumentType.String, Strings.COSBackend.COSAccountDescriptionShort, Strings.COSBackend.COSAccountDescriptionLong), + new CommandLineArgument(COS_REGION, CommandLineArgument.ArgumentType.String, Strings.COSBackend.COSLocationDescriptionShort, Strings.COSBackend.COSLocationDescriptionLong), + new CommandLineArgument(COS_SECRET_ID, CommandLineArgument.ArgumentType.String, Strings.COSBackend.COSAPISecretIdDescriptionShort, Strings.COSBackend.COSAPISecretIdDescriptionLong, null, [AuthOptionsHelper.AuthUsername]), + new CommandLineArgument(COS_SECRET_KEY, CommandLineArgument.ArgumentType.Password, Strings.COSBackend.COSAPISecretKeyDescriptionShort, Strings.COSBackend.COSAPISecretKeyDescriptionLong, null, [AuthOptionsHelper.AuthPassword]), + new CommandLineArgument(COS_BUCKET, CommandLineArgument.ArgumentType.String, Strings.COSBackend.COSBucketDescriptionShort, Strings.COSBackend.COSBucketDescriptionLong), + new CommandLineArgument(COS_STORAGE_CLASS, CommandLineArgument.ArgumentType.String, Strings.COSBackend.COSStorageClassDescriptionShort, Strings.COSBackend.COSStorageClassDescriptionLong), + .. TimeoutOptionsHelper.GetOptions() + .Where(x => x.Name != TimeoutOptionsHelper.READ_WRITE_TIMEOUT_OPTION) + ]; public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(Array.Empty()); private string GetFullKey(string name) - { - if (string.IsNullOrWhiteSpace(_cosOptions?.Path)) - { - return name; - } - return _cosOptions.Path + name; - } + => string.IsNullOrWhiteSpace(_cosOptions?.Path) ? name : _cosOptions.Path + name; } } diff --git a/Duplicati/Library/Backend/TencentCOS/Duplicati.Library.Backend.TencentCOS.csproj b/Duplicati/Library/Backend/TencentCOS/Duplicati.Library.Backend.TencentCOS.csproj index 0f7454f32..47dece887 100644 --- a/Duplicati/Library/Backend/TencentCOS/Duplicati.Library.Backend.TencentCOS.csproj +++ b/Duplicati/Library/Backend/TencentCOS/Duplicati.Library.Backend.TencentCOS.csproj @@ -3,6 +3,8 @@ net8.0 Library Copyright © 2025 Team Duplicati, MIT license + enable + enable diff --git a/Duplicati/Library/Backend/TencentCOS/Strings.cs b/Duplicati/Library/Backend/TencentCOS/Strings.cs index 9c5c8596a..0546f1816 100644 --- a/Duplicati/Library/Backend/TencentCOS/Strings.cs +++ b/Duplicati/Library/Backend/TencentCOS/Strings.cs @@ -24,20 +24,20 @@ namespace Duplicati.Library.Backend.Strings { internal static class COSBackend { - public static string Description { get { return LC.L(@"This backend can read and write data to the Tencent COS."); } } - public static string DisplayName { get { return LC.L(@"Tencent COS (Cloud Object Storage)"); } } - public static string COSAccountDescriptionLong { get { return LC.L(@"Account ID of Tencent Cloud Account."); } } - public static string COSAccountDescriptionShort { get { return LC.L(@"Account ID"); } } - public static string COSAPISecretIdDescriptionLong { get { return LC.L(@"Cloud API Secret ID."); } } - public static string COSAPISecretIdDescriptionShort { get { return LC.L(@"Secret ID"); } } - public static string COSAPISecretKeyDescriptionLong { get { return LC.L(@"Cloud API Secret Key."); } } - public static string COSAPISecretKeyDescriptionShort { get { return LC.L(@"Secret Key"); } } - public static string COSBucketDescriptionLong { get { return LC.L(@"Bucket name, format: BucketName-APPID"); } } - public static string COSBucketDescriptionShort { get { return LC.L(@"Bucket name"); } } - public static string COSLocationDescriptionLong { get { return LC.L(@"Region is the distribution area of ​​the Tencent cloud hosting machine room. The object storage COS data is stored in the storage buckets of these regions. https://intl.cloud.tencent.com/document/product/436/6224."); } } - public static string COSLocationDescriptionShort { get { return LC.L(@"Specify COS location constraints"); } } - public static string COSStorageClassDescriptionLong { get { return LC.L(@"Storage class of the object; check enumerated values at https://intl.cloud.tencent.com/document/product/436/30925."); } } - public static string COSStorageClassDescriptionShort { get { return LC.L(@"Storage class of the object"); } } + public static string Description => LC.L(@"This backend can read and write data to the Tencent COS."); + public static string DisplayName => LC.L(@"Tencent COS (Cloud Object Storage)"); + public static string COSAccountDescriptionLong => LC.L(@"Account ID of Tencent Cloud Account."); + public static string COSAccountDescriptionShort => LC.L(@"Account ID"); + public static string COSAPISecretIdDescriptionLong => LC.L(@"Cloud API Secret ID."); + public static string COSAPISecretIdDescriptionShort => LC.L(@"Secret ID"); + public static string COSAPISecretKeyDescriptionLong => LC.L(@"Cloud API Secret Key."); + public static string COSAPISecretKeyDescriptionShort => LC.L(@"Secret Key"); + public static string COSBucketDescriptionLong => LC.L(@"Bucket name, format: BucketName-APPID"); + public static string COSBucketDescriptionShort => LC.L(@"Bucket name"); + public static string COSLocationDescriptionLong => LC.L(@"Region is the distribution area of ​​the Tencent cloud hosting machine room. The object storage COS data is stored in the storage buckets of these regions. https://intl.cloud.tencent.com/document/product/436/6224."); + public static string COSLocationDescriptionShort => LC.L(@"Specify COS location constraints"); + public static string COSStorageClassDescriptionLong => LC.L(@"Storage class of the object; check enumerated values at https://intl.cloud.tencent.com/document/product/436/30925."); + public static string COSStorageClassDescriptionShort => LC.L(@"Storage class of the object"); } } From 631c80d6a884684bce4bb40a9087351881c0ecc3 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 24 Mar 2025 10:17:26 +0100 Subject: [PATCH 37/48] Cleaned up some option names --- .../Backend/AliyunOSS/AliyunOSSBackend.cs | 4 +- .../Backend/AzureBlob/AzureBlobBackend.cs | 4 +- Duplicati/Library/Backend/Backblaze/B2.cs | 4 +- .../Library/Backend/CloudFiles/CloudFiles.cs | 4 +- .../Backend/Idrivee2/Idrivee2Backend.cs | 4 +- .../Backend/OpenStack/OpenStackStorage.cs | 10 +-- Duplicati/Library/Backend/Rclone/Rclone.cs | 2 +- Duplicati/Library/Backend/S3/S3Backend.cs | 4 +- .../Library/Backend/SSHv2/SSHv2Backend.cs | 2 +- Duplicati/Library/Backend/Sia/Sia.cs | 2 +- .../Library/Backend/TencentCOS/COSBackend.cs | 6 +- .../Library/Backend/pCloud/pCloudBackend.cs | 2 +- .../Utility/Options/AuthIdOptionsHelper.cs | 6 +- .../Utility/Options/AuthOptionsHelper.cs | 61 ++----------------- .../Utility/Options/SslOptionsHelper.cs | 18 +++--- .../Utility/Options/TimeoutOptionsHelper.cs | 24 ++++---- 16 files changed, 54 insertions(+), 103 deletions(-) diff --git a/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs b/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs index 27681285b..d77909b41 100644 --- a/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs +++ b/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs @@ -298,8 +298,8 @@ namespace Duplicati.Library.Backend.AliyunOSS get { return new List([ - new CommandLineArgument(OSS_ACCESS_KEY_ID, CommandLineArgument.ArgumentType.String, Strings.OSSBackend.OSSAccessKeyIdDescriptionShort, Strings.OSSBackend.OSSAccessKeyIdDescriptionLong, null, [AuthOptionsHelper.AuthUsername]), - new CommandLineArgument(OSS_ACCESS_KEY_SECRET, CommandLineArgument.ArgumentType.Password, Strings.OSSBackend.OSSAccessKeySecretDescriptionShort, Strings.OSSBackend.OSSAccessKeySecretDescriptionLong, null, [AuthOptionsHelper.AuthPassword]), + new CommandLineArgument(OSS_ACCESS_KEY_ID, CommandLineArgument.ArgumentType.String, Strings.OSSBackend.OSSAccessKeyIdDescriptionShort, Strings.OSSBackend.OSSAccessKeyIdDescriptionLong, null, [AuthOptionsHelper.AuthUsernameOption]), + new CommandLineArgument(OSS_ACCESS_KEY_SECRET, CommandLineArgument.ArgumentType.Password, Strings.OSSBackend.OSSAccessKeySecretDescriptionShort, Strings.OSSBackend.OSSAccessKeySecretDescriptionLong, null, [AuthOptionsHelper.AuthPasswordOption]), new CommandLineArgument(OSS_BUCKET_NAME, CommandLineArgument.ArgumentType.String, Strings.OSSBackend.OSSBucketNameDescriptionShort, Strings.OSSBackend.OSSBucketNameDescriptionLong), new CommandLineArgument(OSS_ENDPOINT, CommandLineArgument.ArgumentType.String, Strings.OSSBackend.OSSEndpointDescriptionShort, Strings.OSSBackend.OSSEndpointDescriptionLong), .. TimeoutOptionsHelper.GetOptions() diff --git a/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs b/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs index ca8543911..68810cfef 100644 --- a/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs +++ b/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs @@ -108,13 +108,13 @@ namespace Duplicati.Library.Backend.AzureBlob Strings.AzureBlobBackend.StorageAccountNameDescriptionShort, Strings.AzureBlobBackend.StorageAccountNameDescriptionLong, null, - [AuthOptionsHelper.AuthUsername]), + [AuthOptionsHelper.AuthUsernameOption]), new CommandLineArgument("azure-access-key", CommandLineArgument.ArgumentType.Password, Strings.AzureBlobBackend.AccessKeyDescriptionShort, Strings.AzureBlobBackend.AccessKeyDescriptionLong, null, - [AuthOptionsHelper.AuthPassword]), + [AuthOptionsHelper.AuthPasswordOption]), new CommandLineArgument("azure-access-sas-token", CommandLineArgument.ArgumentType.Password, Strings.AzureBlobBackend.SasTokenDescriptionShort, diff --git a/Duplicati/Library/Backend/Backblaze/B2.cs b/Duplicati/Library/Backend/Backblaze/B2.cs index 0d7c27477..6d219a052 100644 --- a/Duplicati/Library/Backend/Backblaze/B2.cs +++ b/Duplicati/Library/Backend/Backblaze/B2.cs @@ -298,8 +298,8 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend /// public IList SupportedCommands => new List([ - new CommandLineArgument(B2_ID_OPTION, CommandLineArgument.ArgumentType.String, Strings.B2.B2accountidDescriptionShort, Strings.B2.B2accountidDescriptionLong, null, [AuthOptionsHelper.AuthUsername], null), - new CommandLineArgument(B2_KEY_OPTION, CommandLineArgument.ArgumentType.Password, Strings.B2.B2applicationkeyDescriptionShort, Strings.B2.B2applicationkeyDescriptionLong, null, [AuthOptionsHelper.AuthPassword], null), + new CommandLineArgument(B2_ID_OPTION, CommandLineArgument.ArgumentType.String, Strings.B2.B2accountidDescriptionShort, Strings.B2.B2accountidDescriptionLong, null, [AuthOptionsHelper.AuthUsernameOption], null), + new CommandLineArgument(B2_KEY_OPTION, CommandLineArgument.ArgumentType.Password, Strings.B2.B2applicationkeyDescriptionShort, Strings.B2.B2applicationkeyDescriptionLong, null, [AuthOptionsHelper.AuthPasswordOption], null), new CommandLineArgument(B2_CREATE_BUCKET_TYPE_OPTION, CommandLineArgument.ArgumentType.String, Strings.B2.B2createbuckettypeDescriptionShort, Strings.B2.B2createbuckettypeDescriptionLong, DEFAULT_BUCKET_TYPE), new CommandLineArgument(B2_PAGESIZE_OPTION, CommandLineArgument.ArgumentType.Integer, Strings.B2.B2pagesizeDescriptionShort, Strings.B2.B2pagesizeDescriptionLong, DEFAULT_PAGE_SIZE.ToString()), new CommandLineArgument(B2_DOWNLOAD_URL_OPTION, CommandLineArgument.ArgumentType.String, Strings.B2.B2downloadurlDescriptionShort, Strings.B2.B2downloadurlDescriptionLong), diff --git a/Duplicati/Library/Backend/CloudFiles/CloudFiles.cs b/Duplicati/Library/Backend/CloudFiles/CloudFiles.cs index 72e627f4a..80541a569 100644 --- a/Duplicati/Library/Backend/CloudFiles/CloudFiles.cs +++ b/Duplicati/Library/Backend/CloudFiles/CloudFiles.cs @@ -243,8 +243,8 @@ namespace Duplicati.Library.Backend public IList SupportedCommands => [ - new CommandLineArgument(AUTH_USERNAME_OPTION, CommandLineArgument.ArgumentType.String, Strings.CloudFiles.DescriptionUsernameShort, Strings.CloudFiles.DescriptionUsernameLong, null, [AuthOptionsHelper.AuthUsername] ), - new CommandLineArgument(AUTH_PASSWORD_OPTION, CommandLineArgument.ArgumentType.Password, Strings.CloudFiles.DescriptionPasswordShort, Strings.CloudFiles.DescriptionPasswordLong, null, [AuthOptionsHelper.AuthPassword]), + new CommandLineArgument(AUTH_USERNAME_OPTION, CommandLineArgument.ArgumentType.String, Strings.CloudFiles.DescriptionUsernameShort, Strings.CloudFiles.DescriptionUsernameLong, null, [AuthOptionsHelper.AuthUsernameOption] ), + new CommandLineArgument(AUTH_PASSWORD_OPTION, CommandLineArgument.ArgumentType.Password, Strings.CloudFiles.DescriptionPasswordShort, Strings.CloudFiles.DescriptionPasswordLong, null, [AuthOptionsHelper.AuthPasswordOption]), new CommandLineArgument("cloudfiles-uk-account", CommandLineArgument.ArgumentType.Boolean, Strings.CloudFiles.DescriptionUKAccountShort, Strings.CloudFiles.DescriptionUKAccountLong("cloudfiles-authentication-url", AUTH_URL_UK)), new CommandLineArgument("cloudfiles-authentication-url", CommandLineArgument.ArgumentType.String, Strings.CloudFiles.DescriptionAuthenticationURLShort, Strings.CloudFiles.DescriptionAuthenticationURLLong_v2("cloudfiles-uk-account"), AUTH_URL_US), .. TimeoutOptionsHelper.GetOptions(), diff --git a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs index 5fd7fe442..d4f5d7809 100644 --- a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs +++ b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs @@ -146,8 +146,8 @@ namespace Duplicati.Library.Backend public IList SupportedCommands => [ .. S3AwsClient.GetAwsExtendedOptions(), - new CommandLineArgument(AUTH_USERNAME_OPTION, CommandLineArgument.ArgumentType.String, Strings.Idrivee2Backend.KeyIDDescriptionShort, Strings.Idrivee2Backend.KeyIDDescriptionLong,null, [AuthOptionsHelper.AuthUsername], null), - new CommandLineArgument(AUTH_PASSWORD_OPTION, CommandLineArgument.ArgumentType.Password, Strings.Idrivee2Backend.KeySecretDescriptionShort, Strings.Idrivee2Backend.KeySecretDescriptionLong, null, [AuthOptionsHelper.AuthPassword ], null), + new CommandLineArgument(AUTH_USERNAME_OPTION, CommandLineArgument.ArgumentType.String, Strings.Idrivee2Backend.KeyIDDescriptionShort, Strings.Idrivee2Backend.KeyIDDescriptionLong,null, [AuthOptionsHelper.AuthUsernameOption], null), + new CommandLineArgument(AUTH_PASSWORD_OPTION, CommandLineArgument.ArgumentType.Password, Strings.Idrivee2Backend.KeySecretDescriptionShort, Strings.Idrivee2Backend.KeySecretDescriptionLong, null, [AuthOptionsHelper.AuthPasswordOption ], null), .. TimeoutOptionsHelper.GetOptions(), ]; diff --git a/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs b/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs index 416200c64..7ea70dc4d 100644 --- a/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs +++ b/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs @@ -356,7 +356,7 @@ namespace Duplicati.Library.Backend.OpenStack options.TryGetValue(REGION_OPTION, out m_region); if (!auth.HasUsername) - throw new UserInformationException(Strings.OpenStack.MissingOptionError(AuthOptionsHelper.AuthUsername), "OpenStackMissingUsername"); + throw new UserInformationException(Strings.OpenStack.MissingOptionError(AuthOptionsHelper.AuthUsernameOption), "OpenStackMissingUsername"); m_username = auth.Username!; m_password = auth.Password; @@ -372,7 +372,7 @@ namespace Duplicati.Library.Backend.OpenStack { case "v3": if (string.IsNullOrWhiteSpace(m_password)) - throw new UserInformationException(Strings.OpenStack.MissingOptionError(AuthOptionsHelper.AuthPassword), "OpenStackMissingPassword"); + throw new UserInformationException(Strings.OpenStack.MissingOptionError(AuthOptionsHelper.AuthPasswordOption), "OpenStackMissingPassword"); if (string.IsNullOrWhiteSpace(m_domainName)) throw new UserInformationException(Strings.OpenStack.MissingOptionError(DOMAINNAME_OPTION), "OpenStackMissingDomainName"); if (string.IsNullOrWhiteSpace(m_tenantName)) @@ -383,7 +383,7 @@ namespace Duplicati.Library.Backend.OpenStack if (string.IsNullOrWhiteSpace(m_apikey)) { if (string.IsNullOrWhiteSpace(m_password)) - throw new UserInformationException(Strings.OpenStack.MissingOptionError(AuthOptionsHelper.AuthPassword), "OpenStackMissingPassword"); + throw new UserInformationException(Strings.OpenStack.MissingOptionError(AuthOptionsHelper.AuthPasswordOption), "OpenStackMissingPassword"); if (string.IsNullOrWhiteSpace(m_tenantName)) throw new UserInformationException(Strings.OpenStack.MissingOptionError(TENANTNAME_OPTION), "OpenStackMissingTenantName"); } @@ -649,8 +649,8 @@ namespace Duplicati.Library.Backend.OpenStack return [ new CommandLineArgument(DOMAINNAME_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.DomainnameOptionShort, Strings.OpenStack.DomainnameOptionLong), - new CommandLineArgument(AuthOptionsHelper.AuthUsername, CommandLineArgument.ArgumentType.String, Strings.OpenStack.UsernameOptionShort, Strings.OpenStack.UsernameOptionLong), - new CommandLineArgument(AuthOptionsHelper.AuthPassword, CommandLineArgument.ArgumentType.Password, Strings.OpenStack.PasswordOptionShort, Strings.OpenStack.PasswordOptionLong(TENANTNAME_OPTION)), + new CommandLineArgument(AuthOptionsHelper.AuthUsernameOption, CommandLineArgument.ArgumentType.String, Strings.OpenStack.UsernameOptionShort, Strings.OpenStack.UsernameOptionLong), + new CommandLineArgument(AuthOptionsHelper.AuthPasswordOption, CommandLineArgument.ArgumentType.Password, Strings.OpenStack.PasswordOptionShort, Strings.OpenStack.PasswordOptionLong(TENANTNAME_OPTION)), new CommandLineArgument(TENANTNAME_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.TenantnameOptionShort, Strings.OpenStack.TenantnameOptionLong), new CommandLineArgument(APIKEY_OPTION, CommandLineArgument.ArgumentType.Password, Strings.OpenStack.ApikeyOptionShort, Strings.OpenStack.ApikeyOptionLong), new CommandLineArgument(AUTHURI_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.AuthuriOptionShort, Strings.OpenStack.AuthuriOptionLong(authuris.ToString())), diff --git a/Duplicati/Library/Backend/Rclone/Rclone.cs b/Duplicati/Library/Backend/Rclone/Rclone.cs index e05081dbd..99f1873ff 100644 --- a/Duplicati/Library/Backend/Rclone/Rclone.cs +++ b/Duplicati/Library/Backend/Rclone/Rclone.cs @@ -289,7 +289,7 @@ namespace Duplicati.Library.Backend new CommandLineArgument(OPTION_RCLONE, CommandLineArgument.ArgumentType.String, Strings.Rclone.RcloneOptionRcloneShort, Strings.Rclone.RcloneOptionRcloneLong, ""), new CommandLineArgument(OPTION_RCLONE_EXECUTABLE, CommandLineArgument.ArgumentType.String, Strings.Rclone.RcloneExecutableShort, Strings.Rclone.RcloneExecutableLong, "rclone"), .. TimeoutOptionsHelper.GetOptions() - .Where(x => x.Name != TimeoutOptionsHelper.READ_WRITE_TIMEOUT_OPTION) + .Where(x => x.Name != TimeoutOptionsHelper.ReadWriteTimeoutOption) ]; public string Description => Strings.Rclone.Description; diff --git a/Duplicati/Library/Backend/S3/S3Backend.cs b/Duplicati/Library/Backend/S3/S3Backend.cs index 1b452ec4d..0e0f7c440 100644 --- a/Duplicati/Library/Backend/S3/S3Backend.cs +++ b/Duplicati/Library/Backend/S3/S3Backend.cs @@ -315,8 +315,8 @@ namespace Duplicati.Library.Backend var exts = S3AwsClient.GetAwsExtendedOptions(); return [ - new CommandLineArgument("aws-access-key-id", CommandLineArgument.ArgumentType.String, Strings.S3Backend.AMZUserIDDescriptionShort, Strings.S3Backend.AMZUserIDDescriptionLong, null, [AuthOptionsHelper.AuthUsername], null), - new CommandLineArgument("aws-secret-access-key", CommandLineArgument.ArgumentType.Password, Strings.S3Backend.AMZKeyDescriptionShort, Strings.S3Backend.AMZKeyDescriptionLong,null, [AuthOptionsHelper.AuthPassword], null ), + new CommandLineArgument("aws-access-key-id", CommandLineArgument.ArgumentType.String, Strings.S3Backend.AMZUserIDDescriptionShort, Strings.S3Backend.AMZUserIDDescriptionLong, null, [AuthOptionsHelper.AuthUsernameOption], null), + new CommandLineArgument("aws-secret-access-key", CommandLineArgument.ArgumentType.Password, Strings.S3Backend.AMZKeyDescriptionShort, Strings.S3Backend.AMZKeyDescriptionLong,null, [AuthOptionsHelper.AuthPasswordOption], null ), new CommandLineArgument(STORAGECLASS_OPTION, CommandLineArgument.ArgumentType.String, Strings.S3Backend.S3StorageclassDescriptionShort, Strings.S3Backend.S3StorageclassDescriptionLong), new CommandLineArgument(SERVER_NAME, CommandLineArgument.ArgumentType.String, Strings.S3Backend.S3ServerNameDescriptionShort, Strings.S3Backend.S3ServerNameDescriptionLong(hostnames.ToString()), DEFAULT_S3_HOST), new CommandLineArgument(LOCATION_OPTION, CommandLineArgument.ArgumentType.String, Strings.S3Backend.S3LocationDescriptionShort, Strings.S3Backend.S3LocationDescriptionLong(locations.ToString())), diff --git a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs index 02c6a2395..e715a3eff 100644 --- a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs +++ b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs @@ -190,7 +190,7 @@ namespace Duplicati.Library.Backend Strings.SSHv2Backend.DescriptionSshkeyLong(KEYFILE_URI)), new CommandLineArgument(SSH_TIMEOUT_OPTION, CommandLineArgument.ArgumentType.Timespan, Strings.SSHv2Backend.DescriptionSshtimeoutShort, Strings.SSHv2Backend.DescriptionSshtimeoutLong, - "0", null,null, Strings.SSHv2Backend.TimeoutDeprecated(SSH_TIMEOUT_OPTION, TimeoutOptionsHelper.SHORT_TIMEOUT_OPTION, TimeoutOptionsHelper.LIST_TIMEOUT_OPTION, TimeoutOptionsHelper.READ_WRITE_TIMEOUT_OPTION)), + "0", null,null, Strings.SSHv2Backend.TimeoutDeprecated(SSH_TIMEOUT_OPTION, TimeoutOptionsHelper.ShortTimeoutOption, TimeoutOptionsHelper.ListTimeoutOption, TimeoutOptionsHelper.ReadWriteTimeoutOption)), new CommandLineArgument(SSH_KEEPALIVE_OPTION, CommandLineArgument.ArgumentType.Timespan, Strings.SSHv2Backend.DescriptionSshkeepaliveShort, Strings.SSHv2Backend.DescriptionSshkeepaliveLong, "0"), diff --git a/Duplicati/Library/Backend/Sia/Sia.cs b/Duplicati/Library/Backend/Sia/Sia.cs index 56144e239..2d41d9e0d 100644 --- a/Duplicati/Library/Backend/Sia/Sia.cs +++ b/Duplicati/Library/Backend/Sia/Sia.cs @@ -453,7 +453,7 @@ namespace Duplicati.Library.Backend.Sia new CommandLineArgument(SIA_PASSWORD, CommandLineArgument.ArgumentType.Password, Strings.Sia.SiaPasswordShort, Strings.Sia.SiaPasswordLong, null), new CommandLineArgument(SIA_REDUNDANCY, CommandLineArgument.ArgumentType.Decimal, Strings.Sia.SiaRedundancyDescriptionShort, Strings.Sia.SiaRedundancyDescriptionLong, "1.5"), .. TimeoutOptionsHelper.GetOptions() - .Where(x => x.Name != TimeoutOptionsHelper.READ_WRITE_TIMEOUT_OPTION) + .Where(x => x.Name != TimeoutOptionsHelper.ReadWriteTimeoutOption) ]); public string Description => Strings.Sia.Description; diff --git a/Duplicati/Library/Backend/TencentCOS/COSBackend.cs b/Duplicati/Library/Backend/TencentCOS/COSBackend.cs index ad62ce76e..332018b21 100644 --- a/Duplicati/Library/Backend/TencentCOS/COSBackend.cs +++ b/Duplicati/Library/Backend/TencentCOS/COSBackend.cs @@ -396,12 +396,12 @@ namespace Duplicati.Library.Backend.TencentCOS public IList SupportedCommands => [ new CommandLineArgument(COS_APP_ID, CommandLineArgument.ArgumentType.String, Strings.COSBackend.COSAccountDescriptionShort, Strings.COSBackend.COSAccountDescriptionLong), new CommandLineArgument(COS_REGION, CommandLineArgument.ArgumentType.String, Strings.COSBackend.COSLocationDescriptionShort, Strings.COSBackend.COSLocationDescriptionLong), - new CommandLineArgument(COS_SECRET_ID, CommandLineArgument.ArgumentType.String, Strings.COSBackend.COSAPISecretIdDescriptionShort, Strings.COSBackend.COSAPISecretIdDescriptionLong, null, [AuthOptionsHelper.AuthUsername]), - new CommandLineArgument(COS_SECRET_KEY, CommandLineArgument.ArgumentType.Password, Strings.COSBackend.COSAPISecretKeyDescriptionShort, Strings.COSBackend.COSAPISecretKeyDescriptionLong, null, [AuthOptionsHelper.AuthPassword]), + new CommandLineArgument(COS_SECRET_ID, CommandLineArgument.ArgumentType.String, Strings.COSBackend.COSAPISecretIdDescriptionShort, Strings.COSBackend.COSAPISecretIdDescriptionLong, null, [AuthOptionsHelper.AuthUsernameOption]), + new CommandLineArgument(COS_SECRET_KEY, CommandLineArgument.ArgumentType.Password, Strings.COSBackend.COSAPISecretKeyDescriptionShort, Strings.COSBackend.COSAPISecretKeyDescriptionLong, null, [AuthOptionsHelper.AuthPasswordOption]), new CommandLineArgument(COS_BUCKET, CommandLineArgument.ArgumentType.String, Strings.COSBackend.COSBucketDescriptionShort, Strings.COSBackend.COSBucketDescriptionLong), new CommandLineArgument(COS_STORAGE_CLASS, CommandLineArgument.ArgumentType.String, Strings.COSBackend.COSStorageClassDescriptionShort, Strings.COSBackend.COSStorageClassDescriptionLong), .. TimeoutOptionsHelper.GetOptions() - .Where(x => x.Name != TimeoutOptionsHelper.READ_WRITE_TIMEOUT_OPTION) + .Where(x => x.Name != TimeoutOptionsHelper.ReadWriteTimeoutOption) ]; public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(Array.Empty()); diff --git a/Duplicati/Library/Backend/pCloud/pCloudBackend.cs b/Duplicati/Library/Backend/pCloud/pCloudBackend.cs index 9b208f03a..7fa0edb48 100644 --- a/Duplicati/Library/Backend/pCloud/pCloudBackend.cs +++ b/Duplicati/Library/Backend/pCloud/pCloudBackend.cs @@ -162,7 +162,7 @@ public class pCloudBackend : IStreamingBackend /// Implementation of interface property to return supported command parameters /// public IList SupportedCommands => [ - new CommandLineArgument(AuthIdOptionsHelper.AuthId, + new CommandLineArgument(AuthIdOptionsHelper.AuthIdOption, CommandLineArgument.ArgumentType.Password, Strings.pCloudBackend.AuthPasswordDescriptionShort, Strings.pCloudBackend.AuthPasswordDescriptionLong(TOKEN_URL)), diff --git a/Duplicati/Library/Utility/Options/AuthIdOptionsHelper.cs b/Duplicati/Library/Utility/Options/AuthIdOptionsHelper.cs index c2126a0d9..2689d19d0 100644 --- a/Duplicati/Library/Utility/Options/AuthIdOptionsHelper.cs +++ b/Duplicati/Library/Utility/Options/AuthIdOptionsHelper.cs @@ -35,7 +35,7 @@ public static class AuthIdOptionsHelper /// /// The authentication ID option, without a prefix /// - public const string AuthId = "authid"; + public const string AuthIdOption = "authid"; /// /// Gets the authentication options /// @@ -44,7 +44,7 @@ public static class AuthIdOptionsHelper /// The authentication options public static CommandLineArgument[] GetOptions(string tokenurl, string? prefix = null) => [ - new CommandLineArgument($"{prefix}{AuthId}", CommandLineArgument.ArgumentType.String, Strings.AuthIdSettingsHelper.AuthidShort, Strings.AuthIdSettingsHelper.AuthidLong(tokenurl)) + new CommandLineArgument($"{prefix}{AuthIdOption}", CommandLineArgument.ArgumentType.String, Strings.AuthIdSettingsHelper.AuthidShort, Strings.AuthIdSettingsHelper.AuthidLong(tokenurl)) ]; /// @@ -54,7 +54,7 @@ public static class AuthIdOptionsHelper /// An optional prefix for the options /// The parsed authentication options public static AuthIdOptions Parse(IReadOnlyDictionary options, string? prefix = null) - => new AuthIdOptions(options.GetValueOrDefault($"{prefix}{AuthId}")); + => new AuthIdOptions(options.GetValueOrDefault($"{prefix}{AuthIdOption}")); /// /// Structure to hold the authentication options diff --git a/Duplicati/Library/Utility/Options/AuthOptionsHelper.cs b/Duplicati/Library/Utility/Options/AuthOptionsHelper.cs index bcd06b2c6..a812244f7 100644 --- a/Duplicati/Library/Utility/Options/AuthOptionsHelper.cs +++ b/Duplicati/Library/Utility/Options/AuthOptionsHelper.cs @@ -35,11 +35,11 @@ public static class AuthOptionsHelper /// /// The authentication username option, without a prefix /// - public const string AuthUsername = "auth-username"; + public const string AuthUsernameOption = "auth-username"; /// /// The authentication password option, without a prefix /// - public const string AuthPassword = "auth-password"; + public const string AuthPasswordOption = "auth-password"; /// /// Gets the authentication options @@ -48,59 +48,10 @@ public static class AuthOptionsHelper /// The authentication options public static CommandLineArgument[] GetOptions(string? prefix = null) => [ - new CommandLineArgument($"{prefix}auth-username", CommandLineArgument.ArgumentType.String, Strings.AuthSettingsHelper.DescriptionAuthUsernameShort, Strings.AuthSettingsHelper.DescriptionAuthUsernameLong), - new CommandLineArgument($"{prefix}auth-password", CommandLineArgument.ArgumentType.Password, Strings.AuthSettingsHelper.DescriptionAuthPasswordShort, Strings.AuthSettingsHelper.DescriptionAuthPasswordLong) + new CommandLineArgument($"{prefix}{AuthUsernameOption}", CommandLineArgument.ArgumentType.String, Strings.AuthSettingsHelper.DescriptionAuthUsernameShort, Strings.AuthSettingsHelper.DescriptionAuthUsernameLong), + new CommandLineArgument($"{prefix}{AuthPasswordOption}", CommandLineArgument.ArgumentType.Password, Strings.AuthSettingsHelper.DescriptionAuthPasswordShort, Strings.AuthSettingsHelper.DescriptionAuthPasswordLong) ]; - /// - /// Gets the authentication options - /// - /// The name of the username options - /// The name of the password options - /// The authentication options - public static CommandLineArgument[] GetOptionsWithAlias(string username, string password) => - [ - new CommandLineArgument(username, CommandLineArgument.ArgumentType.String, Strings.AuthSettingsHelper.DescriptionAuthUsernameShort, Strings.AuthSettingsHelper.DescriptionAuthUsernameLong, null, ["auth-username"]), - new CommandLineArgument(password, CommandLineArgument.ArgumentType.Password, Strings.AuthSettingsHelper.DescriptionAuthPasswordShort, Strings.AuthSettingsHelper.DescriptionAuthPasswordLong, null, ["auth-password"]) - ]; - - /* - - TODO: See if we can do without these, so we only have one way of parsing the auth options - - /// - /// Parses the authentication options from a dictionary - /// - /// The dictionary to parse - /// An optional prefix for the options - /// The parsed authentication options - public static AuthOptions Parse(IReadOnlyDictionary options, string? prefix = null) - => new AuthOptions( - options.GetValueOrDefault($"{prefix}auth-username"), - options.GetValueOrDefault($"{prefix}auth-password") - ); - - /// - /// Parses the authentication options from a dictionary - /// - /// The dictionary to parse - /// The name of the username options - /// The name of the password options - /// The parsed authentication options - public static AuthOptions ParseWithAlias(IReadOnlyDictionary options, string username, string password) - { - var optionUsername = options.GetValueOrDefault(username); - var optionPassword = options.GetValueOrDefault(password); - - if (string.IsNullOrWhiteSpace(optionUsername)) - optionUsername = options.GetValueOrDefault("auth-username"); - if (string.IsNullOrWhiteSpace(optionPassword)) - optionPassword = options.GetValueOrDefault("auth-password"); - - return new AuthOptions(optionUsername, optionPassword); - } - */ - /// /// Parses the authentication options from a dictionary /// @@ -133,8 +84,8 @@ public static class AuthOptionsHelper /// The parsed authentication options public static AuthOptions Parse(IReadOnlyDictionary options, Uri uri, string? prefix = null) { - var optionUsername = options.GetValueOrDefault($"{prefix}auth-username"); - var optionPassword = options.GetValueOrDefault($"{prefix}auth-password"); + var optionUsername = options.GetValueOrDefault($"{prefix}{AuthUsernameOption}"); + var optionPassword = options.GetValueOrDefault($"{prefix}{AuthPasswordOption}"); // Prefer the URL values, if set string? username = null; diff --git a/Duplicati/Library/Utility/Options/SslOptionsHelper.cs b/Duplicati/Library/Utility/Options/SslOptionsHelper.cs index e065731a7..d4d568faf 100644 --- a/Duplicati/Library/Utility/Options/SslOptionsHelper.cs +++ b/Duplicati/Library/Utility/Options/SslOptionsHelper.cs @@ -36,15 +36,15 @@ public static class SslOptionsHelper /// /// The name of the ssl option without a prefix /// - public const string USE_SSL_OPTION = "use-ssl"; + public const string UseSSLOption = "use-ssl"; /// /// The name of the accept any ssl certificate option without a prefix /// - public const string ACCEPT_ANY_SSL_CERTIFICATE_OPTION = "accept-any-ssl-certificate"; + public const string AcceptAnySslCertificateOption = "accept-any-ssl-certificate"; /// /// The name of the accept specified ssl hash option without a prefix /// - public const string ACCEPT_SPECIFIED_SSL_HASH_OPTION = "accept-specified-ssl-hash"; + public const string AcceptSpecifiedSslHashOption = "accept-specified-ssl-hash"; /// /// Gets the SSL certificate options for certificate only @@ -53,8 +53,8 @@ public static class SslOptionsHelper /// The SSL certificate options public static CommandLineArgument[] GetCertOnlyOptions(string? prefix = null) => [ - new CommandLineArgument($"{prefix}{ACCEPT_ANY_SSL_CERTIFICATE_OPTION}", CommandLineArgument.ArgumentType.Boolean, Strings.SslOptionsHelper.DescriptionAcceptAnyCertificateShort, Strings.SslOptionsHelper.DescriptionAcceptAnyCertificateLong, "false"), - new CommandLineArgument($"{prefix}{ACCEPT_SPECIFIED_SSL_HASH_OPTION}", CommandLineArgument.ArgumentType.String, Strings.SslOptionsHelper.DescriptionAcceptHashShort, Strings.SslOptionsHelper.DescriptionAcceptHashLong) + new CommandLineArgument($"{prefix}{AcceptAnySslCertificateOption}", CommandLineArgument.ArgumentType.Boolean, Strings.SslOptionsHelper.DescriptionAcceptAnyCertificateShort, Strings.SslOptionsHelper.DescriptionAcceptAnyCertificateLong, "false"), + new CommandLineArgument($"{prefix}{AcceptSpecifiedSslHashOption}", CommandLineArgument.ArgumentType.String, Strings.SslOptionsHelper.DescriptionAcceptHashShort, Strings.SslOptionsHelper.DescriptionAcceptHashLong) ]; /// @@ -64,7 +64,7 @@ public static class SslOptionsHelper /// The SSL certificate options public static CommandLineArgument[] GetSslOnlyOption(string? prefix = null) => [ - new CommandLineArgument($"{prefix}{USE_SSL_OPTION}", CommandLineArgument.ArgumentType.Boolean, Strings.SslOptionsHelper.DescriptionUseSSLShort, Strings.SslOptionsHelper.DescriptionUseSSLLong, "false") + new CommandLineArgument($"{prefix}{UseSSLOption}", CommandLineArgument.ArgumentType.Boolean, Strings.SslOptionsHelper.DescriptionUseSSLShort, Strings.SslOptionsHelper.DescriptionUseSSLLong, "false") ]; /// @@ -86,10 +86,10 @@ public static class SslOptionsHelper /// The parsed SSL certificate options public static SslCertificateOptions Parse(IReadOnlyDictionary options, string? prefix = null) { - var acceptSpecificCertificatesString = options.TryGetValue($"{prefix}{ACCEPT_SPECIFIED_SSL_HASH_OPTION}", out var value) ? value : null; + var acceptSpecificCertificatesString = options.TryGetValue($"{prefix}{AcceptSpecifiedSslHashOption}", out var value) ? value : null; return new SslCertificateOptions( - Utility.ParseBoolOption(options, $"{prefix}{USE_SSL_OPTION}"), - Utility.ParseBoolOption(options, $"{prefix}{ACCEPT_ANY_SSL_CERTIFICATE_OPTION}"), + Utility.ParseBoolOption(options, $"{prefix}{UseSSLOption}"), + Utility.ParseBoolOption(options, $"{prefix}{AcceptAnySslCertificateOption}"), string.IsNullOrWhiteSpace(acceptSpecificCertificatesString) ? [] : acceptSpecificCertificatesString.Split([",", ";"], StringSplitOptions.RemoveEmptyEntries) ); } diff --git a/Duplicati/Library/Utility/Options/TimeoutOptionsHelper.cs b/Duplicati/Library/Utility/Options/TimeoutOptionsHelper.cs index cd93d862c..1864df23e 100644 --- a/Duplicati/Library/Utility/Options/TimeoutOptionsHelper.cs +++ b/Duplicati/Library/Utility/Options/TimeoutOptionsHelper.cs @@ -36,30 +36,30 @@ public static class TimeoutOptionsHelper /// /// The default timeout in seconds for List operations /// - public const string DEFAULT_LIST_OPERATION_TIMEOUT = "10m"; + public const string DefaultListOperationTimeout = "10m"; /// /// The default timeout in seconds for Delete/CreateFolder operations /// - public const string DEFAULT_SHORT_OPERATION_TIMEOUT = "30s"; + public const string DefaultShortOperationTimeout = "30s"; /// /// The timeout for no activity during transfers /// - public const string DEFAULT_READ_WRITE_TIMEOUT = "30s"; + public const string DefaultReadWriteTimeout = "30s"; /// /// The name of the short timeout option, without a prefix /// - public const string SHORT_TIMEOUT_OPTION = "short-timeout"; + public const string ShortTimeoutOption = "short-timeout"; /// /// The name of the list timeout option, without a prefix /// - public const string LIST_TIMEOUT_OPTION = "list-timeout"; + public const string ListTimeoutOption = "list-timeout"; /// /// The name of the read-write timeout option, without a prefix /// - public const string READ_WRITE_TIMEOUT_OPTION = "read-write-timeout"; + public const string ReadWriteTimeoutOption = "read-write-timeout"; /// /// Gets the timeout options @@ -68,9 +68,9 @@ public static class TimeoutOptionsHelper /// The timeout options public static CommandLineArgument[] GetOptions(string? prefix = null) => [ - new CommandLineArgument($"{prefix}{SHORT_TIMEOUT_OPTION}", CommandLineArgument.ArgumentType.Timespan, Strings.TimeoutSettingsHelper.DescriptionShortTimeoutShort, Strings.TimeoutSettingsHelper.DescriptionShortTimeoutLong, DEFAULT_SHORT_OPERATION_TIMEOUT), - new CommandLineArgument($"{prefix}{LIST_TIMEOUT_OPTION}", CommandLineArgument.ArgumentType.Timespan, Strings.TimeoutSettingsHelper.DescriptionListTimeoutShort, Strings.TimeoutSettingsHelper.DescriptionListTimeoutLong, DEFAULT_LIST_OPERATION_TIMEOUT), - new CommandLineArgument($"{prefix}{READ_WRITE_TIMEOUT_OPTION}", CommandLineArgument.ArgumentType.Timespan, Strings.TimeoutSettingsHelper.DescriptionReadWriteTimeoutShort, Strings.TimeoutSettingsHelper.DescriptionReadWriteTimeoutLong, DEFAULT_READ_WRITE_TIMEOUT) + new CommandLineArgument($"{prefix}{ShortTimeoutOption}", CommandLineArgument.ArgumentType.Timespan, Strings.TimeoutSettingsHelper.DescriptionShortTimeoutShort, Strings.TimeoutSettingsHelper.DescriptionShortTimeoutLong, DefaultShortOperationTimeout), + new CommandLineArgument($"{prefix}{ListTimeoutOption}", CommandLineArgument.ArgumentType.Timespan, Strings.TimeoutSettingsHelper.DescriptionListTimeoutShort, Strings.TimeoutSettingsHelper.DescriptionListTimeoutLong, DefaultListOperationTimeout), + new CommandLineArgument($"{prefix}{ReadWriteTimeoutOption}", CommandLineArgument.ArgumentType.Timespan, Strings.TimeoutSettingsHelper.DescriptionReadWriteTimeoutShort, Strings.TimeoutSettingsHelper.DescriptionReadWriteTimeoutLong, DefaultReadWriteTimeout) ]; /// @@ -81,9 +81,9 @@ public static class TimeoutOptionsHelper /// The timeout configuration public static Timeouts Parse(IReadOnlyDictionary options, string? prefix = null) => new Timeouts( - Utility.ParseTimespanOption(options, $"{prefix}{SHORT_TIMEOUT_OPTION}", DEFAULT_SHORT_OPERATION_TIMEOUT), - Utility.ParseTimespanOption(options, $"{prefix}{LIST_TIMEOUT_OPTION}", DEFAULT_LIST_OPERATION_TIMEOUT), - Utility.ParseTimespanOption(options, $"{prefix}{READ_WRITE_TIMEOUT_OPTION}", DEFAULT_READ_WRITE_TIMEOUT) + Utility.ParseTimespanOption(options, $"{prefix}{ShortTimeoutOption}", DefaultShortOperationTimeout), + Utility.ParseTimespanOption(options, $"{prefix}{ListTimeoutOption}", DefaultListOperationTimeout), + Utility.ParseTimespanOption(options, $"{prefix}{ReadWriteTimeoutOption}", DefaultReadWriteTimeout) ); /// From bf1ef0e794b607e859881f5924ea8ea87c5994c5 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 24 Mar 2025 10:31:52 +0100 Subject: [PATCH 38/48] Fixed nullability warnings --- .../Library/Backend/AliyunOSS/Extensions.cs | 3 ++- Duplicati/Library/Backend/Backblaze/B2.cs | 9 +++++++-- .../Backend/Backblaze/Model/AccountIDEntity.cs | 2 +- .../Backend/Backblaze/Model/AuthResponse.cs | 8 ++++---- .../Backend/Backblaze/Model/BucketIDEntity.cs | 2 +- .../Backend/Backblaze/Model/DeleteRequest.cs | 4 ++-- .../Backend/Backblaze/Model/ErrorResponse.cs | 4 ++-- .../Backend/Backblaze/Model/FileEntity.cs | 6 +++--- .../Backblaze/Model/ListBucketsResponse.cs | 2 +- .../Backend/Backblaze/Model/ListFilesResponse.cs | 6 +++--- .../Backblaze/Model/UploadFileResponse.cs | 10 +++++----- .../Backend/Backblaze/Model/UploadUrlResponse.cs | 4 ++-- .../Library/Backend/CloudFiles/CloudFiles.cs | 3 +++ .../Backend/Jottacloud/JottacloudAuthHelper.cs | 16 ++++++++-------- .../Library/Backend/OneDrive/SharePointV2.cs | 4 ++-- .../Backend/pCloud/Model/pCloudFileEntry.cs | 2 +- .../Library/Main/Database/ExtensionMethods.cs | 2 +- 17 files changed, 48 insertions(+), 39 deletions(-) diff --git a/Duplicati/Library/Backend/AliyunOSS/Extensions.cs b/Duplicati/Library/Backend/AliyunOSS/Extensions.cs index 7e04cc71e..df36d9909 100644 --- a/Duplicati/Library/Backend/AliyunOSS/Extensions.cs +++ b/Duplicati/Library/Backend/AliyunOSS/Extensions.cs @@ -28,7 +28,8 @@ namespace Duplicati.Library.Backend.AliyunOSS /// /// The path to trim. /// The trimmed path. - public static string TrimPath(this string path) + [return: System.Diagnostics.CodeAnalysis.NotNullIfNotNull("path")] + public static string? TrimPath(this string? path) { return path?.Trim().Trim('/').Trim('\\').Trim('/').Trim(); } diff --git a/Duplicati/Library/Backend/Backblaze/B2.cs b/Duplicati/Library/Backend/Backblaze/B2.cs index 6d219a052..9c92e4f51 100644 --- a/Duplicati/Library/Backend/Backblaze/B2.cs +++ b/Duplicati/Library/Backend/Backblaze/B2.cs @@ -280,12 +280,14 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend private async Task GetFileId(string filename, CancellationToken cancelToken) { if (_filecache != null && _filecache.TryGetValue(filename, out var value)) - return value.OrderByDescending(x => x.UploadTimestamp).First().FileID; + return value.OrderByDescending(x => x.UploadTimestamp).First().FileID + ?? throw new InvalidDataException("Missing file ID"); await RebuildFileCache(cancelToken).ConfigureAwait(false); if (_filecache != null && _filecache.TryGetValue(filename, out var value1)) - return value1.OrderByDescending(x => x.UploadTimestamp).First().FileID; + return value1.OrderByDescending(x => x.UploadTimestamp).First().FileID + ?? throw new InvalidDataException("Missing file ID"); throw new FileMissingException(); } @@ -493,6 +495,9 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend foreach (var file in resp.Files) { + if (string.IsNullOrWhiteSpace(file.FileName)) + continue; + if (!file.FileName.StartsWith(_prefix, StringComparison.Ordinal)) continue; diff --git a/Duplicati/Library/Backend/Backblaze/Model/AccountIDEntity.cs b/Duplicati/Library/Backend/Backblaze/Model/AccountIDEntity.cs index f8925388b..822cbd1a0 100644 --- a/Duplicati/Library/Backend/Backblaze/Model/AccountIDEntity.cs +++ b/Duplicati/Library/Backend/Backblaze/Model/AccountIDEntity.cs @@ -29,5 +29,5 @@ namespace Duplicati.Library.Backend.Backblaze.Model; internal abstract class AccountIDEntity { [JsonProperty("accountId")] - public string AccountID { get; set; } + public string? AccountID { get; set; } } \ No newline at end of file diff --git a/Duplicati/Library/Backend/Backblaze/Model/AuthResponse.cs b/Duplicati/Library/Backend/Backblaze/Model/AuthResponse.cs index 416cf92b2..d5d16761d 100644 --- a/Duplicati/Library/Backend/Backblaze/Model/AuthResponse.cs +++ b/Duplicati/Library/Backend/Backblaze/Model/AuthResponse.cs @@ -29,11 +29,11 @@ namespace Duplicati.Library.Backend.Backblaze.Model; internal class AuthResponse { [JsonProperty("accountId")] - public string AccountID { get; set; } + public string? AccountID { get; set; } [JsonProperty("apiUrl")] - public string APIUrl { get; set; } + public string? APIUrl { get; set; } [JsonProperty("authorizationToken")] - public string AuthorizationToken { get; set; } + public string? AuthorizationToken { get; set; } [JsonProperty("downloadUrl")] - public string DownloadUrl { get; set; } + public string? DownloadUrl { get; set; } } \ No newline at end of file diff --git a/Duplicati/Library/Backend/Backblaze/Model/BucketIDEntity.cs b/Duplicati/Library/Backend/Backblaze/Model/BucketIDEntity.cs index 723183bcc..772b3be1c 100644 --- a/Duplicati/Library/Backend/Backblaze/Model/BucketIDEntity.cs +++ b/Duplicati/Library/Backend/Backblaze/Model/BucketIDEntity.cs @@ -32,5 +32,5 @@ internal abstract class BucketIDEntity /// Gets or sets the unique identifier of the bucket. /// [JsonProperty("bucketId")] - public string BucketID { get; set; } + public string? BucketID { get; set; } } \ No newline at end of file diff --git a/Duplicati/Library/Backend/Backblaze/Model/DeleteRequest.cs b/Duplicati/Library/Backend/Backblaze/Model/DeleteRequest.cs index c04f95a8a..02255bbb2 100644 --- a/Duplicati/Library/Backend/Backblaze/Model/DeleteRequest.cs +++ b/Duplicati/Library/Backend/Backblaze/Model/DeleteRequest.cs @@ -34,12 +34,12 @@ internal class DeleteRequest /// This is the full path of the file in the bucket. /// [JsonProperty("fileName")] - public string FileName { get; set; } + public string? FileName { get; set; } /// /// Gets or sets the unique identifier of the file to be deleted. /// This ID is assigned by Backblaze B2 when the file is uploaded. /// [JsonProperty("fileId")] - public string FileId { get; set; } + public string? FileId { get; set; } } \ No newline at end of file diff --git a/Duplicati/Library/Backend/Backblaze/Model/ErrorResponse.cs b/Duplicati/Library/Backend/Backblaze/Model/ErrorResponse.cs index 9aef5c29e..a534f1263 100644 --- a/Duplicati/Library/Backend/Backblaze/Model/ErrorResponse.cs +++ b/Duplicati/Library/Backend/Backblaze/Model/ErrorResponse.cs @@ -29,9 +29,9 @@ namespace Duplicati.Library.Backend.Backblaze.Model; internal class ErrorResponse { [JsonProperty("code")] - public string Code { get; set; } + public string? Code { get; set; } [JsonProperty("message")] - public string Message { get; set; } + public string? Message { get; set; } [JsonProperty("status")] public long Status { get; set; } } \ No newline at end of file diff --git a/Duplicati/Library/Backend/Backblaze/Model/FileEntity.cs b/Duplicati/Library/Backend/Backblaze/Model/FileEntity.cs index eb79338c0..8df78cdb1 100644 --- a/Duplicati/Library/Backend/Backblaze/Model/FileEntity.cs +++ b/Duplicati/Library/Backend/Backblaze/Model/FileEntity.cs @@ -34,14 +34,14 @@ internal class FileEntity /// This ID is used to reference the file in various API operations. /// [JsonProperty("fileId")] - public string FileID { get; set; } + public string? FileID { get; set; } /// /// Gets or sets the name of the file as stored in Backblaze B2. /// This is the full path of the file within the bucket. /// [JsonProperty("fileName")] - public string FileName { get; set; } + public string? FileName { get; set; } /// /// Gets or sets the action performed on the file. @@ -49,7 +49,7 @@ internal class FileEntity /// and "delete" for deleted files. /// [JsonProperty("action")] - public string Action { get; set; } + public string? Action { get; set; } /// /// Gets or sets the size of the file in bytes. diff --git a/Duplicati/Library/Backend/Backblaze/Model/ListBucketsResponse.cs b/Duplicati/Library/Backend/Backblaze/Model/ListBucketsResponse.cs index 2deeb8f9a..58e22d6b6 100644 --- a/Duplicati/Library/Backend/Backblaze/Model/ListBucketsResponse.cs +++ b/Duplicati/Library/Backend/Backblaze/Model/ListBucketsResponse.cs @@ -34,5 +34,5 @@ internal class ListBucketsResponse /// Each element contains information about a single bucket in the account. /// [JsonProperty("buckets")] - public BucketEntity[] Buckets { get; set; } + public BucketEntity[]? Buckets { get; set; } } \ No newline at end of file diff --git a/Duplicati/Library/Backend/Backblaze/Model/ListFilesResponse.cs b/Duplicati/Library/Backend/Backblaze/Model/ListFilesResponse.cs index 575c5f48a..97d2869e1 100644 --- a/Duplicati/Library/Backend/Backblaze/Model/ListFilesResponse.cs +++ b/Duplicati/Library/Backend/Backblaze/Model/ListFilesResponse.cs @@ -34,19 +34,19 @@ internal class ListFilesResponse /// This value should be used in the next request to continue listing files. /// [JsonProperty("nextFileName")] - public string NextFileName { get; set; } + public string? NextFileName { get; set; } /// /// Gets or sets the ID of the next file to use for pagination. /// This value should be used in conjunction with NextFileName for subsequent requests. /// [JsonProperty("nextFileId")] - public string NextFileID { get; set; } + public string? NextFileID { get; set; } /// /// Gets or sets the array of file entities returned by this request. /// Each element contains information about a single file in the bucket. /// [JsonProperty("files")] - public FileEntity[] Files { get; set; } + public FileEntity[]? Files { get; set; } } \ No newline at end of file diff --git a/Duplicati/Library/Backend/Backblaze/Model/UploadFileResponse.cs b/Duplicati/Library/Backend/Backblaze/Model/UploadFileResponse.cs index a0aa3d7a3..9e82faaad 100644 --- a/Duplicati/Library/Backend/Backblaze/Model/UploadFileResponse.cs +++ b/Duplicati/Library/Backend/Backblaze/Model/UploadFileResponse.cs @@ -33,19 +33,19 @@ internal class UploadFileResponse : AccountIDEntity /// Gets or sets the unique identifier of the bucket containing the file. /// [JsonProperty("bucketId")] - public string BucketID { get; set; } + public string? BucketID { get; set; } /// /// Gets or sets the unique identifier assigned to the uploaded file. /// [JsonProperty("fileId")] - public string FileID { get; set; } + public string? FileID { get; set; } /// /// Gets or sets the name of the uploaded file. /// [JsonProperty("fileName")] - public string FileName { get; set; } + public string? FileName { get; set; } /// /// Gets or sets the size of the uploaded file in bytes. @@ -57,11 +57,11 @@ internal class UploadFileResponse : AccountIDEntity /// Gets or sets the SHA1 hash of the file content. /// [JsonProperty("contentSha1")] - public string ContentSha1 { get; set; } + public string? ContentSha1 { get; set; } /// /// Gets or sets the MIME type of the uploaded file. /// [JsonProperty("contentType")] - public string ContentType { get; set; } + public string? ContentType { get; set; } } \ No newline at end of file diff --git a/Duplicati/Library/Backend/Backblaze/Model/UploadUrlResponse.cs b/Duplicati/Library/Backend/Backblaze/Model/UploadUrlResponse.cs index 0a72dfbe7..42bb641f1 100644 --- a/Duplicati/Library/Backend/Backblaze/Model/UploadUrlResponse.cs +++ b/Duplicati/Library/Backend/Backblaze/Model/UploadUrlResponse.cs @@ -34,12 +34,12 @@ internal class UploadUrlResponse : BucketIDEntity /// This URL is specific to the bucket and should be used for subsequent upload requests. /// [JsonProperty("uploadUrl")] - public string UploadUrl { get; set; } + public string? UploadUrl { get; set; } /// /// Gets or sets the authorization token required for upload requests. /// This token must be included in the headers when uploading files using the UploadUrl. /// [JsonProperty("authorizationToken")] - public string AuthorizationToken { get; set; } + public string? AuthorizationToken { get; set; } } \ No newline at end of file diff --git a/Duplicati/Library/Backend/CloudFiles/CloudFiles.cs b/Duplicati/Library/Backend/CloudFiles/CloudFiles.cs index 80541a569..956cbd1f3 100644 --- a/Duplicati/Library/Backend/CloudFiles/CloudFiles.cs +++ b/Duplicati/Library/Backend/CloudFiles/CloudFiles.cs @@ -177,6 +177,9 @@ namespace Duplicati.Library.Backend catch { } //Ignore } + if (lst == null) + yield break; + var lastItemName = ""; foreach (System.Xml.XmlNode n in lst) { diff --git a/Duplicati/Library/Backend/Jottacloud/JottacloudAuthHelper.cs b/Duplicati/Library/Backend/Jottacloud/JottacloudAuthHelper.cs index 0e7b7bd90..1f24f6450 100644 --- a/Duplicati/Library/Backend/Jottacloud/JottacloudAuthHelper.cs +++ b/Duplicati/Library/Backend/Jottacloud/JottacloudAuthHelper.cs @@ -52,23 +52,23 @@ namespace Duplicati.Library.Backend private class UserInfo { [JsonProperty("sub")] - public string Subject { get; set; } + public string? Subject { get; set; } [JsonProperty("email_verified")] public bool EmailVerified { get; set; } [JsonProperty("name")] - public string Name { get; set; } + public string? Name { get; set; } [JsonProperty("realm")] - public string Realm { get; set; } + public string? Realm { get; set; } [JsonProperty("preferred_username")] - public string PreferredUsername { get; set; } // The numeric internal username, same as Username + public string? PreferredUsername { get; set; } // The numeric internal username, same as Username [JsonProperty("given_name")] - public string GivenName { get; set; } + public string? GivenName { get; set; } [JsonProperty("family_name")] - public string FamilyName { get; set; } + public string? FamilyName { get; set; } [JsonProperty("email")] - public string Email { get; set; } + public string? Email { get; set; } [JsonProperty("username")] - public string Username { get; set; } // The numeric internal username, same as PreferredUsername + public string? Username { get; set; } // The numeric internal username, same as PreferredUsername } } diff --git a/Duplicati/Library/Backend/OneDrive/SharePointV2.cs b/Duplicati/Library/Backend/OneDrive/SharePointV2.cs index e130cac88..859ce18ca 100644 --- a/Duplicati/Library/Backend/OneDrive/SharePointV2.cs +++ b/Duplicati/Library/Backend/OneDrive/SharePointV2.cs @@ -86,7 +86,7 @@ namespace Duplicati.Library.Backend uri = new Uri(string.Format("{0}:{1}", uri.Scheme, uri.LocalPath)); } - var site = await this.GetSharePointSite(uri, cancelToken).ConfigureAwait(false); + var site = await GetSharePointSite(uri, cancelToken).ConfigureAwait(false); if (site != null) { // Get the web URL of the site's main drive @@ -112,7 +112,7 @@ namespace Duplicati.Library.Backend return await base.GetRootPathFromUrlAsync(url, cancelToken).ConfigureAwait(false); } - private async Task GetSharePointSite(Uri url, CancellationToken cancelToken) + private async Task GetSharePointSite(Uri url, CancellationToken cancelToken) { var uri = new UriBuilder(url); diff --git a/Duplicati/Library/Backend/pCloud/Model/pCloudFileEntry.cs b/Duplicati/Library/Backend/pCloud/Model/pCloudFileEntry.cs index b280ed94e..9f2ed6264 100644 --- a/Duplicati/Library/Backend/pCloud/Model/pCloudFileEntry.cs +++ b/Duplicati/Library/Backend/pCloud/Model/pCloudFileEntry.cs @@ -51,7 +51,7 @@ internal class pCloudFileEntry : IFileEntry /// /// Filename or folder name /// - public string Name { get; set; } + public string? Name { get; set; } /// /// Size of the object in bytes diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index 098518811..3e7a071e4 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -440,7 +440,7 @@ public static class ExtensionMethods /// The command string to execute /// The values to use as parameters. The parameters must already be added. /// A instance - public static IDataReader ExecuteReader(this IDbCommand self, string cmd, Dictionary values) + public static IDataReader ExecuteReader(this IDbCommand self, string cmd, Dictionary values) { return ExecuteReader(self, true, cmd, values); } From 646b1f9b972740590fb280b12e9e63e4c6c87a92 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 24 Mar 2025 14:35:16 +0100 Subject: [PATCH 39/48] Add extra parameters to reports. This adds the option to set additional parameters to any report sent. This fixes #3448 --- .../Library/Modules/Builtin/ReportHelper.cs | 38 ++++++++++- .../Modules/Builtin/SendHttpMessage.cs | 51 +++++++------- .../Modules/Builtin/SendJabberMessage.cs | 39 ++++++----- Duplicati/Library/Modules/Builtin/SendMail.cs | 44 +++++++------ .../Modules/Builtin/SendTelegramMessage.cs | 66 ++++++++++--------- Duplicati/Library/Modules/Builtin/Strings.cs | 8 +++ 6 files changed, 149 insertions(+), 97 deletions(-) diff --git a/Duplicati/Library/Modules/Builtin/ReportHelper.cs b/Duplicati/Library/Modules/Builtin/ReportHelper.cs index e37ee17ac..5d8b35fac 100644 --- a/Duplicati/Library/Modules/Builtin/ReportHelper.cs +++ b/Duplicati/Library/Modules/Builtin/ReportHelper.cs @@ -83,6 +83,10 @@ namespace Duplicati.Library.Modules.Builtin /// Name of the option used to the output format /// protected abstract string ResultFormatOptionName { get; } + /// + /// Name of the option used to specify extra data to include in the report + /// + protected abstract string ExtraDataOptionName { get; } /// /// The default subject or title line @@ -184,13 +188,17 @@ namespace Duplicati.Library.Modules.Builtin /// private bool m_sendAll; /// + /// The extra values to include in the report + /// + private Dictionary m_extraValues; + /// /// The log scope that should be disposed /// private IDisposable m_logscope; /// /// The log storage /// - private Utility.FileBackedStringList m_logstorage; + private FileBackedStringList m_logstorage; /// /// Serializer to use when serializing the message. /// @@ -224,6 +232,21 @@ namespace Duplicati.Library.Modules.Builtin m_isConfigured = true; m_options.TryGetValue(SubjectOptionName, out m_subject); m_options.TryGetValue(BodyOptionName, out m_body); + m_options.TryGetValue(ExtraDataOptionName, out var extraData); + if (!string.IsNullOrWhiteSpace(extraData)) + { + if (extraData.Trim().StartsWith("{") && extraData.Trim().EndsWith("}")) + { + m_extraValues = System.Text.Json.JsonSerializer.Deserialize>(extraData); + } + else + { + var values = Utility.Uri.ParseQueryString(extraData); + m_extraValues = values.AllKeys + .Where(x => !string.IsNullOrWhiteSpace(x)) + .ToDictionary(key => key, key => values[key]); + } + } string tmp; m_options.TryGetValue(ActionLevelOptionName, out tmp); @@ -263,7 +286,7 @@ namespace Duplicati.Library.Modules.Builtin m_body = DEFAULT_BODY; m_options.TryGetValue(LogFilterOptionName, out var logfilterstring); - var filter = Utility.FilterExpression.ParseLogFilter(logfilterstring); + var filter = FilterExpression.ParseLogFilter(logfilterstring); var logLevel = Utility.Utility.ParseEnumOption(m_options, LogLevelOptionName, DEFAULT_LOG_LEVEL); m_logstorage = new FileBackedStringList(); @@ -439,7 +462,7 @@ namespace Duplicati.Library.Modules.Builtin extra[key] = GetDefaultValue(key); // Add any options that are whitelisted or used in the template - foreach (KeyValuePair kv in m_options) + foreach (var kv in m_options) if (EXTRA_TEMPLATE_KEYS.Contains(kv.Key) || input.IndexOf($"%{kv.Key}%", StringComparison.OrdinalIgnoreCase) >= 0) extra[kv.Key] = kv.Value; @@ -448,6 +471,10 @@ namespace Duplicati.Library.Modules.Builtin if (!extra.ContainsKey(key)) extra[key] = GetDefaultValue(key); + if (m_extraValues != null) + foreach (var v in m_extraValues) + extra[v.Key] = v.Value; + return resultFormatSerializer.Serialize(result, exception, LogLines, extra); } else @@ -473,6 +500,11 @@ namespace Duplicati.Library.Modules.Builtin if (!m_options.ContainsKey(key)) input = Regex.Replace(input, $"%{key}%", GetDefaultValue(key) ?? "", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + if (m_extraValues != null) + foreach (var v in m_extraValues) + input = Regex.Replace(input, $"%{v.Key}%", v.Value, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + + // Remove any remaining template keys input = Regex.Replace(input, "\\%[^\\%]+\\%", ""); return input; diff --git a/Duplicati/Library/Modules/Builtin/SendHttpMessage.cs b/Duplicati/Library/Modules/Builtin/SendHttpMessage.cs index 6bd57fdec..9e83ed4b2 100644 --- a/Duplicati/Library/Modules/Builtin/SendHttpMessage.cs +++ b/Duplicati/Library/Modules/Builtin/SendHttpMessage.cs @@ -81,6 +81,10 @@ namespace Duplicati.Library.Modules.Builtin /// private const string OPTION_EXTRA_PARAMETERS = "send-http-extra-parameters"; /// + /// Option used to specify extra parameters + /// + private const string OPTION_EXTRA_DATA_PARAMETERS = "send-http-extra-data-parameters"; + /// /// Option used to specify report level /// private const string OPTION_SENDLEVEL = "send-http-level"; @@ -211,36 +215,32 @@ namespace Duplicati.Library.Modules.Builtin /// /// Gets a list of supported commandline arguments /// - public override IList SupportedCommands - { - get - { - return new List(new ICommandLineArgument[] { - new CommandLineArgument(OPTION_URL, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.SendhttpurlShort, Strings.SendHttpMessage.SendhttpurlLong), - new CommandLineArgument(OPTION_MESSAGE, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.SendhttpmessageShort, Strings.SendHttpMessage.SendhttpmessageLong, DEFAULT_BODY), - new CommandLineArgument(OPTION_MESSAGE_PARAMETER_NAME, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.SendhttpmessageparameternameShort, Strings.SendHttpMessage.SendhttpmessageparameternameLong, DEFAULT_MESSAGE_PARAMETER_NAME), - new CommandLineArgument(OPTION_EXTRA_PARAMETERS, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.SendhttpextraparametersShort, Strings.SendHttpMessage.SendhttpextraparametersLong), - new CommandLineArgument(OPTION_SENDLEVEL, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.SendhttplevelShort, Strings.SendHttpMessage.SendhttplevelLong(ParsedResultType.Success.ToString(), ParsedResultType.Warning.ToString(), ParsedResultType.Error.ToString(), ParsedResultType.Fatal.ToString(), "All"), DEFAULT_LEVEL, null, Enum.GetNames(typeof(ParsedResultType)).Union(new string[] { "All" } ).ToArray()), - new CommandLineArgument(OPTION_SENDALL, CommandLineArgument.ArgumentType.Boolean, Strings.SendHttpMessage.SendhttpanyoperationShort, Strings.SendHttpMessage.SendhttpanyoperationLong), + public override IList SupportedCommands => + [ + new CommandLineArgument(OPTION_URL, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.SendhttpurlShort, Strings.SendHttpMessage.SendhttpurlLong), + new CommandLineArgument(OPTION_MESSAGE, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.SendhttpmessageShort, Strings.SendHttpMessage.SendhttpmessageLong, DEFAULT_BODY), + new CommandLineArgument(OPTION_MESSAGE_PARAMETER_NAME, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.SendhttpmessageparameternameShort, Strings.SendHttpMessage.SendhttpmessageparameternameLong, DEFAULT_MESSAGE_PARAMETER_NAME), + new CommandLineArgument(OPTION_EXTRA_PARAMETERS, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.SendhttpextraparametersShort, Strings.SendHttpMessage.SendhttpextraparametersLong), + new CommandLineArgument(OPTION_EXTRA_DATA_PARAMETERS, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.SendhttpextradataparametersShort, Strings.SendHttpMessage.SendhttpextradataparametersLong), + new CommandLineArgument(OPTION_SENDLEVEL, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.SendhttplevelShort, Strings.SendHttpMessage.SendhttplevelLong(ParsedResultType.Success.ToString(), ParsedResultType.Warning.ToString(), ParsedResultType.Error.ToString(), ParsedResultType.Fatal.ToString(), "All"), DEFAULT_LEVEL, null, Enum.GetNames(typeof(ParsedResultType)).Union(new string[] { "All" } ).ToArray()), + new CommandLineArgument(OPTION_SENDALL, CommandLineArgument.ArgumentType.Boolean, Strings.SendHttpMessage.SendhttpanyoperationShort, Strings.SendHttpMessage.SendhttpanyoperationLong), - new CommandLineArgument(OPTION_VERB, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.HttpverbShort, Strings.SendHttpMessage.HttpverbLong, "POST"), - new CommandLineArgument(OPTION_LOG_LEVEL, CommandLineArgument.ArgumentType.Enumeration, Strings.ReportHelper.OptionLoglevelShort, Strings.ReportHelper.OptionLoglevelLong, DEFAULT_LOG_LEVEL.ToString(), null, Enum.GetNames(typeof(Logging.LogMessageType))), - new CommandLineArgument(OPTION_LOG_FILTER, CommandLineArgument.ArgumentType.String, Strings.ReportHelper.OptionLogfilterShort, Strings.ReportHelper.OptionLogfilterLong), - new CommandLineArgument(OPTION_MAX_LOG_LINES, CommandLineArgument.ArgumentType.Integer, Strings.ReportHelper.OptionmaxloglinesShort, Strings.ReportHelper.OptionmaxloglinesLong, DEFAULT_LOGLINES.ToString()), + new CommandLineArgument(OPTION_VERB, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.HttpverbShort, Strings.SendHttpMessage.HttpverbLong, "POST"), + new CommandLineArgument(OPTION_LOG_LEVEL, CommandLineArgument.ArgumentType.Enumeration, Strings.ReportHelper.OptionLoglevelShort, Strings.ReportHelper.OptionLoglevelLong, DEFAULT_LOG_LEVEL.ToString(), null, Enum.GetNames(typeof(Logging.LogMessageType))), + new CommandLineArgument(OPTION_LOG_FILTER, CommandLineArgument.ArgumentType.String, Strings.ReportHelper.OptionLogfilterShort, Strings.ReportHelper.OptionLogfilterLong), + new CommandLineArgument(OPTION_MAX_LOG_LINES, CommandLineArgument.ArgumentType.Integer, Strings.ReportHelper.OptionmaxloglinesShort, Strings.ReportHelper.OptionmaxloglinesLong, DEFAULT_LOGLINES.ToString()), - new CommandLineArgument(OPTION_RESULT_FORMAT, CommandLineArgument.ArgumentType.Enumeration, Strings.ReportHelper.ResultFormatShort, Strings.ReportHelper.ResultFormatLong(Enum.GetNames(typeof(ResultExportFormat))), DEFAULT_EXPORT_FORMAT.ToString(), null, Enum.GetNames(typeof(ResultExportFormat))), + new CommandLineArgument(OPTION_RESULT_FORMAT, CommandLineArgument.ArgumentType.Enumeration, Strings.ReportHelper.ResultFormatShort, Strings.ReportHelper.ResultFormatLong(Enum.GetNames(typeof(ResultExportFormat))), DEFAULT_EXPORT_FORMAT.ToString(), null, Enum.GetNames(typeof(ResultExportFormat))), - new CommandLineArgument(OPTION_URL_FORM, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.SendhttpurlsformShort, Strings.SendHttpMessage.SendhttpurlsformLong), - new CommandLineArgument(OPTION_URL_JSON, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.SendhttpurlsjsonShort, Strings.SendHttpMessage.SendhttpurlsjsonLong), + new CommandLineArgument(OPTION_URL_FORM, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.SendhttpurlsformShort, Strings.SendHttpMessage.SendhttpurlsformLong), + new CommandLineArgument(OPTION_URL_JSON, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.SendhttpurlsjsonShort, Strings.SendHttpMessage.SendhttpurlsjsonLong), - new CommandLineArgument(OPTION_ACCEPT_ANY_CERTIFICATE, CommandLineArgument.ArgumentType.Boolean, Strings.SendHttpMessage.AcceptAnyCertificateShort, Strings.SendHttpMessage.AcceptAnyCertificateLong), - new CommandLineArgument(OPTION_ACCEPT_SPECIFIED_CERTIFICATE, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.AcceptSpecifiedCertificateShort, Strings.SendHttpMessage.AcceptSpecifiedCertificateLong), + new CommandLineArgument(OPTION_ACCEPT_ANY_CERTIFICATE, CommandLineArgument.ArgumentType.Boolean, Strings.SendHttpMessage.AcceptAnyCertificateShort, Strings.SendHttpMessage.AcceptAnyCertificateLong), + new CommandLineArgument(OPTION_ACCEPT_SPECIFIED_CERTIFICATE, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.AcceptSpecifiedCertificateShort, Strings.SendHttpMessage.AcceptSpecifiedCertificateLong), - new CommandLineArgument(OPTION_SEND_HTTP_RETRIES, CommandLineArgument.ArgumentType.Integer, Strings.SendHttpMessage.SendHttpRetriesShort, Strings.SendHttpMessage.SendHttpRetriesLong, DEFAULT_RETRIES.ToString()), - new CommandLineArgument(OPTION_SEND_HTTP_RETRY_DELAY, CommandLineArgument.ArgumentType.Integer, Strings.SendHttpMessage.SendHttpRetryDelayShort, Strings.SendHttpMessage.SendHttpRetryDelayLong, DEFAULT_RETRY_DELAY), - }); - } - } + new CommandLineArgument(OPTION_SEND_HTTP_RETRIES, CommandLineArgument.ArgumentType.Integer, Strings.SendHttpMessage.SendHttpRetriesShort, Strings.SendHttpMessage.SendHttpRetriesLong, DEFAULT_RETRIES.ToString()), + new CommandLineArgument(OPTION_SEND_HTTP_RETRY_DELAY, CommandLineArgument.ArgumentType.Integer, Strings.SendHttpMessage.SendHttpRetryDelayShort, Strings.SendHttpMessage.SendHttpRetryDelayLong, DEFAULT_RETRY_DELAY), + ]; protected override string SubjectOptionName => OPTION_MESSAGE; protected override string BodyOptionName => OPTION_MESSAGE; @@ -250,6 +250,7 @@ namespace Duplicati.Library.Modules.Builtin protected override string LogFilterOptionName => OPTION_LOG_FILTER; protected override string LogLinesOptionName => OPTION_MAX_LOG_LINES; protected override string ResultFormatOptionName => OPTION_RESULT_FORMAT; + protected override string ExtraDataOptionName => OPTION_EXTRA_DATA_PARAMETERS; /// /// This method is the interception where the module can interact with the execution environment and modify the settings. diff --git a/Duplicati/Library/Modules/Builtin/SendJabberMessage.cs b/Duplicati/Library/Modules/Builtin/SendJabberMessage.cs index 81c54e07b..049e09a39 100644 --- a/Duplicati/Library/Modules/Builtin/SendJabberMessage.cs +++ b/Duplicati/Library/Modules/Builtin/SendJabberMessage.cs @@ -50,6 +50,11 @@ namespace Duplicati.Library.Modules.Builtin /// Option used to specify report body /// private const string OPTION_MESSAGE = "send-xmpp-message"; + /// + /// Option used to specify extra parameters + /// + private const string OPTION_EXTRA_DATA_PARAMETERS = "send-xmpp-extra-data-parameters"; + /// /// Option used to specify report level /// @@ -133,26 +138,23 @@ namespace Duplicati.Library.Modules.Builtin /// /// Gets a list of supported commandline arguments /// - public override IList SupportedCommands - { - get - { - return new List(new ICommandLineArgument[] { - new CommandLineArgument(OPTION_RECIPIENT, CommandLineArgument.ArgumentType.String, Strings.SendJabberMessage.SendxmpptoShort, Strings.SendJabberMessage.SendxmpptoLong), - new CommandLineArgument(OPTION_MESSAGE, CommandLineArgument.ArgumentType.String, Strings.SendJabberMessage.SendxmppmessageShort, Strings.SendJabberMessage.SendxmppmessageLong, DEFAULT_BODY), - new CommandLineArgument(OPTION_USERNAME, CommandLineArgument.ArgumentType.String, Strings.SendJabberMessage.SendxmppusernameShort, Strings.SendJabberMessage.SendxmppusernameLong), - new CommandLineArgument(OPTION_PASSWORD, CommandLineArgument.ArgumentType.String, Strings.SendJabberMessage.SendxmpppasswordShort, Strings.SendJabberMessage.SendxmpppasswordLong), - new CommandLineArgument(OPTION_SENDLEVEL, CommandLineArgument.ArgumentType.String, Strings.SendJabberMessage.SendxmpplevelShort, Strings.SendJabberMessage.SendxmpplevelLong(ParsedResultType.Success.ToString(), ParsedResultType.Warning.ToString(), ParsedResultType.Error.ToString(), ParsedResultType.Fatal.ToString(), "All"), DEFAULT_LEVEL, null, Enum.GetNames(typeof(ParsedResultType)).Union(new string[] { "All" } ).ToArray()), - new CommandLineArgument(OPTION_SENDALL, CommandLineArgument.ArgumentType.Boolean, Strings.SendJabberMessage.SendxmppanyoperationShort, Strings.SendJabberMessage.SendxmppanyoperationLong), + public override IList SupportedCommands => + [ + new CommandLineArgument(OPTION_RECIPIENT, CommandLineArgument.ArgumentType.String, Strings.SendJabberMessage.SendxmpptoShort, Strings.SendJabberMessage.SendxmpptoLong), + new CommandLineArgument(OPTION_MESSAGE, CommandLineArgument.ArgumentType.String, Strings.SendJabberMessage.SendxmppmessageShort, Strings.SendJabberMessage.SendxmppmessageLong, DEFAULT_BODY), + new CommandLineArgument(OPTION_EXTRA_DATA_PARAMETERS, CommandLineArgument.ArgumentType.String, Strings.SendJabberMessage.SendxmppextradataparametersShort, Strings.SendJabberMessage.SendxmppextradataparametersLong), + new CommandLineArgument(OPTION_USERNAME, CommandLineArgument.ArgumentType.String, Strings.SendJabberMessage.SendxmppusernameShort, Strings.SendJabberMessage.SendxmppusernameLong), + new CommandLineArgument(OPTION_PASSWORD, CommandLineArgument.ArgumentType.String, Strings.SendJabberMessage.SendxmpppasswordShort, Strings.SendJabberMessage.SendxmpppasswordLong), + new CommandLineArgument(OPTION_SENDLEVEL, CommandLineArgument.ArgumentType.String, Strings.SendJabberMessage.SendxmpplevelShort, Strings.SendJabberMessage.SendxmpplevelLong(ParsedResultType.Success.ToString(), ParsedResultType.Warning.ToString(), ParsedResultType.Error.ToString(), ParsedResultType.Fatal.ToString(), "All"), DEFAULT_LEVEL, null, Enum.GetNames(typeof(ParsedResultType)).Union(new string[] { "All" } ).ToArray()), + new CommandLineArgument(OPTION_SENDALL, CommandLineArgument.ArgumentType.Boolean, Strings.SendJabberMessage.SendxmppanyoperationShort, Strings.SendJabberMessage.SendxmppanyoperationLong), - new CommandLineArgument(OPTION_LOG_LEVEL, CommandLineArgument.ArgumentType.Enumeration, Strings.ReportHelper.OptionLoglevelShort, Strings.ReportHelper.OptionLoglevelLong, DEFAULT_LOG_LEVEL.ToString(), null, Enum.GetNames(typeof(Logging.LogMessageType))), - new CommandLineArgument(OPTION_LOG_FILTER, CommandLineArgument.ArgumentType.String, Strings.ReportHelper.OptionLogfilterShort, Strings.ReportHelper.OptionLogfilterLong), - new CommandLineArgument(OPTION_MAX_LOG_LINES, CommandLineArgument.ArgumentType.Integer, Strings.ReportHelper.OptionmaxloglinesShort, Strings.ReportHelper.OptionmaxloglinesLong, DEFAULT_LOGLINES.ToString()), + new CommandLineArgument(OPTION_LOG_LEVEL, CommandLineArgument.ArgumentType.Enumeration, Strings.ReportHelper.OptionLoglevelShort, Strings.ReportHelper.OptionLoglevelLong, DEFAULT_LOG_LEVEL.ToString(), null, Enum.GetNames(typeof(Logging.LogMessageType))), + new CommandLineArgument(OPTION_LOG_FILTER, CommandLineArgument.ArgumentType.String, Strings.ReportHelper.OptionLogfilterShort, Strings.ReportHelper.OptionLogfilterLong), + new CommandLineArgument(OPTION_MAX_LOG_LINES, CommandLineArgument.ArgumentType.Integer, Strings.ReportHelper.OptionmaxloglinesShort, Strings.ReportHelper.OptionmaxloglinesLong, DEFAULT_LOGLINES.ToString()), + + new CommandLineArgument(OPTION_RESULT_FORMAT, CommandLineArgument.ArgumentType.Enumeration, Strings.ReportHelper.ResultFormatShort, Strings.ReportHelper.ResultFormatLong(Enum.GetNames(typeof(ResultExportFormat))), DEFAULT_EXPORT_FORMAT.ToString(), null, Enum.GetNames(typeof(ResultExportFormat))), + ]; - new CommandLineArgument(OPTION_RESULT_FORMAT, CommandLineArgument.ArgumentType.Enumeration, Strings.ReportHelper.ResultFormatShort, Strings.ReportHelper.ResultFormatLong(Enum.GetNames(typeof(ResultExportFormat))), DEFAULT_EXPORT_FORMAT.ToString(), null, Enum.GetNames(typeof(ResultExportFormat))), - }); - } - } protected override string SubjectOptionName => OPTION_MESSAGE; protected override string BodyOptionName => OPTION_MESSAGE; @@ -162,6 +164,7 @@ namespace Duplicati.Library.Modules.Builtin protected override string LogFilterOptionName => OPTION_LOG_FILTER; protected override string LogLinesOptionName => OPTION_MAX_LOG_LINES; protected override string ResultFormatOptionName => OPTION_RESULT_FORMAT; + protected override string ExtraDataOptionName => OPTION_EXTRA_DATA_PARAMETERS; /// /// This method is the interception where the module can interact with the execution environment and modify the settings. diff --git a/Duplicati/Library/Modules/Builtin/SendMail.cs b/Duplicati/Library/Modules/Builtin/SendMail.cs index 9dc2b9e03..fe23dc6f9 100644 --- a/Duplicati/Library/Modules/Builtin/SendMail.cs +++ b/Duplicati/Library/Modules/Builtin/SendMail.cs @@ -79,6 +79,11 @@ namespace Duplicati.Library.Modules.Builtin /// Option used to specify what format the result is sent in. /// private const string OPTION_RESULT_FORMAT = "send-mail-result-output-format"; + /// + /// Option used to specify extra parameters + /// + private const string OPTION_EXTRA_DATA_PARAMETERS = "send-mail-extra-data-parameters"; + /// /// Option used to set the log level for mail reports /// @@ -150,29 +155,25 @@ namespace Duplicati.Library.Modules.Builtin /// /// Gets a list of supported commandline arguments /// - public override IList SupportedCommands - { - get - { - return new List(new ICommandLineArgument[] { - new CommandLineArgument(OPTION_RECIPIENT, CommandLineArgument.ArgumentType.String, Strings.SendMail.OptionRecipientShort, Strings.SendMail.OptionRecipientLong), - new CommandLineArgument(OPTION_SENDER, CommandLineArgument.ArgumentType.String, Strings.SendMail.OptionSenderShort, Strings.SendMail.OptionSenderLong, DEFAULT_SENDER), - new CommandLineArgument(OPTION_SUBJECT, CommandLineArgument.ArgumentType.String, Strings.SendMail.OptionSubjectShort, Strings.SendMail.OptionSubjectLong(OPTION_BODY), DEFAULT_SUBJECT), - new CommandLineArgument(OPTION_BODY, CommandLineArgument.ArgumentType.String, Strings.SendMail.OptionBodyShort, Strings.SendMail.OptionBodyLong, DEFAULT_BODY), - new CommandLineArgument(OPTION_SERVER, CommandLineArgument.ArgumentType.String, Strings.SendMail.OptionServerShort, Strings.SendMail.OptionServerLong), - new CommandLineArgument(OPTION_USERNAME, CommandLineArgument.ArgumentType.String, Strings.SendMail.OptionUsernameShort, Strings.SendMail.OptionUsernameLong), - new CommandLineArgument(OPTION_PASSWORD, CommandLineArgument.ArgumentType.String, Strings.SendMail.OptionPasswordShort, Strings.SendMail.OptionPasswordLong), - new CommandLineArgument(OPTION_SENDLEVEL, CommandLineArgument.ArgumentType.String, Strings.SendMail.OptionSendlevelShort, Strings.SendMail.OptionSendlevelLong(ParsedResultType.Success.ToString(), ParsedResultType.Warning.ToString(), ParsedResultType.Error.ToString(), ParsedResultType.Fatal.ToString(), "All"), DEFAULT_LEVEL, null, Enum.GetNames(typeof(ParsedResultType)).Union(new string [] { "All" }).ToArray()), - new CommandLineArgument(OPTION_SENDALL, CommandLineArgument.ArgumentType.Boolean, Strings.SendHttpMessage.SendhttpanyoperationShort, Strings.SendHttpMessage.SendhttpanyoperationLong), + public override IList SupportedCommands => + [ + new CommandLineArgument(OPTION_RECIPIENT, CommandLineArgument.ArgumentType.String, Strings.SendMail.OptionRecipientShort, Strings.SendMail.OptionRecipientLong), + new CommandLineArgument(OPTION_SENDER, CommandLineArgument.ArgumentType.String, Strings.SendMail.OptionSenderShort, Strings.SendMail.OptionSenderLong, DEFAULT_SENDER), + new CommandLineArgument(OPTION_SUBJECT, CommandLineArgument.ArgumentType.String, Strings.SendMail.OptionSubjectShort, Strings.SendMail.OptionSubjectLong(OPTION_BODY), DEFAULT_SUBJECT), + new CommandLineArgument(OPTION_BODY, CommandLineArgument.ArgumentType.String, Strings.SendMail.OptionBodyShort, Strings.SendMail.OptionBodyLong, DEFAULT_BODY), + new CommandLineArgument(OPTION_SERVER, CommandLineArgument.ArgumentType.String, Strings.SendMail.OptionServerShort, Strings.SendMail.OptionServerLong), + new CommandLineArgument(OPTION_USERNAME, CommandLineArgument.ArgumentType.String, Strings.SendMail.OptionUsernameShort, Strings.SendMail.OptionUsernameLong), + new CommandLineArgument(OPTION_PASSWORD, CommandLineArgument.ArgumentType.String, Strings.SendMail.OptionPasswordShort, Strings.SendMail.OptionPasswordLong), + new CommandLineArgument(OPTION_SENDLEVEL, CommandLineArgument.ArgumentType.String, Strings.SendMail.OptionSendlevelShort, Strings.SendMail.OptionSendlevelLong(ParsedResultType.Success.ToString(), ParsedResultType.Warning.ToString(), ParsedResultType.Error.ToString(), ParsedResultType.Fatal.ToString(), "All"), DEFAULT_LEVEL, null, Enum.GetNames(typeof(ParsedResultType)).Union(new string [] { "All" }).ToArray()), + new CommandLineArgument(OPTION_SENDALL, CommandLineArgument.ArgumentType.Boolean, Strings.SendHttpMessage.SendhttpanyoperationShort, Strings.SendHttpMessage.SendhttpanyoperationLong), + new CommandLineArgument(OPTION_EXTRA_DATA_PARAMETERS, CommandLineArgument.ArgumentType.String, Strings.SendMail.SendmailextradataparametersShort, Strings.SendMail.SendmailextradataparametersLong), - new CommandLineArgument(OPTION_LOG_LEVEL, CommandLineArgument.ArgumentType.Enumeration, Strings.ReportHelper.OptionLoglevelShort, Strings.ReportHelper.OptionLoglevelLong, DEFAULT_LOG_LEVEL.ToString(), null, Enum.GetNames(typeof(Logging.LogMessageType))), - new CommandLineArgument(OPTION_LOG_FILTER, CommandLineArgument.ArgumentType.String, Strings.ReportHelper.OptionLogfilterShort, Strings.ReportHelper.OptionLogfilterLong), - new CommandLineArgument(OPTION_MAX_LOG_LINES, CommandLineArgument.ArgumentType.Integer, Strings.ReportHelper.OptionmaxloglinesShort, Strings.ReportHelper.OptionmaxloglinesLong, DEFAULT_LOGLINES.ToString()), + new CommandLineArgument(OPTION_LOG_LEVEL, CommandLineArgument.ArgumentType.Enumeration, Strings.ReportHelper.OptionLoglevelShort, Strings.ReportHelper.OptionLoglevelLong, DEFAULT_LOG_LEVEL.ToString(), null, Enum.GetNames(typeof(Logging.LogMessageType))), + new CommandLineArgument(OPTION_LOG_FILTER, CommandLineArgument.ArgumentType.String, Strings.ReportHelper.OptionLogfilterShort, Strings.ReportHelper.OptionLogfilterLong), + new CommandLineArgument(OPTION_MAX_LOG_LINES, CommandLineArgument.ArgumentType.Integer, Strings.ReportHelper.OptionmaxloglinesShort, Strings.ReportHelper.OptionmaxloglinesLong, DEFAULT_LOGLINES.ToString()), - new CommandLineArgument(OPTION_RESULT_FORMAT, CommandLineArgument.ArgumentType.Enumeration, Strings.ReportHelper.ResultFormatShort, Strings.ReportHelper.ResultFormatLong(Enum.GetNames(typeof(ResultExportFormat))), DEFAULT_EXPORT_FORMAT.ToString(), null, Enum.GetNames(typeof(ResultExportFormat))), - }); - } - } + new CommandLineArgument(OPTION_RESULT_FORMAT, CommandLineArgument.ArgumentType.Enumeration, Strings.ReportHelper.ResultFormatShort, Strings.ReportHelper.ResultFormatLong(Enum.GetNames(typeof(ResultExportFormat))), DEFAULT_EXPORT_FORMAT.ToString(), null, Enum.GetNames(typeof(ResultExportFormat))), + ]; protected override string SubjectOptionName => OPTION_SUBJECT; protected override string BodyOptionName => OPTION_BODY; @@ -182,6 +183,7 @@ namespace Duplicati.Library.Modules.Builtin protected override string LogFilterOptionName => OPTION_LOG_FILTER; protected override string LogLinesOptionName => OPTION_MAX_LOG_LINES; protected override string ResultFormatOptionName => OPTION_RESULT_FORMAT; + protected override string ExtraDataOptionName => OPTION_EXTRA_DATA_PARAMETERS; /// /// This method is the interception where the module can interact with the execution environment and modify the settings. diff --git a/Duplicati/Library/Modules/Builtin/SendTelegramMessage.cs b/Duplicati/Library/Modules/Builtin/SendTelegramMessage.cs index b4a73948a..0a2ab52e2 100644 --- a/Duplicati/Library/Modules/Builtin/SendTelegramMessage.cs +++ b/Duplicati/Library/Modules/Builtin/SendTelegramMessage.cs @@ -75,6 +75,10 @@ public class SendTelegramMessage : ReportHelper /// Option used to specify what format the result is sent in. /// private const string OPTION_RESULT_FORMAT = "send-telegram-result-output-format"; + /// + /// Option used to specify extra parameters + /// + private const string OPTION_EXTRA_DATA_PARAMETERS = "send-telegram-extra-data-parameters"; /// /// Option used to set the log level @@ -88,7 +92,7 @@ public class SendTelegramMessage : ReportHelper /// Option used to set the maximum number of log lines /// private const string OPTION_MAX_LOG_LINES = "send-telegram-max-log-lines"; - + /// /// Telegram message maximum length. /// @@ -135,21 +139,22 @@ public class SendTelegramMessage : ReportHelper /// /// Gets a list of supported commandline arguments /// - public override IList SupportedCommands - => [ - new CommandLineArgument(OPTION_CHANNEL, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramChannelShort, Strings.SendTelegramMessage.SendTelegramChannelLong), - new CommandLineArgument(OPTION_MESSAGE, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramMessageShort, Strings.SendTelegramMessage.SendTelegramMessageLong, DEFAULT_BODY), - new CommandLineArgument(OPTION_BOTID, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramBotIdShort, Strings.SendTelegramMessage.SendTelegramBotIdLong), - new CommandLineArgument(OPTION_APIKEY, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramApiKeyShort, Strings.SendTelegramMessage.SendTelegramApiKeyLong), - new CommandLineArgument(OPTION_SENDLEVEL, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramLevelShort, Strings.SendTelegramMessage.SendTelegramlevelLong(ParsedResultType.Success.ToString(), ParsedResultType.Warning.ToString(), ParsedResultType.Error.ToString(), ParsedResultType.Fatal.ToString(), "All"), DEFAULT_LEVEL, null, Enum.GetNames(typeof(ParsedResultType)).Union(new string[] { "All" } ).ToArray()), - new CommandLineArgument(OPTION_SENDALL, CommandLineArgument.ArgumentType.Boolean, Strings.SendTelegramMessage.SendTelegramManyOperationShort, Strings.SendTelegramMessage.SendTelegramManyOperationLong), - new CommandLineArgument(OPTION_LOG_LEVEL, CommandLineArgument.ArgumentType.Enumeration, Strings.ReportHelper.OptionLoglevelShort, Strings.ReportHelper.OptionLoglevelLong, DEFAULT_LOG_LEVEL.ToString(), null, Enum.GetNames(typeof(Logging.LogMessageType))), - new CommandLineArgument(OPTION_LOG_FILTER, CommandLineArgument.ArgumentType.String, Strings.ReportHelper.OptionLogfilterShort, Strings.ReportHelper.OptionLogfilterLong), - new CommandLineArgument(OPTION_MAX_LOG_LINES, CommandLineArgument.ArgumentType.Integer, Strings.ReportHelper.OptionmaxloglinesShort, Strings.ReportHelper.OptionmaxloglinesLong, DEFAULT_LOGLINES.ToString()), - new CommandLineArgument(OPTION_RESULT_FORMAT, CommandLineArgument.ArgumentType.Enumeration, Strings.ReportHelper.ResultFormatShort, Strings.ReportHelper.ResultFormatLong(Enum.GetNames(typeof(ResultExportFormat))), DEFAULT_EXPORT_FORMAT.ToString(), null, Enum.GetNames(typeof(ResultExportFormat))), - new CommandLineArgument(OPTION_TOPICID, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramTopicShort, Strings.SendTelegramMessage.SendTelegramTopicLong), - ]; - + public override IList SupportedCommands => + [ + new CommandLineArgument(OPTION_CHANNEL, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramChannelShort, Strings.SendTelegramMessage.SendTelegramChannelLong), + new CommandLineArgument(OPTION_MESSAGE, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramMessageShort, Strings.SendTelegramMessage.SendTelegramMessageLong, DEFAULT_BODY), + new CommandLineArgument(OPTION_EXTRA_DATA_PARAMETERS, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramExtraDataParametersShort, Strings.SendTelegramMessage.SendTelegramExtraDataParametersLong), + new CommandLineArgument(OPTION_BOTID, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramBotIdShort, Strings.SendTelegramMessage.SendTelegramBotIdLong), + new CommandLineArgument(OPTION_APIKEY, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramApiKeyShort, Strings.SendTelegramMessage.SendTelegramApiKeyLong), + new CommandLineArgument(OPTION_SENDLEVEL, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramLevelShort, Strings.SendTelegramMessage.SendTelegramlevelLong(ParsedResultType.Success.ToString(), ParsedResultType.Warning.ToString(), ParsedResultType.Error.ToString(), ParsedResultType.Fatal.ToString(), "All"), DEFAULT_LEVEL, null, Enum.GetNames(typeof(ParsedResultType)).Union(new string[] { "All" } ).ToArray()), + new CommandLineArgument(OPTION_SENDALL, CommandLineArgument.ArgumentType.Boolean, Strings.SendTelegramMessage.SendTelegramManyOperationShort, Strings.SendTelegramMessage.SendTelegramManyOperationLong), + new CommandLineArgument(OPTION_LOG_LEVEL, CommandLineArgument.ArgumentType.Enumeration, Strings.ReportHelper.OptionLoglevelShort, Strings.ReportHelper.OptionLoglevelLong, DEFAULT_LOG_LEVEL.ToString(), null, Enum.GetNames(typeof(Logging.LogMessageType))), + new CommandLineArgument(OPTION_LOG_FILTER, CommandLineArgument.ArgumentType.String, Strings.ReportHelper.OptionLogfilterShort, Strings.ReportHelper.OptionLogfilterLong), + new CommandLineArgument(OPTION_MAX_LOG_LINES, CommandLineArgument.ArgumentType.Integer, Strings.ReportHelper.OptionmaxloglinesShort, Strings.ReportHelper.OptionmaxloglinesLong, DEFAULT_LOGLINES.ToString()), + new CommandLineArgument(OPTION_RESULT_FORMAT, CommandLineArgument.ArgumentType.Enumeration, Strings.ReportHelper.ResultFormatShort, Strings.ReportHelper.ResultFormatLong(Enum.GetNames(typeof(ResultExportFormat))), DEFAULT_EXPORT_FORMAT.ToString(), null, Enum.GetNames(typeof(ResultExportFormat))), + new CommandLineArgument(OPTION_TOPICID, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramTopicShort, Strings.SendTelegramMessage.SendTelegramTopicLong), + ]; + protected override string SubjectOptionName => OPTION_MESSAGE; protected override string BodyOptionName => OPTION_MESSAGE; protected override string ActionLevelOptionName => OPTION_SENDLEVEL; @@ -158,6 +163,7 @@ public class SendTelegramMessage : ReportHelper protected override string LogFilterOptionName => OPTION_LOG_FILTER; protected override string LogLinesOptionName => OPTION_MAX_LOG_LINES; protected override string ResultFormatOptionName => OPTION_RESULT_FORMAT; + protected override string ExtraDataOptionName => OPTION_EXTRA_DATA_PARAMETERS; /// /// The server username @@ -195,14 +201,14 @@ public class SendTelegramMessage : ReportHelper } #endregion - + /// protected override string ReplaceTemplate(string input, object result, Exception exception, bool subjectline) { // No need to do the expansion as we throw away the result return subjectline ? string.Empty : base.ReplaceTemplate(input, result, exception, subjectline); } - + /// /// Sends message to telegram /// @@ -213,13 +219,13 @@ public class SendTelegramMessage : ReportHelper try { if (string.IsNullOrEmpty(body)) - return; - + return; + // Combine subject and body body = string.Join(Environment.NewLine, subject, body); // Split message into chunks if needed - var messages = body.Length <= TELEGRAM_MAX_LENGTH + var messages = body.Length <= TELEGRAM_MAX_LENGTH ? [body] : Enumerable.Range(0, (body.Length + TELEGRAM_MAX_LENGTH - 1) / TELEGRAM_MAX_LENGTH) .Select(i => body.Substring( @@ -233,11 +239,11 @@ public class SendTelegramMessage : ReportHelper } catch (Exception e) { - Logging.Log.WriteWarningMessage(LOGTAG, "telegramSendError", e, + Logging.Log.WriteWarningMessage(LOGTAG, "telegramSendError", e, "Failed to process message for telegram: {0}", e.Message); } } - + /// /// Sends the message/partial message to telegram /// @@ -252,13 +258,13 @@ public class SendTelegramMessage : ReportHelper throw new Exception("Telegram Bot ID is required and not set"); if (string.IsNullOrWhiteSpace(m_apikey)) throw new Exception("Telegram API Key is required and not set"); - + var p = new { chat_id = Uri.EscapeDataString(m_channelId), message_thread_id = string.IsNullOrWhiteSpace(m_topicId) ? null : Uri.EscapeDataString(m_topicId), parse_mode = "Markdown", - text = Uri.EscapeDataString(totalParts > 1 + text = Uri.EscapeDataString(totalParts > 1 ? $"Part {partNumber}/{totalParts}:\n{message}" : message), botId = Uri.EscapeDataString(m_botid), @@ -268,7 +274,7 @@ public class SendTelegramMessage : ReportHelper var baseUrl = $"https://api.telegram.org/bot{p.botId}:{p.apiKey}/sendMessage"; var url = $"{baseUrl}?chat_id={p.chat_id}&parse_mode={p.parse_mode}&text={p.text}" + $"{(!string.IsNullOrWhiteSpace(m_topicId) ? $"&message_thread_id={m_topicId}" : string.Empty)}"; - + using var timeoutToken = new CancellationTokenSource(); timeoutToken.CancelAfter(REQUEST_TIMEOUT); @@ -277,14 +283,14 @@ public class SendTelegramMessage : ReportHelper var responseContent = await response.Content.ReadAsStringAsync(timeoutToken.Token).ConfigureAwait(false); if (!responseContent.Contains("\"ok\":true")) - Logging.Log.WriteWarningMessage(LOGTAG, "telegramSendError", null, - "Failed to send to telegram messages part {0}/{1}: {2}", + Logging.Log.WriteWarningMessage(LOGTAG, "telegramSendError", null, + "Failed to send to telegram messages part {0}/{1}: {2}", partNumber, totalParts, responseContent); } catch (Exception e) { - Logging.Log.WriteWarningMessage(LOGTAG, "telegramSendError", e, - "Failed to send to telegram messages part {0}/{1}: {2}", + Logging.Log.WriteWarningMessage(LOGTAG, "telegramSendError", e, + "Failed to send to telegram messages part {0}/{1}: {2}", partNumber, totalParts, e.Message); } } diff --git a/Duplicati/Library/Modules/Builtin/Strings.cs b/Duplicati/Library/Modules/Builtin/Strings.cs index bcd66b666..fbccba239 100644 --- a/Duplicati/Library/Modules/Builtin/Strings.cs +++ b/Duplicati/Library/Modules/Builtin/Strings.cs @@ -136,6 +136,8 @@ To enable SMTP over SSL, use the format smtps://example.com. To enable SMTP STAR public static string SendMailLog(string message) { return LC.L(@"Whole SMTP communication: {0}", message); } public static string SendMailFailedRetryError(string failedserver, string message, string retryserver) { return LC.L(@"Failed to send email with server: {0}, message: {1}, retrying with {2}", failedserver, message, retryserver); } public static string SendMailSuccess(string server) { return LC.L(@"Email sent successfully using server: {0}", server); } + public static string SendmailextradataparametersLong { get { return LC.L(@"Use this option to set extra parameters for the message body. This parameter can either be a querystring (e.g. ""parameter1=value1¶meter2=value2"") or a JSON key/value object (e.g. ""{\""parameter1\"": \""value1\"", \""parameter2\"": \""value2\""}"")."); } } + public static string SendmailextradataparametersShort { get { return LC.L(@"Extra parameters for the message sent"); } } } internal static class SendJabberMessage { @@ -163,6 +165,8 @@ You can supply multiple options with a comma separator, e.g. ""{0},{1}"". The sp public static string SendxmppanyoperationLong { get { return LC.L(@"By default, messages will only be sent after a backup operation. Use this option to send messages for all operations."); } } public static string SendxmppanyoperationShort { get { return LC.L(@"Send messages for all operations"); } } public static string LoginTimeoutError { get { return LC.L(@"Timeout occurred while logging in to jabber server"); } } + public static string SendxmppextradataparametersLong { get { return LC.L(@"Use this option to set extra parameters for the message body. This parameter can either be a querystring (e.g. ""parameter1=value1¶meter2=value2"") or a JSON key/value object (e.g. ""{\""parameter1\"": \""value1\"", \""parameter2\"": \""value2\""}"")."); } } + public static string SendxmppextradataparametersShort { get { return LC.L(@"Extra parameters for the message sent"); } } } internal static class SendTelegramMessage @@ -194,6 +198,8 @@ You can supply multiple options with a comma separator, e.g. ""{0},{1}"". The sp public static string SendTelegramLevelShort => LC.L(@"The messages to send"); public static string SendTelegramManyOperationLong => LC.L(@"By default, messages will only be sent after a backup operation. Use this option to send messages for all operations."); public static string SendTelegramManyOperationShort => LC.L(@"Send messages for all operations"); + public static string SendTelegramExtraDataParametersLong { get { return LC.L(@"Use this option to set extra parameters for the message body. This parameter can either be a querystring (e.g. ""parameter1=value1¶meter2=value2"") or a JSON key/value object (e.g. ""{\""parameter1\"": \""value1\"", \""parameter2\"": \""value2\""}"")."); } } + public static string SendTelegramExtraDataParametersShort { get { return LC.L(@"Extra parameters for the message sent"); } } } internal static class SendHttpMessage @@ -216,6 +222,8 @@ All command line options are also reported within %value%, e.g. %volsize%. Any u public static string SendhttpmessageparameternameShort { get { return LC.L(@"The name of the parameter to send the message as"); } } public static string SendhttpextraparametersLong { get { return LC.L(@"Use this option to set extra parameters to add to the http message, e.g. ""parameter1=value1¶meter2=value2"""); } } public static string SendhttpextraparametersShort { get { return LC.L(@"Extra parameters to add to the http message"); } } + public static string SendhttpextradataparametersLong { get { return LC.L(@"Use this option to set extra parameters for the message body. This parameter can either be a querystring (e.g. ""parameter1=value1¶meter2=value2"") or a JSON key/value object (e.g. ""{\""parameter1\"": \""value1\"", \""parameter2\"": \""value2\""}"")."); } } + public static string SendhttpextradataparametersShort { get { return LC.L(@"Extra parameters for the message sent"); } } public static string SendhttplevelLong(string success, string warning, string error, string fatal, string all) { return LC.L(@"You can specify one of ""{0}"", ""{1}"", ""{2}"", ""{3}"". You can supply multiple options with a comma separator, e.g. ""{0},{1}"". The special value ""{4}"" is a shorthand for ""{0},{1},{2},{3}"" and will cause all backup operations to send a message.", success, warning, error, fatal, all); } public static string SendhttplevelShort { get { return LC.L(@"The messages to send"); } } From bf3ff44e32269fad1fa51181a2dd2a0c85ec3ee8 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 24 Mar 2025 14:43:00 +0100 Subject: [PATCH 40/48] Collapsed into single option --- .../Library/Modules/Builtin/SendHttpMessage.cs | 7 +------ .../Modules/Builtin/SendJabberMessage.cs | 6 +++--- Duplicati/Library/Modules/Builtin/SendMail.cs | 6 +++--- .../Modules/Builtin/SendTelegramMessage.cs | 6 +++--- Duplicati/Library/Modules/Builtin/Strings.cs | 18 ++++++++---------- 5 files changed, 18 insertions(+), 25 deletions(-) diff --git a/Duplicati/Library/Modules/Builtin/SendHttpMessage.cs b/Duplicati/Library/Modules/Builtin/SendHttpMessage.cs index 9e83ed4b2..1f16ac323 100644 --- a/Duplicati/Library/Modules/Builtin/SendHttpMessage.cs +++ b/Duplicati/Library/Modules/Builtin/SendHttpMessage.cs @@ -81,10 +81,6 @@ namespace Duplicati.Library.Modules.Builtin /// private const string OPTION_EXTRA_PARAMETERS = "send-http-extra-parameters"; /// - /// Option used to specify extra parameters - /// - private const string OPTION_EXTRA_DATA_PARAMETERS = "send-http-extra-data-parameters"; - /// /// Option used to specify report level /// private const string OPTION_SENDLEVEL = "send-http-level"; @@ -221,7 +217,6 @@ namespace Duplicati.Library.Modules.Builtin new CommandLineArgument(OPTION_MESSAGE, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.SendhttpmessageShort, Strings.SendHttpMessage.SendhttpmessageLong, DEFAULT_BODY), new CommandLineArgument(OPTION_MESSAGE_PARAMETER_NAME, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.SendhttpmessageparameternameShort, Strings.SendHttpMessage.SendhttpmessageparameternameLong, DEFAULT_MESSAGE_PARAMETER_NAME), new CommandLineArgument(OPTION_EXTRA_PARAMETERS, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.SendhttpextraparametersShort, Strings.SendHttpMessage.SendhttpextraparametersLong), - new CommandLineArgument(OPTION_EXTRA_DATA_PARAMETERS, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.SendhttpextradataparametersShort, Strings.SendHttpMessage.SendhttpextradataparametersLong), new CommandLineArgument(OPTION_SENDLEVEL, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.SendhttplevelShort, Strings.SendHttpMessage.SendhttplevelLong(ParsedResultType.Success.ToString(), ParsedResultType.Warning.ToString(), ParsedResultType.Error.ToString(), ParsedResultType.Fatal.ToString(), "All"), DEFAULT_LEVEL, null, Enum.GetNames(typeof(ParsedResultType)).Union(new string[] { "All" } ).ToArray()), new CommandLineArgument(OPTION_SENDALL, CommandLineArgument.ArgumentType.Boolean, Strings.SendHttpMessage.SendhttpanyoperationShort, Strings.SendHttpMessage.SendhttpanyoperationLong), @@ -250,7 +245,7 @@ namespace Duplicati.Library.Modules.Builtin protected override string LogFilterOptionName => OPTION_LOG_FILTER; protected override string LogLinesOptionName => OPTION_MAX_LOG_LINES; protected override string ResultFormatOptionName => OPTION_RESULT_FORMAT; - protected override string ExtraDataOptionName => OPTION_EXTRA_DATA_PARAMETERS; + protected override string ExtraDataOptionName => OPTION_EXTRA_PARAMETERS; /// /// This method is the interception where the module can interact with the execution environment and modify the settings. diff --git a/Duplicati/Library/Modules/Builtin/SendJabberMessage.cs b/Duplicati/Library/Modules/Builtin/SendJabberMessage.cs index 049e09a39..c740bd613 100644 --- a/Duplicati/Library/Modules/Builtin/SendJabberMessage.cs +++ b/Duplicati/Library/Modules/Builtin/SendJabberMessage.cs @@ -53,7 +53,7 @@ namespace Duplicati.Library.Modules.Builtin /// /// Option used to specify extra parameters /// - private const string OPTION_EXTRA_DATA_PARAMETERS = "send-xmpp-extra-data-parameters"; + private const string OPTION_EXTRA_PARAMETERS = "send-xmpp-extra-parameters"; /// /// Option used to specify report level @@ -142,7 +142,7 @@ namespace Duplicati.Library.Modules.Builtin [ new CommandLineArgument(OPTION_RECIPIENT, CommandLineArgument.ArgumentType.String, Strings.SendJabberMessage.SendxmpptoShort, Strings.SendJabberMessage.SendxmpptoLong), new CommandLineArgument(OPTION_MESSAGE, CommandLineArgument.ArgumentType.String, Strings.SendJabberMessage.SendxmppmessageShort, Strings.SendJabberMessage.SendxmppmessageLong, DEFAULT_BODY), - new CommandLineArgument(OPTION_EXTRA_DATA_PARAMETERS, CommandLineArgument.ArgumentType.String, Strings.SendJabberMessage.SendxmppextradataparametersShort, Strings.SendJabberMessage.SendxmppextradataparametersLong), + new CommandLineArgument(OPTION_EXTRA_PARAMETERS, CommandLineArgument.ArgumentType.String, Strings.SendJabberMessage.SendxmppextraparametersShort, Strings.SendJabberMessage.SendxmppextraparametersLong), new CommandLineArgument(OPTION_USERNAME, CommandLineArgument.ArgumentType.String, Strings.SendJabberMessage.SendxmppusernameShort, Strings.SendJabberMessage.SendxmppusernameLong), new CommandLineArgument(OPTION_PASSWORD, CommandLineArgument.ArgumentType.String, Strings.SendJabberMessage.SendxmpppasswordShort, Strings.SendJabberMessage.SendxmpppasswordLong), new CommandLineArgument(OPTION_SENDLEVEL, CommandLineArgument.ArgumentType.String, Strings.SendJabberMessage.SendxmpplevelShort, Strings.SendJabberMessage.SendxmpplevelLong(ParsedResultType.Success.ToString(), ParsedResultType.Warning.ToString(), ParsedResultType.Error.ToString(), ParsedResultType.Fatal.ToString(), "All"), DEFAULT_LEVEL, null, Enum.GetNames(typeof(ParsedResultType)).Union(new string[] { "All" } ).ToArray()), @@ -164,7 +164,7 @@ namespace Duplicati.Library.Modules.Builtin protected override string LogFilterOptionName => OPTION_LOG_FILTER; protected override string LogLinesOptionName => OPTION_MAX_LOG_LINES; protected override string ResultFormatOptionName => OPTION_RESULT_FORMAT; - protected override string ExtraDataOptionName => OPTION_EXTRA_DATA_PARAMETERS; + protected override string ExtraDataOptionName => OPTION_EXTRA_PARAMETERS; /// /// This method is the interception where the module can interact with the execution environment and modify the settings. diff --git a/Duplicati/Library/Modules/Builtin/SendMail.cs b/Duplicati/Library/Modules/Builtin/SendMail.cs index fe23dc6f9..d33b7964b 100644 --- a/Duplicati/Library/Modules/Builtin/SendMail.cs +++ b/Duplicati/Library/Modules/Builtin/SendMail.cs @@ -82,7 +82,7 @@ namespace Duplicati.Library.Modules.Builtin /// /// Option used to specify extra parameters /// - private const string OPTION_EXTRA_DATA_PARAMETERS = "send-mail-extra-data-parameters"; + private const string OPTION_EXTRA_PARAMETERS = "send-mail-extra-parameters"; /// /// Option used to set the log level for mail reports @@ -166,7 +166,7 @@ namespace Duplicati.Library.Modules.Builtin new CommandLineArgument(OPTION_PASSWORD, CommandLineArgument.ArgumentType.String, Strings.SendMail.OptionPasswordShort, Strings.SendMail.OptionPasswordLong), new CommandLineArgument(OPTION_SENDLEVEL, CommandLineArgument.ArgumentType.String, Strings.SendMail.OptionSendlevelShort, Strings.SendMail.OptionSendlevelLong(ParsedResultType.Success.ToString(), ParsedResultType.Warning.ToString(), ParsedResultType.Error.ToString(), ParsedResultType.Fatal.ToString(), "All"), DEFAULT_LEVEL, null, Enum.GetNames(typeof(ParsedResultType)).Union(new string [] { "All" }).ToArray()), new CommandLineArgument(OPTION_SENDALL, CommandLineArgument.ArgumentType.Boolean, Strings.SendHttpMessage.SendhttpanyoperationShort, Strings.SendHttpMessage.SendhttpanyoperationLong), - new CommandLineArgument(OPTION_EXTRA_DATA_PARAMETERS, CommandLineArgument.ArgumentType.String, Strings.SendMail.SendmailextradataparametersShort, Strings.SendMail.SendmailextradataparametersLong), + new CommandLineArgument(OPTION_EXTRA_PARAMETERS, CommandLineArgument.ArgumentType.String, Strings.SendMail.SendmailextraparametersShort, Strings.SendMail.SendmailextraparametersLong), new CommandLineArgument(OPTION_LOG_LEVEL, CommandLineArgument.ArgumentType.Enumeration, Strings.ReportHelper.OptionLoglevelShort, Strings.ReportHelper.OptionLoglevelLong, DEFAULT_LOG_LEVEL.ToString(), null, Enum.GetNames(typeof(Logging.LogMessageType))), new CommandLineArgument(OPTION_LOG_FILTER, CommandLineArgument.ArgumentType.String, Strings.ReportHelper.OptionLogfilterShort, Strings.ReportHelper.OptionLogfilterLong), @@ -183,7 +183,7 @@ namespace Duplicati.Library.Modules.Builtin protected override string LogFilterOptionName => OPTION_LOG_FILTER; protected override string LogLinesOptionName => OPTION_MAX_LOG_LINES; protected override string ResultFormatOptionName => OPTION_RESULT_FORMAT; - protected override string ExtraDataOptionName => OPTION_EXTRA_DATA_PARAMETERS; + protected override string ExtraDataOptionName => OPTION_EXTRA_PARAMETERS; /// /// This method is the interception where the module can interact with the execution environment and modify the settings. diff --git a/Duplicati/Library/Modules/Builtin/SendTelegramMessage.cs b/Duplicati/Library/Modules/Builtin/SendTelegramMessage.cs index 0a2ab52e2..5f7323f5c 100644 --- a/Duplicati/Library/Modules/Builtin/SendTelegramMessage.cs +++ b/Duplicati/Library/Modules/Builtin/SendTelegramMessage.cs @@ -78,7 +78,7 @@ public class SendTelegramMessage : ReportHelper /// /// Option used to specify extra parameters /// - private const string OPTION_EXTRA_DATA_PARAMETERS = "send-telegram-extra-data-parameters"; + private const string OPTION_EXTRA_PARAMETERS = "send-telegram-extra-parameters"; /// /// Option used to set the log level @@ -143,7 +143,7 @@ public class SendTelegramMessage : ReportHelper [ new CommandLineArgument(OPTION_CHANNEL, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramChannelShort, Strings.SendTelegramMessage.SendTelegramChannelLong), new CommandLineArgument(OPTION_MESSAGE, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramMessageShort, Strings.SendTelegramMessage.SendTelegramMessageLong, DEFAULT_BODY), - new CommandLineArgument(OPTION_EXTRA_DATA_PARAMETERS, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramExtraDataParametersShort, Strings.SendTelegramMessage.SendTelegramExtraDataParametersLong), + new CommandLineArgument(OPTION_EXTRA_PARAMETERS, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramExtraParametersShort, Strings.SendTelegramMessage.SendTelegramExtraParametersLong), new CommandLineArgument(OPTION_BOTID, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramBotIdShort, Strings.SendTelegramMessage.SendTelegramBotIdLong), new CommandLineArgument(OPTION_APIKEY, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramApiKeyShort, Strings.SendTelegramMessage.SendTelegramApiKeyLong), new CommandLineArgument(OPTION_SENDLEVEL, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramLevelShort, Strings.SendTelegramMessage.SendTelegramlevelLong(ParsedResultType.Success.ToString(), ParsedResultType.Warning.ToString(), ParsedResultType.Error.ToString(), ParsedResultType.Fatal.ToString(), "All"), DEFAULT_LEVEL, null, Enum.GetNames(typeof(ParsedResultType)).Union(new string[] { "All" } ).ToArray()), @@ -163,7 +163,7 @@ public class SendTelegramMessage : ReportHelper protected override string LogFilterOptionName => OPTION_LOG_FILTER; protected override string LogLinesOptionName => OPTION_MAX_LOG_LINES; protected override string ResultFormatOptionName => OPTION_RESULT_FORMAT; - protected override string ExtraDataOptionName => OPTION_EXTRA_DATA_PARAMETERS; + protected override string ExtraDataOptionName => OPTION_EXTRA_PARAMETERS; /// /// The server username diff --git a/Duplicati/Library/Modules/Builtin/Strings.cs b/Duplicati/Library/Modules/Builtin/Strings.cs index fbccba239..eec8f6f92 100644 --- a/Duplicati/Library/Modules/Builtin/Strings.cs +++ b/Duplicati/Library/Modules/Builtin/Strings.cs @@ -136,8 +136,8 @@ To enable SMTP over SSL, use the format smtps://example.com. To enable SMTP STAR public static string SendMailLog(string message) { return LC.L(@"Whole SMTP communication: {0}", message); } public static string SendMailFailedRetryError(string failedserver, string message, string retryserver) { return LC.L(@"Failed to send email with server: {0}, message: {1}, retrying with {2}", failedserver, message, retryserver); } public static string SendMailSuccess(string server) { return LC.L(@"Email sent successfully using server: {0}", server); } - public static string SendmailextradataparametersLong { get { return LC.L(@"Use this option to set extra parameters for the message body. This parameter can either be a querystring (e.g. ""parameter1=value1¶meter2=value2"") or a JSON key/value object (e.g. ""{\""parameter1\"": \""value1\"", \""parameter2\"": \""value2\""}"")."); } } - public static string SendmailextradataparametersShort { get { return LC.L(@"Extra parameters for the message sent"); } } + public static string SendmailextraparametersLong { get { return LC.L(@"Use this option to set extra parameters for the message body. This parameter can either be a querystring (e.g. ""parameter1=value1¶meter2=value2"") or a JSON key/value object (e.g. ""{\""parameter1\"": \""value1\"", \""parameter2\"": \""value2\""}"")."); } } + public static string SendmailextraparametersShort { get { return LC.L(@"Extra parameters for the message sent"); } } } internal static class SendJabberMessage { @@ -165,8 +165,8 @@ You can supply multiple options with a comma separator, e.g. ""{0},{1}"". The sp public static string SendxmppanyoperationLong { get { return LC.L(@"By default, messages will only be sent after a backup operation. Use this option to send messages for all operations."); } } public static string SendxmppanyoperationShort { get { return LC.L(@"Send messages for all operations"); } } public static string LoginTimeoutError { get { return LC.L(@"Timeout occurred while logging in to jabber server"); } } - public static string SendxmppextradataparametersLong { get { return LC.L(@"Use this option to set extra parameters for the message body. This parameter can either be a querystring (e.g. ""parameter1=value1¶meter2=value2"") or a JSON key/value object (e.g. ""{\""parameter1\"": \""value1\"", \""parameter2\"": \""value2\""}"")."); } } - public static string SendxmppextradataparametersShort { get { return LC.L(@"Extra parameters for the message sent"); } } + public static string SendxmppextraparametersLong { get { return LC.L(@"Use this option to set extra parameters for the message body. This parameter can either be a querystring (e.g. ""parameter1=value1¶meter2=value2"") or a JSON key/value object (e.g. ""{\""parameter1\"": \""value1\"", \""parameter2\"": \""value2\""}"")."); } } + public static string SendxmppextraparametersShort { get { return LC.L(@"Extra parameters for the message sent"); } } } internal static class SendTelegramMessage @@ -198,8 +198,8 @@ You can supply multiple options with a comma separator, e.g. ""{0},{1}"". The sp public static string SendTelegramLevelShort => LC.L(@"The messages to send"); public static string SendTelegramManyOperationLong => LC.L(@"By default, messages will only be sent after a backup operation. Use this option to send messages for all operations."); public static string SendTelegramManyOperationShort => LC.L(@"Send messages for all operations"); - public static string SendTelegramExtraDataParametersLong { get { return LC.L(@"Use this option to set extra parameters for the message body. This parameter can either be a querystring (e.g. ""parameter1=value1¶meter2=value2"") or a JSON key/value object (e.g. ""{\""parameter1\"": \""value1\"", \""parameter2\"": \""value2\""}"")."); } } - public static string SendTelegramExtraDataParametersShort { get { return LC.L(@"Extra parameters for the message sent"); } } + public static string SendTelegramExtraParametersLong { get { return LC.L(@"Use this option to set extra parameters for the message body. This parameter can either be a querystring (e.g. ""parameter1=value1¶meter2=value2"") or a JSON key/value object (e.g. ""{\""parameter1\"": \""value1\"", \""parameter2\"": \""value2\""}"")."); } } + public static string SendTelegramExtraParametersShort { get { return LC.L(@"Extra parameters for the message sent"); } } } internal static class SendHttpMessage @@ -220,10 +220,8 @@ All command line options are also reported within %value%, e.g. %volsize%. Any u public static string SendhttpmessageShort { get { return LC.L(@"The message template"); } } public static string SendhttpmessageparameternameLong { get { return LC.L(@"Use this option to set a name of the parameter to send the message as."); } } public static string SendhttpmessageparameternameShort { get { return LC.L(@"The name of the parameter to send the message as"); } } - public static string SendhttpextraparametersLong { get { return LC.L(@"Use this option to set extra parameters to add to the http message, e.g. ""parameter1=value1¶meter2=value2"""); } } - public static string SendhttpextraparametersShort { get { return LC.L(@"Extra parameters to add to the http message"); } } - public static string SendhttpextradataparametersLong { get { return LC.L(@"Use this option to set extra parameters for the message body. This parameter can either be a querystring (e.g. ""parameter1=value1¶meter2=value2"") or a JSON key/value object (e.g. ""{\""parameter1\"": \""value1\"", \""parameter2\"": \""value2\""}"")."); } } - public static string SendhttpextradataparametersShort { get { return LC.L(@"Extra parameters for the message sent"); } } + public static string SendhttpextraparametersLong { get { return LC.L(@"Use this option to set extra parameters for the message body. This parameter can either be a querystring (e.g. ""parameter1=value1¶meter2=value2"") or a JSON key/value object (e.g. ""{\""parameter1\"": \""value1\"", \""parameter2\"": \""value2\""}"")."); } } + public static string SendhttpextraparametersShort { get { return LC.L(@"Extra parameters for the message sent"); } } public static string SendhttplevelLong(string success, string warning, string error, string fatal, string all) { return LC.L(@"You can specify one of ""{0}"", ""{1}"", ""{2}"", ""{3}"". You can supply multiple options with a comma separator, e.g. ""{0},{1}"". The special value ""{4}"" is a shorthand for ""{0},{1},{2},{3}"" and will cause all backup operations to send a message.", success, warning, error, fatal, all); } public static string SendhttplevelShort { get { return LC.L(@"The messages to send"); } } From 89612753afd63d78d185bdb57bae31d5563c153d Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 24 Mar 2025 17:08:36 +0100 Subject: [PATCH 41/48] Updates to fic nullability issues with B2. Also fixed a few places where blocking calls were made. --- Duplicati/Library/Backend/Backblaze/B2.cs | 65 +++--- .../Library/Backend/Backblaze/B2AuthHelper.cs | 195 +++++++++--------- .../{AuthResponse.cs => ApiAuthResponse.cs} | 2 +- .../Backblaze/Model/ListBucketsRequest.cs | 2 +- 4 files changed, 134 insertions(+), 130 deletions(-) rename Duplicati/Library/Backend/Backblaze/Model/{AuthResponse.cs => ApiAuthResponse.cs} (98%) diff --git a/Duplicati/Library/Backend/Backblaze/B2.cs b/Duplicati/Library/Backend/Backblaze/B2.cs index 9c92e4f51..74462d421 100644 --- a/Duplicati/Library/Backend/Backblaze/B2.cs +++ b/Duplicati/Library/Backend/Backblaze/B2.cs @@ -207,8 +207,8 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend _httpClient = HttpClientHelper.CreateClient(); _httpClient.Timeout = Timeout.InfiniteTimeSpan; - _b2AuthHelper = new B2AuthHelper(auth.Username!, auth.Password!, _httpClient); _timeouts = TimeoutOptionsHelper.Parse(options); + _b2AuthHelper = new B2AuthHelper(auth.Username!, auth.Password!, _httpClient, _timeouts); } @@ -224,16 +224,18 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend /// private async Task GetBucketAsync(CancellationToken cancellationToken) { - if (_bucket != null) return _bucket; + if (_bucket != null) + return _bucket; + var config = await _b2AuthHelper.GetConfigAsync(cancellationToken).ConfigureAwait(false); await _cacheLock.WaitAsync(cancellationToken).ConfigureAwait(false); try { var buckets = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct => await _b2AuthHelper.PostAndGetJsonDataAsync( - $"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_list_buckets", - new ListBucketsRequest(accountId: _b2AuthHelper.AccountId), + $"{config.APIUrl}/b2api/v1/b2_list_buckets", + new ListBucketsRequest(accountId: config.AccountID), ct).ConfigureAwait(false) ).ConfigureAwait(false); @@ -256,12 +258,15 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend if (_uploadUrl != null) return _uploadUrl; + var config = await _b2AuthHelper.GetConfigAsync(cancellationToken).ConfigureAwait(false); + var bucket = await GetBucketAsync(cancellationToken).ConfigureAwait(false); + _uploadUrl = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct => await _b2AuthHelper.PostAndGetJsonDataAsync( - $"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_get_upload_url", + $"{config.APIUrl}/b2api/v1/b2_get_upload_url", new UploadUrlRequest { - BucketID = (await GetBucketAsync(cancellationToken)).BucketID + BucketID = bucket.BucketID ?? throw new Exception("BucketID is null") }, ct).ConfigureAwait(false) @@ -292,11 +297,6 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend throw new FileMissingException(); } - /// - /// Returns the DownloadURL, either cached, or making a call to the server's /b2_authorize_account - /// - private string DownloadUrl => string.IsNullOrEmpty(_downloadUrl) ? _b2AuthHelper.DownloadUrl : _downloadUrl; - /// public IList SupportedCommands => new List([ @@ -418,15 +418,16 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend /// Exceptions arising from either code execution or FileMissingException public async Task GetAsync(string remotename, Stream stream, CancellationToken cancellationToken) { - if (_filecache == null || !_filecache.ContainsKey(remotename)) await RebuildFileCache(cancellationToken).ConfigureAwait(false); + var config = await _b2AuthHelper.GetConfigAsync(cancellationToken).ConfigureAwait(false); + using var request = _filecache != null && _filecache.ContainsKey(remotename) - ? _b2AuthHelper.CreateRequest( - $"{DownloadUrl}/b2api/v1/b2_download_file_by_id?fileId={Utility.Uri.UrlEncode(await GetFileId(remotename, cancellationToken))}") - : _b2AuthHelper.CreateRequest( - $"{DownloadUrl}/{_urlencodedPrefix}{Utility.Uri.UrlPathEncode(remotename)}"); + ? await _b2AuthHelper.CreateRequestAsync( + $"{config.DownloadUrl}/b2api/v1/b2_download_file_by_id?fileId={Utility.Uri.UrlEncode(await GetFileId(remotename, cancellationToken))}", HttpMethod.Get, cancellationToken).ConfigureAwait(false) + : await _b2AuthHelper.CreateRequestAsync( + $"{config.DownloadUrl}/{_urlencodedPrefix}{Utility.Uri.UrlPathEncode(remotename)}", HttpMethod.Get, cancellationToken).ConfigureAwait(false); HttpResponseMessage? response = null; try @@ -444,7 +445,7 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend if (B2AuthHelper.GetExceptionStatusCode(ex) == HttpStatusCode.NotFound) throw new FileMissingException(); - _b2AuthHelper.AttemptParseAndThrowException(ex, response); + await _b2AuthHelper.AttemptParseAndThrowExceptionAsync(ex, response, cancellationToken).ConfigureAwait(false); throw; } finally @@ -463,7 +464,8 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend var cache = new Dictionary>(); string? nextFileId = null; string? nextFileName = null; - var listVersionUrl = $"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_list_file_versions"; + var config = await _b2AuthHelper.GetConfigAsync(cancellationToken).ConfigureAwait(false); + var listVersionUrl = $"{config.APIUrl}/b2api/v1/b2_list_file_versions"; var listRetryHelper = RetryAfterHelper.CreateOrGetRetryAfterHelper(listVersionUrl); @@ -473,7 +475,7 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend { var resp = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct => await _b2AuthHelper.PostAndGetJsonDataAsync( - $"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_list_file_versions", + $"{config.APIUrl}/b2api/v1/b2_list_file_versions", new ListFilesRequest { BucketID = (await GetBucketAsync(cancellationToken)).BucketID @@ -582,10 +584,12 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend if (!filecache.TryGetValue(remotename, out var value)) throw new FileMissingException(); + var config = await _b2AuthHelper.GetConfigAsync(cancellationToken).ConfigureAwait(false); + foreach (var n in value.OrderBy(x => x.UploadTimestamp)) await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct => await _b2AuthHelper.PostAndGetJsonDataAsync( - $"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_delete_file_version", + $"{config.APIUrl}/b2api/v1/b2_delete_file_version", new DeleteRequest { FileName = _prefix + remotename, @@ -619,12 +623,13 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend /// public async Task CreateFolderAsync(CancellationToken cancellationToken) { + var config = await _b2AuthHelper.GetConfigAsync(cancellationToken); _bucket = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct => await _b2AuthHelper.PostAndGetJsonDataAsync( - $"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_create_bucket", + $"{config.APIUrl}/b2api/v1/b2_create_bucket", new BucketEntity { - AccountID = _b2AuthHelper.AccountId, + AccountID = config.AccountID, BucketName = _bucketName, BucketType = _bucketType }, @@ -643,12 +648,18 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend public string Description => Strings.B2.Description; /// - public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new string?[] { - new System.Uri(B2AuthHelper.AUTH_URL).Host, - _b2AuthHelper?.ApiDnsName, - _b2AuthHelper?.DownloadDnsName + public async Task GetDNSNamesAsync(CancellationToken cancelToken) + { + var config = await _b2AuthHelper.GetConfigAsync(cancelToken); + return new string?[] { + B2AuthHelper.AUTH_URL, + config.APIUrl, + config.DownloadUrl }.WhereNotNullOrWhiteSpace() - .ToArray()); + .Select(x => new System.Uri(x).Host) + .Distinct() + .ToArray(); + } /// /// Handles disposal of the backend's resources. diff --git a/Duplicati/Library/Backend/Backblaze/B2AuthHelper.cs b/Duplicati/Library/Backend/Backblaze/B2AuthHelper.cs index 534168d38..0b569ede8 100644 --- a/Duplicati/Library/Backend/Backblaze/B2AuthHelper.cs +++ b/Duplicati/Library/Backend/Backblaze/B2AuthHelper.cs @@ -23,9 +23,9 @@ using System.Net; using System.Text; using Duplicati.Library.Backend.Backblaze.Model; using Duplicati.Library.Interface; +using Duplicati.Library.Utility.Options; using Newtonsoft.Json; using Exception = System.Exception; -using Uri = System.Uri; namespace Duplicati.Library.Backend.Backblaze; @@ -35,8 +35,17 @@ namespace Duplicati.Library.Backend.Backblaze; /// Username /// Password /// HttpClient instance to use and pass along to JsonWebHelperHttpClient -public class B2AuthHelper(string userid, string password, HttpClient httpClient) : JsonWebHelperHttpClient(httpClient) +public class B2AuthHelper(string userid, string password, HttpClient httpClient, TimeoutOptionsHelper.Timeouts timeouts) : JsonWebHelperHttpClient(httpClient) { + /// + /// The configuration details after authorization + /// + /// The account ID + /// The API URL + /// The authorization token + /// The download URL + public record AuthResponse(string AccountID, string APIUrl, string AuthorizationToken, string DownloadUrl); + /// /// Cached authorization response /// @@ -57,46 +66,22 @@ public class B2AuthHelper(string userid, string password, HttpClient httpClient) /// internal const string AUTH_URL = "https://api.backblazeb2.com/b2api/v1/b2_authorize_account"; - /// - /// Timeout for authentication requests - /// - private const int AUTHENTICATION_TIMEOUT_SECONDS = 10; - /// /// Maximum number of retries for authorization /// private const int MAX_AUTHORIZATION_RETRIES = 5; - /// - /// Authorization token (fetches from Config, which will refresh if needed) - /// - private string AuthorizationToken => Config.AuthorizationToken; - - /// - /// API URL (fetches from Config, which will refresh if needed) - /// - public string ApiUrl => Config.APIUrl; - - /// - /// Download URL (fetches from Config, which will refresh if needed) - /// - public string DownloadUrl => Config.DownloadUrl; - - /// - /// Account ID (fetches from Config, which will refresh if needed) - /// - public string AccountId => Config.AccountID; - /// /// Creates an HTTP request message with authorization header /// /// The URL for the request /// HTTP method (defaults to GET if null) /// Configured HttpRequestMessage - public override HttpRequestMessage CreateRequest(string url, string? method = null) + public override async Task CreateRequestAsync(string url, HttpMethod method, CancellationToken cancellationToken) { - HttpRequestMessage request = base.CreateRequest(url, method); - request.Headers.TryAddWithoutValidation("Authorization", AuthorizationToken); + var request = await base.CreateRequestAsync(url, method, cancellationToken).ConfigureAwait(false); + var config = await GetConfigAsync(cancellationToken).ConfigureAwait(false); + request.Headers.TryAddWithoutValidation("Authorization", config.AuthorizationToken); request.Headers.Add("User-Agent", UserAgent); return request; } @@ -105,88 +90,91 @@ public class B2AuthHelper(string userid, string password, HttpClient httpClient) /// Cleans the url to remove trailing slashes /// /// URL - private string DropTrailingSlashes(string url) + private static string DropTrailingSlashes(string url) { + if (string.IsNullOrWhiteSpace(url)) + return url; + while (url.EndsWith("/", StringComparison.Ordinal)) url = url.Substring(0, url.Length - 1); return url; } - /// - /// API DNS Name - /// - public string? ApiDnsName => - string.IsNullOrWhiteSpace(_CachedAuthResponse?.APIUrl) - ? null - : new Uri(_CachedAuthResponse.APIUrl).Host; - - /// - /// Download DNS Name - /// - public string? DownloadDnsName => - string.IsNullOrWhiteSpace(_CachedAuthResponse?.DownloadUrl) - ? null - : new Uri(_CachedAuthResponse.DownloadUrl).Host; - - private AuthResponse Config + public async Task GetConfigAsync(CancellationToken cancelToken) { - get + if (_CachedAuthResponse != null && _configExpires >= DateTime.UtcNow) + return _CachedAuthResponse; + + var retries = 0; + + while (true) { - if (_CachedAuthResponse != null && _configExpires >= DateTime.UtcNow) return _CachedAuthResponse; - var retries = 0; - - while (true) + HttpResponseMessage? response = null; + try { - HttpResponseMessage? response = null; - try + using var request = await base.CreateRequestAsync(AUTH_URL, HttpMethod.Get, cancelToken).ConfigureAwait(false); + + request.Headers.Add("Authorization", $"Basic {Convert.ToBase64String(Encoding.UTF8.GetBytes(userid + ":" + password))}"); + request.Headers.Add("ContentType", "application/json; charset=utf-8"); + + + var authResponse = await Utility.Utility.WithTimeout(timeouts.ShortTimeout, cancelToken, async ct => { - - using var request = base.CreateRequest(AUTH_URL); - - request.Headers.Add("Authorization", $"Basic {Convert.ToBase64String(Encoding.UTF8.GetBytes(userid + ":" + password))}"); - request.Headers.Add("ContentType", "application/json; charset=utf-8"); - - using var timeoutToken = new CancellationTokenSource(); - timeoutToken.CancelAfter(TimeSpan.FromSeconds(AUTHENTICATION_TIMEOUT_SECONDS)); - - response = _httpClient.Send(request, timeoutToken.Token); + response = await _httpClient.SendAsync(request, ct).ConfigureAwait(false); response.EnsureSuccessStatusCode(); - _CachedAuthResponse = ReadJsonResponse(response); - _CachedAuthResponse.APIUrl = DropTrailingSlashes(_CachedAuthResponse.APIUrl); - _CachedAuthResponse.DownloadUrl = DropTrailingSlashes(_CachedAuthResponse.DownloadUrl); - _configExpires = DateTime.UtcNow + TimeSpan.FromMinutes(CACHE_EXPIRATION_MINUTES); + return await ReadJsonResponseAsync(response, ct).ConfigureAwait(false) + ?? throw new Exception("Failed to parse authorization response"); + }).ConfigureAwait(false); - response.Dispose(); - return _CachedAuthResponse; - } - catch (Exception ex) + if (string.IsNullOrWhiteSpace(authResponse.AccountID)) + throw new Exception("Account ID is missing from authorization response"); + + if (string.IsNullOrWhiteSpace(authResponse.APIUrl)) + throw new Exception("API URL is missing from authorization response"); + + if (string.IsNullOrWhiteSpace(authResponse.AuthorizationToken)) + throw new Exception("Authorization token is missing from authorization response"); + + if (string.IsNullOrWhiteSpace(authResponse.DownloadUrl)) + throw new Exception("Download URL is missing from authorization response"); + + authResponse.APIUrl = DropTrailingSlashes(authResponse.APIUrl); + authResponse.DownloadUrl = DropTrailingSlashes(authResponse.DownloadUrl); + _configExpires = DateTime.UtcNow + TimeSpan.FromMinutes(CACHE_EXPIRATION_MINUTES); + + return _CachedAuthResponse = new AuthResponse(authResponse.AccountID, authResponse.APIUrl, authResponse.AuthorizationToken, authResponse.DownloadUrl); + } + catch (Exception ex) + { + var clientError = false; + + try { - var clientError = false; - - try + // Only retry once on client errors + if (ex is HttpRequestException { StatusCode: not null } exception) { - // Only retry once on client errors - if (ex is HttpRequestException { StatusCode: not null } exception) - { - var sc = (int)exception.StatusCode; - clientError = sc is >= 400 and <= 499; - } + var sc = (int)exception.StatusCode; + clientError = sc is >= 400 and <= 499; } - catch - { - // ignored - } - - if (retries >= (clientError ? 1 : MAX_AUTHORIZATION_RETRIES)) - { - AttemptParseAndThrowException(ex, response); - throw; - } - - Thread.Sleep(TimeSpan.FromSeconds(Math.Pow(2, retries))); - retries++; } + catch + { + // ignored + } + + if (retries >= (clientError ? 1 : MAX_AUTHORIZATION_RETRIES)) + { + await AttemptParseAndThrowExceptionAsync(ex, response, cancelToken); + throw; + } + + Thread.Sleep(TimeSpan.FromSeconds(Math.Pow(2, retries))); + retries++; + } + finally + { + response?.Dispose(); } } } @@ -197,7 +185,7 @@ public class B2AuthHelper(string userid, string password, HttpClient httpClient) /// Exception to be parsed /// Response context /// New detailed exception - public override void AttemptParseAndThrowException(Exception ex, HttpResponseMessage? responseContext = null) + public override async Task AttemptParseAndThrowExceptionAsync(Exception ex, HttpResponseMessage? responseContext, CancellationToken cancelToken) { if (ex is not HttpRequestException || responseContext == null) return; @@ -205,11 +193,16 @@ public class B2AuthHelper(string userid, string password, HttpClient httpClient) if (ex is HttpRequestException && responseContext.StatusCode == HttpStatusCode.TooManyRequests) throw new TooManyRequestException(responseContext.Headers.RetryAfter); - using var stream = responseContext.Content.ReadAsStream(); - using var reader = new StreamReader(stream); - var rawData = reader.ReadToEnd(); - var errorResponse = JsonConvert.DeserializeObject(rawData); - throw new UserInformationException($"Backblaze ErrorResponse: {errorResponse?.Status} - {errorResponse?.Code}: {errorResponse?.Message}", "BackblazeErrorResponse"); + var rawData = await Utility.Utility.WithTimeout(timeouts.ShortTimeout, cancelToken, ct => + { + using var stream = responseContext.Content.ReadAsStream(); + using var reader = new StreamReader(stream); + return reader.ReadToEnd(); + }).ConfigureAwait(false); + + var errorResponse = JsonConvert.DeserializeObject(rawData) + ?? new ErrorResponse { Status = (int)responseContext.StatusCode, Message = responseContext.ReasonPhrase }; + throw new UserInformationException($"Backblaze ErrorResponse: {errorResponse.Status} - {errorResponse.Code}: {errorResponse.Message}", "BackblazeErrorResponse"); } /// diff --git a/Duplicati/Library/Backend/Backblaze/Model/AuthResponse.cs b/Duplicati/Library/Backend/Backblaze/Model/ApiAuthResponse.cs similarity index 98% rename from Duplicati/Library/Backend/Backblaze/Model/AuthResponse.cs rename to Duplicati/Library/Backend/Backblaze/Model/ApiAuthResponse.cs index d5d16761d..50582ef6a 100644 --- a/Duplicati/Library/Backend/Backblaze/Model/AuthResponse.cs +++ b/Duplicati/Library/Backend/Backblaze/Model/ApiAuthResponse.cs @@ -26,7 +26,7 @@ namespace Duplicati.Library.Backend.Backblaze.Model; /// /// Model for JSON parsing of AuthResponse /// -internal class AuthResponse +internal class ApiAuthResponse { [JsonProperty("accountId")] public string? AccountID { get; set; } diff --git a/Duplicati/Library/Backend/Backblaze/Model/ListBucketsRequest.cs b/Duplicati/Library/Backend/Backblaze/Model/ListBucketsRequest.cs index 7fa3a2735..75c9d7958 100644 --- a/Duplicati/Library/Backend/Backblaze/Model/ListBucketsRequest.cs +++ b/Duplicati/Library/Backend/Backblaze/Model/ListBucketsRequest.cs @@ -31,7 +31,7 @@ internal class ListBucketsRequest : AccountIDEntity /// Initializes a new instance of the ListBucketsRequest class. /// /// The account ID for which to list buckets. - public ListBucketsRequest(string accountId) + public ListBucketsRequest(string? accountId) { AccountID = accountId; } From 53ee9526ca1b860715185a2ff35411de4492baf0 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 24 Mar 2025 17:10:15 +0100 Subject: [PATCH 42/48] Fixed some nullability warnings --- Duplicati/Library/Backend/File/Win32.cs | 43 ++++++++++---------- Duplicati/Library/Backend/Storj/StorjFile.cs | 2 +- Duplicati/Library/Utility/Utility.cs | 1 - 3 files changed, 22 insertions(+), 24 deletions(-) diff --git a/Duplicati/Library/Backend/File/Win32.cs b/Duplicati/Library/Backend/File/Win32.cs index 1feb2e4f6..e13d68a9b 100644 --- a/Duplicati/Library/Backend/File/Win32.cs +++ b/Duplicati/Library/Backend/File/Win32.cs @@ -1,25 +1,24 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System; using System.Runtime.InteropServices; using System.Runtime.Versioning; @@ -41,7 +40,7 @@ namespace Duplicati.Library.Backend public ResourceType dwType; public ResourceDisplayType dwDisplayType; public ResourceUsage dwUsage; - public string LocalName; + public string? LocalName; public string RemoteName; public string Comment; public string Provider; @@ -111,7 +110,7 @@ namespace Duplicati.Library.Backend try { - NETRESOURCE rsc = new NETRESOURCE(); + var rsc = new NETRESOURCE(); rsc.dwScope = ResourceScope.RESOURCE_GLOBALNET; rsc.dwType = ResourceType.RESOURCETYPE_DISK; diff --git a/Duplicati/Library/Backend/Storj/StorjFile.cs b/Duplicati/Library/Backend/Storj/StorjFile.cs index 6a4b18076..2e3512476 100644 --- a/Duplicati/Library/Backend/Storj/StorjFile.cs +++ b/Duplicati/Library/Backend/Storj/StorjFile.cs @@ -41,7 +41,7 @@ namespace Duplicati.Library.Backend.Storj public StorjFile() { - + Name = string.Empty; } public StorjFile(uplink.NET.Models.Object tardigradeObject) diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index 0d6fd40bd..e1a56a9fc 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -1717,7 +1717,6 @@ namespace Duplicati.Library.Utility throw new TimeoutException(); throw; } - } /// From d665828a834bdb4936ee1f4fda21fdeb32d8d30e Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 24 Mar 2025 17:14:21 +0100 Subject: [PATCH 43/48] Re-implemented box.com on new version. Removed some non-async methods on the JsonWebHelperHttpClient. --- Duplicati/Library/Backend/Box/BoxBackend.cs | 104 +++++----- .../Backend/Box/Model/CreateItemRequest.cs | 4 +- .../Backend/Box/Model/ErrorResponse.cs | 10 +- .../Library/Backend/Box/Model/FileEntity.cs | 3 +- .../Library/Backend/Box/Model/FileList.cs | 2 +- .../Library/Backend/Box/Model/FolderList.cs | 2 +- .../Library/Backend/Box/Model/IDReference.cs | 2 +- .../Backend/Box/Model/ListFolderResponse.cs | 20 +- .../Library/Backend/Box/Model/MiniFolder.cs | 8 +- .../Library/Backend/Box/Model/MiniUser.cs | 6 +- .../Library/Backend/Box/Model/OrderEntry.cs | 4 +- .../Backend/Box/Model/ShortListResponse.cs | 2 +- .../Library/Backend/Box/Model/UploadEmail.cs | 4 +- .../OAuthHelper/JSONWebHelperHttpClient.cs | 77 ++------ .../OAuthHelper/OAuthHelperHttpClient.cs | 177 +++++++++--------- 15 files changed, 197 insertions(+), 228 deletions(-) diff --git a/Duplicati/Library/Backend/Box/BoxBackend.cs b/Duplicati/Library/Backend/Box/BoxBackend.cs index ec1c21cde..5ecbe7ce8 100644 --- a/Duplicati/Library/Backend/Box/BoxBackend.cs +++ b/Duplicati/Library/Backend/Box/BoxBackend.cs @@ -19,20 +19,13 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; using System.Net; -using System.Net.Http; using System.Net.Http.Json; using System.Runtime.CompilerServices; -using System.Threading; -using System.Threading.Tasks; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; -using Duplicati.Library.Logging; using Duplicati.Library.Utility; +using Duplicati.Library.Utility.Options; using Newtonsoft.Json; using Uri = Duplicati.Library.Utility.Uri; @@ -40,8 +33,7 @@ namespace Duplicati.Library.Backend.Box { public class BoxBackend : IStreamingBackend { - private static readonly string LOGTAG = Log.LogTagFromType(); - + private static readonly string TOKEN_URL = OAuthHelper.OAUTH_LOGIN_URL("box.com"); private const string AUTHID_OPTION = "authid"; private const string REALLY_DELETE_OPTION = "box-delete-from-trash"; @@ -54,24 +46,21 @@ namespace Duplicati.Library.Backend.Box private readonly string _path; private readonly bool _deleteFromTrash; - private string _currentFolder; + private string? _currentFolder; private readonly Dictionary _fileCache = new(); + private readonly TimeoutOptionsHelper.Timeouts _timeouts; private class BoxHelper : OAuthHelperHttpClient { - public BoxHelper(string authid) + private readonly TimeoutOptionsHelper.Timeouts _timeouts; + public BoxHelper(string authid, TimeoutOptionsHelper.Timeouts timeouts) : base(authid, "box.com") { AutoAuthHeader = true; + _timeouts = timeouts; } - public override void AttemptParseAndThrowException(Exception ex, HttpResponseMessage responseContext = null) - { - AttemptParseAndThrowExceptionAsync(ex, responseContext).Await(); - } - - public override async Task AttemptParseAndThrowExceptionAsync(Exception ex, HttpResponseMessage responseContext = null, - CancellationToken cancellationToken = default) + public override async Task AttemptParseAndThrowExceptionAsync(Exception ex, HttpResponseMessage responseContext, CancellationToken cancellationToken) { if (ex is not HttpRequestException || responseContext == null) return; @@ -81,66 +70,74 @@ namespace Duplicati.Library.Backend.Box await using var stream = await responseContext.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); using var reader = new StreamReader(stream); - var rawData = await reader.ReadToEndAsync(cancellationToken).ConfigureAwait(false); - var errorResponse = JsonConvert.DeserializeObject(rawData); + var rawData = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, ct => reader.ReadToEndAsync(ct)).ConfigureAwait(false); + var errorResponse = JsonConvert.DeserializeObject(rawData) + ?? new ErrorResponse { Status = (int)responseContext.StatusCode, Code = "Unknown", Message = rawData }; throw new UserInformationException($"Box.com ErrorResponse: {errorResponse.Status} - {errorResponse.Code}: {errorResponse.Message}", "box.com"); } } public BoxBackend() { + _oAuthHelper = null!; + _path = null!; + _timeouts = null!; } - public BoxBackend(string url, Dictionary options) + public BoxBackend(string url, Dictionary options) { var uri = new Uri(url); _path = Util.AppendDirSeparator(uri.HostAndPath, "/"); - string authid = null; - if (options.TryGetValue(AUTHID_OPTION, out var option)) - authid = option; + var authid = AuthIdOptionsHelper.Parse(options) + .RequireCredentials(TOKEN_URL) + .AuthId!; _deleteFromTrash = Utility.Utility.ParseBoolOption(options, REALLY_DELETE_OPTION); + _timeouts = TimeoutOptionsHelper.Parse(options); + + _oAuthHelper = new BoxHelper(authid, _timeouts); - _oAuthHelper = new BoxHelper(authid); } private async Task GetCurrentFolderWithCacheAsync(CancellationToken cancelToken) { if (_currentFolder == null) - await GetCurrentFolderAsync(false, cancelToken).ConfigureAwait(false); + return await GetCurrentFolderAsync(false, cancelToken).ConfigureAwait(false); return _currentFolder; } - private async Task GetCurrentFolderAsync(bool create, CancellationToken cancelToken) + private async Task GetCurrentFolderAsync(bool create, CancellationToken cancelToken) { var parentid = "0"; foreach (var p in _path.Split(["/"], StringSplitOptions.RemoveEmptyEntries)) { - var el = (MiniFolder)await PagedFileListResponse(parentid, true, cancelToken).FirstOrDefaultAsync(x => x.Name == p, cancellationToken: cancelToken).ConfigureAwait(false); + var el = (MiniFolder?)await PagedFileListResponse(parentid, true, cancelToken).FirstOrDefaultAsync(x => x.Name == p, cancellationToken: cancelToken).ConfigureAwait(false); if (el == null) { if (!create) throw new FolderMissingException(); - el = await _oAuthHelper.PostAndGetJsonDataAsync( + el = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancelToken, ct => _oAuthHelper.PostAndGetJsonDataAsync( $"{BOX_API_URL}/folders", new CreateItemRequest { Name = p, Parent = new IDReference { ID = parentid } }, - cancelToken - ).ConfigureAwait(false); + ct + )).ConfigureAwait(false); } parentid = el.ID; + if (string.IsNullOrWhiteSpace(parentid)) + throw new InvalidDataException($"Invalid folder ID for {p} in {_path}"); } - _currentFolder = parentid; + return _currentFolder = parentid; } private async Task GetFileIdAsync(string name, CancellationToken cancelToken) @@ -168,13 +165,16 @@ namespace Duplicati.Library.Backend.Box do { - var resp = await _oAuthHelper.GetJsonDataAsync($"{BOX_API_URL}/folders/{parentid}/items?limit={PAGE_SIZE}&offset={offset}&fields=name,size,modified_at", cancelToken).ConfigureAwait(false); + var resp = await Utility.Utility.WithTimeout(_timeouts.ListTimeout, cancelToken, ct => _oAuthHelper.GetJsonDataAsync($"{BOX_API_URL}/folders/{parentid}/items?limit={PAGE_SIZE}&offset={offset}&fields=name,size,modified_at", ct)).ConfigureAwait(false); if (resp.Entries == null || resp.Entries.Length == 0) break; foreach (var f in resp.Entries) { + if (string.IsNullOrWhiteSpace(f.Name) || string.IsNullOrWhiteSpace(f.ID)) + continue; + if (onlyfolders && f.Type != "folder") { done = true; @@ -226,9 +226,12 @@ namespace Duplicati.Library.Backend.Box multipartForm.Add(JsonContent.Create(createreq), "attributes"); } - multipartForm.Add(new StreamContent(stream), "file", remotename); + using var timeoutStream = stream.ObserveReadTimeout(_timeouts.ReadWriteTimeout, false); + multipartForm.Add(new StreamContent(timeoutStream), "file", remotename); - var res = (await _oAuthHelper.PostMultipartAndGetJsonDataAsync(url, cancelToken, multipartForm)).Entries.First(); + var res = (await _oAuthHelper.PostMultipartAndGetJsonDataAsync(url, cancelToken, multipartForm)).Entries?.FirstOrDefault(); + if (res == null || string.IsNullOrWhiteSpace(res.ID)) + throw new InvalidDataException("No file ID returned after upload"); _fileCache[remotename] = res.ID; } catch @@ -241,11 +244,12 @@ namespace Duplicati.Library.Backend.Box public async Task GetAsync(string remotename, Stream stream, CancellationToken cancelToken) { var fileId = await GetFileIdAsync(remotename, cancelToken).ConfigureAwait(false); - using var request = _oAuthHelper.CreateRequest($"{BOX_API_URL}/files/{fileId}/content"); - using var resp = await _oAuthHelper.GetResponseAsync(request, cancelToken).ConfigureAwait(false); + using var request = await _oAuthHelper.CreateRequestAsync($"{BOX_API_URL}/files/{fileId}/content", HttpMethod.Get, cancelToken).ConfigureAwait(false); + using var resp = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancelToken, ct => _oAuthHelper.GetResponseAsync(request, HttpCompletionOption.ResponseHeadersRead, ct)).ConfigureAwait(false); resp.EnsureSuccessStatusCode(); - await using var responseStream = await resp.Content.ReadAsStreamAsync(cancelToken).ConfigureAwait(false); - await Utility.Utility.CopyStreamAsync(responseStream, stream, cancelToken).ConfigureAwait(false); + await using var responseStream = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancelToken, ct => resp.Content.ReadAsStreamAsync(ct)).ConfigureAwait(false); + using var ts = responseStream.ObserveReadTimeout(_timeouts.ReadWriteTimeout); + await Utility.Utility.CopyStreamAsync(ts, stream, cancelToken).ConfigureAwait(false); } public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) @@ -272,16 +276,15 @@ namespace Duplicati.Library.Backend.Box var fileId = await GetFileIdAsync(remotename, cancelToken).ConfigureAwait(false); try { - using (var request = _oAuthHelper.CreateRequest($"{BOX_API_URL}/files/{fileId}", "DELETE")) - using (var r = await _oAuthHelper.GetResponseAsync(request, cancelToken).ConfigureAwait(false)) + using (var request = await _oAuthHelper.CreateRequestAsync($"{BOX_API_URL}/files/{fileId}", HttpMethod.Delete, cancelToken).ConfigureAwait(false)) + using (var r = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancelToken, ct => _oAuthHelper.GetResponseAsync(request, HttpCompletionOption.ResponseContentRead, ct)).ConfigureAwait(false)) { } if (_deleteFromTrash) { - using var request = _oAuthHelper.CreateRequest($"{BOX_API_URL}/files/{fileId}/trash", "DELETE"); - using (var r = await _oAuthHelper - .GetResponseAsync(request, cancelToken).ConfigureAwait(false)) + using (var request = await _oAuthHelper.CreateRequestAsync($"{BOX_API_URL}/files/{fileId}/trash", HttpMethod.Delete, cancelToken).ConfigureAwait(false)) + using (var r = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancelToken, ct => _oAuthHelper.GetResponseAsync(request, HttpCompletionOption.ResponseContentRead, ct)).ConfigureAwait(false)) { } } @@ -306,17 +309,18 @@ namespace Duplicati.Library.Backend.Box public string ProtocolKey => "box"; public IList SupportedCommands => - new List([ - new CommandLineArgument(AUTHID_OPTION, CommandLineArgument.ArgumentType.Password, Strings.Box.AuthidShort, Strings.Box.AuthidLong(OAuthHelper.OAUTH_LOGIN_URL("box.com"))), - new CommandLineArgument(REALLY_DELETE_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.Box.ReallydeleteShort, Strings.Box.ReallydeleteLong) - ]); + [ + .. AuthIdOptionsHelper.GetOptions(TOKEN_URL), + new CommandLineArgument(REALLY_DELETE_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.Box.ReallydeleteShort, Strings.Box.ReallydeleteLong), + .. TimeoutOptionsHelper.GetOptions() + ]; public string Description => Strings.Box.Description; public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new[] { new System.Uri(BOX_API_URL).Host, new System.Uri(BOX_UPLOAD_URL).Host - }); + }.Distinct().WhereNotNullOrWhiteSpace().ToArray()); public void Dispose() { diff --git a/Duplicati/Library/Backend/Box/Model/CreateItemRequest.cs b/Duplicati/Library/Backend/Box/Model/CreateItemRequest.cs index b84ecd32b..c37a29db3 100644 --- a/Duplicati/Library/Backend/Box/Model/CreateItemRequest.cs +++ b/Duplicati/Library/Backend/Box/Model/CreateItemRequest.cs @@ -4,6 +4,6 @@ namespace Duplicati.Library.Backend.Box; public class CreateItemRequest { - [JsonProperty("name")] public string Name { get; set; } - [JsonProperty("parent")] public IDReference Parent { get; set; } + [JsonProperty("name")] public string? Name { get; set; } + [JsonProperty("parent")] public IDReference? Parent { get; set; } } \ No newline at end of file diff --git a/Duplicati/Library/Backend/Box/Model/ErrorResponse.cs b/Duplicati/Library/Backend/Box/Model/ErrorResponse.cs index 627780c8f..888974aed 100644 --- a/Duplicati/Library/Backend/Box/Model/ErrorResponse.cs +++ b/Duplicati/Library/Backend/Box/Model/ErrorResponse.cs @@ -4,10 +4,10 @@ namespace Duplicati.Library.Backend.Box; public class ErrorResponse { - [JsonProperty("type")] public string Type { get; set; } + [JsonProperty("type")] public string? Type { get; set; } [JsonProperty("status")] public int Status { get; set; } - [JsonProperty("code")] public string Code { get; set; } - [JsonProperty("help_url")] public string HelpUrl { get; set; } - [JsonProperty("message")] public string Message { get; set; } - [JsonProperty("request_id")] public string RequestId { get; set; } + [JsonProperty("code")] public string? Code { get; set; } + [JsonProperty("help_url")] public string? HelpUrl { get; set; } + [JsonProperty("message")] public string? Message { get; set; } + [JsonProperty("request_id")] public string? RequestId { get; set; } } \ No newline at end of file diff --git a/Duplicati/Library/Backend/Box/Model/FileEntity.cs b/Duplicati/Library/Backend/Box/Model/FileEntity.cs index 55e222724..c55b2b15a 100644 --- a/Duplicati/Library/Backend/Box/Model/FileEntity.cs +++ b/Duplicati/Library/Backend/Box/Model/FileEntity.cs @@ -1,4 +1,3 @@ -using System; using Newtonsoft.Json; namespace Duplicati.Library.Backend.Box; @@ -10,7 +9,7 @@ public class FileEntity : MiniFolder Size = -1; } - [JsonProperty("sha1")] public string SHA1 { get; set; } + [JsonProperty("sha1")] public string? SHA1 { get; set; } [JsonProperty("size", NullValueHandling = NullValueHandling.Ignore)] public long Size { get; set; } diff --git a/Duplicati/Library/Backend/Box/Model/FileList.cs b/Duplicati/Library/Backend/Box/Model/FileList.cs index 999aab376..5fb514cc9 100644 --- a/Duplicati/Library/Backend/Box/Model/FileList.cs +++ b/Duplicati/Library/Backend/Box/Model/FileList.cs @@ -5,7 +5,7 @@ namespace Duplicati.Library.Backend.Box; public class FileList { [JsonProperty("total_count")] public long TotalCount { get; set; } - [JsonProperty("entries")] public FileEntity[] Entries { get; set; } + [JsonProperty("entries")] public FileEntity[]? Entries { get; set; } [JsonProperty("offset")] public long Offset { get; set; } [JsonProperty("limit")] public long Limit { get; set; } } \ No newline at end of file diff --git a/Duplicati/Library/Backend/Box/Model/FolderList.cs b/Duplicati/Library/Backend/Box/Model/FolderList.cs index 028188322..778c7091c 100644 --- a/Duplicati/Library/Backend/Box/Model/FolderList.cs +++ b/Duplicati/Library/Backend/Box/Model/FolderList.cs @@ -5,5 +5,5 @@ namespace Duplicati.Library.Backend.Box; public class FolderList { [JsonProperty("total_count")] public long TotalCount { get; set; } - [JsonProperty("entries")] public MiniFolder[] Entries { get; set; } + [JsonProperty("entries")] public MiniFolder[]? Entries { get; set; } } \ No newline at end of file diff --git a/Duplicati/Library/Backend/Box/Model/IDReference.cs b/Duplicati/Library/Backend/Box/Model/IDReference.cs index 29ba6cf14..f0e0410c3 100644 --- a/Duplicati/Library/Backend/Box/Model/IDReference.cs +++ b/Duplicati/Library/Backend/Box/Model/IDReference.cs @@ -4,5 +4,5 @@ namespace Duplicati.Library.Backend.Box; public class IDReference { - [JsonProperty("id")] public string ID { get; set; } + [JsonProperty("id")] public string? ID { get; set; } } \ No newline at end of file diff --git a/Duplicati/Library/Backend/Box/Model/ListFolderResponse.cs b/Duplicati/Library/Backend/Box/Model/ListFolderResponse.cs index 395bdaaba..6231d2e17 100644 --- a/Duplicati/Library/Backend/Box/Model/ListFolderResponse.cs +++ b/Duplicati/Library/Backend/Box/Model/ListFolderResponse.cs @@ -7,15 +7,15 @@ public class ListFolderResponse : MiniFolder { [JsonProperty("created_at")] public DateTime CreatedAt { get; set; } [JsonProperty("modified_at")] public DateTime ModifiedAt { get; set; } - [JsonProperty("description")] public string Description { get; set; } + [JsonProperty("description")] public string? Description { get; set; } [JsonProperty("size")] public long Size { get; set; } - [JsonProperty("path_collection")] public FolderList PathCollection { get; set; } - [JsonProperty("created_by")] public MiniUser CreatedBy { get; set; } - [JsonProperty("modified_by")] public MiniUser ModifiedBy { get; set; } - [JsonProperty("owned_by")] public MiniUser OwnedBy { get; set; } - [JsonProperty("shared_link")] public MiniUser SharedLink { get; set; } - [JsonProperty("folder_upload_email")] public UploadEmail FolderUploadEmail { get; set; } - [JsonProperty("parent")] public MiniFolder Parent { get; set; } - [JsonProperty("item_status")] public string ItemStatus { get; set; } - [JsonProperty("item_collection")] public FileList ItemCollection { get; set; } + [JsonProperty("path_collection")] public FolderList? PathCollection { get; set; } + [JsonProperty("created_by")] public MiniUser? CreatedBy { get; set; } + [JsonProperty("modified_by")] public MiniUser? ModifiedBy { get; set; } + [JsonProperty("owned_by")] public MiniUser? OwnedBy { get; set; } + [JsonProperty("shared_link")] public MiniUser? SharedLink { get; set; } + [JsonProperty("folder_upload_email")] public UploadEmail? FolderUploadEmail { get; set; } + [JsonProperty("parent")] public MiniFolder? Parent { get; set; } + [JsonProperty("item_status")] public string? ItemStatus { get; set; } + [JsonProperty("item_collection")] public FileList? ItemCollection { get; set; } } \ No newline at end of file diff --git a/Duplicati/Library/Backend/Box/Model/MiniFolder.cs b/Duplicati/Library/Backend/Box/Model/MiniFolder.cs index 26363b542..5d5f41029 100644 --- a/Duplicati/Library/Backend/Box/Model/MiniFolder.cs +++ b/Duplicati/Library/Backend/Box/Model/MiniFolder.cs @@ -4,8 +4,8 @@ namespace Duplicati.Library.Backend.Box; public class MiniFolder : IDReference { - [JsonProperty("type")] public string Type { get; set; } - [JsonProperty("name")] public string Name { get; set; } - [JsonProperty("etag")] public string ETag { get; set; } - [JsonProperty("sequence_id")] public string SequenceID { get; set; } + [JsonProperty("type")] public string? Type { get; set; } + [JsonProperty("name")] public string? Name { get; set; } + [JsonProperty("etag")] public string? ETag { get; set; } + [JsonProperty("sequence_id")] public string? SequenceID { get; set; } } \ No newline at end of file diff --git a/Duplicati/Library/Backend/Box/Model/MiniUser.cs b/Duplicati/Library/Backend/Box/Model/MiniUser.cs index cc3af66ef..0f96851fc 100644 --- a/Duplicati/Library/Backend/Box/Model/MiniUser.cs +++ b/Duplicati/Library/Backend/Box/Model/MiniUser.cs @@ -4,7 +4,7 @@ namespace Duplicati.Library.Backend.Box; public class MiniUser : IDReference { - [JsonProperty("type")] public string Type { get; set; } - [JsonProperty("name")] public string Name { get; set; } - [JsonProperty("login")] public string Login { get; set; } + [JsonProperty("type")] public string? Type { get; set; } + [JsonProperty("name")] public string? Name { get; set; } + [JsonProperty("login")] public string? Login { get; set; } } \ No newline at end of file diff --git a/Duplicati/Library/Backend/Box/Model/OrderEntry.cs b/Duplicati/Library/Backend/Box/Model/OrderEntry.cs index 5a573fb9c..16e778ce5 100644 --- a/Duplicati/Library/Backend/Box/Model/OrderEntry.cs +++ b/Duplicati/Library/Backend/Box/Model/OrderEntry.cs @@ -4,6 +4,6 @@ namespace Duplicati.Library.Backend.Box; public class OrderEntry { - [JsonProperty("by")] public string By { get; set; } - [JsonProperty("direction")] public string Direction { get; set; } + [JsonProperty("by")] public string? By { get; set; } + [JsonProperty("direction")] public string? Direction { get; set; } } \ No newline at end of file diff --git a/Duplicati/Library/Backend/Box/Model/ShortListResponse.cs b/Duplicati/Library/Backend/Box/Model/ShortListResponse.cs index 4f8db34fd..9e67c8bc5 100644 --- a/Duplicati/Library/Backend/Box/Model/ShortListResponse.cs +++ b/Duplicati/Library/Backend/Box/Model/ShortListResponse.cs @@ -4,5 +4,5 @@ namespace Duplicati.Library.Backend.Box; public class ShortListResponse : FileList { - [JsonProperty("order")] public OrderEntry[] Order { get; set; } + [JsonProperty("order")] public OrderEntry[]? Order { get; set; } } \ No newline at end of file diff --git a/Duplicati/Library/Backend/Box/Model/UploadEmail.cs b/Duplicati/Library/Backend/Box/Model/UploadEmail.cs index de24408d9..2cf749f09 100644 --- a/Duplicati/Library/Backend/Box/Model/UploadEmail.cs +++ b/Duplicati/Library/Backend/Box/Model/UploadEmail.cs @@ -4,6 +4,6 @@ namespace Duplicati.Library.Backend.Box; public class UploadEmail { - [JsonProperty("access")] public string Access { get; set; } - [JsonProperty("email")] public string Email { get; set; } + [JsonProperty("access")] public string? Access { get; set; } + [JsonProperty("email")] public string? Email { get; set; } } \ No newline at end of file diff --git a/Duplicati/Library/Backend/OAuthHelper/JSONWebHelperHttpClient.cs b/Duplicati/Library/Backend/OAuthHelper/JSONWebHelperHttpClient.cs index 214a6122d..325bcbcb3 100644 --- a/Duplicati/Library/Backend/OAuthHelper/JSONWebHelperHttpClient.cs +++ b/Duplicati/Library/Backend/OAuthHelper/JSONWebHelperHttpClient.cs @@ -46,23 +46,20 @@ public class JsonWebHelperHttpClient(HttpClient httpClient) /// Useragent string building method /// protected string UserAgent => $"Duplicati v{System.Reflection.Assembly.GetExecutingAssembly().GetName().Version}"; - public event Action CreateSetupHelper; /// /// Centralized method to prepare a request with the given URL and method setting useragent /// /// Url /// Method - public virtual HttpRequestMessage CreateRequest(string url, string method = null) + public virtual Task CreateRequestAsync(string url, HttpMethod method, CancellationToken cancelToken) { - HttpRequestMessage request = new HttpRequestMessage(string.IsNullOrEmpty(method) ? HttpMethod.Get : new HttpMethod(method), url); + var request = new HttpRequestMessage(method, url); request.Headers.Add("User-Agent", UserAgent); - CreateSetupHelper?.Invoke(request); - - return request; + return Task.FromResult(request); } - + /// /// Performs a multipart post and parses the response as JSON /// @@ -71,13 +68,13 @@ public class JsonWebHelperHttpClient(HttpClient httpClient) /// Token to cancel the operation. /// The multipart items. /// The return type parameter. - public virtual async Task PostMultipartAndGetJsonDataAsync(string url, CancellationToken cancellationToken, MultipartContent parts) + public virtual async Task PostMultipartAndGetJsonDataAsync(string url, CancellationToken cancellationToken, MultipartContent parts) { using var response = await PostMultipartAsync(url, cancellationToken, parts).ConfigureAwait(false); - return ReadJsonResponse(response); + return await ReadJsonResponseAsync(response, cancellationToken).ConfigureAwait(false); } - + /// /// Performs a multipart post /// @@ -87,11 +84,11 @@ public class JsonWebHelperHttpClient(HttpClient httpClient) /// The multipart items. protected virtual async Task PostMultipartAsync(string url, CancellationToken cancellationToken, MultipartContent parts) { - using var req = CreateRequest(url, "POST"); + using var req = await CreateRequestAsync(url, HttpMethod.Post, cancellationToken).ConfigureAwait(false); req.Content = parts; return await _httpClient.SendAsync(req, cancellationToken).ConfigureAwait(false); } - + /// /// Execute Get request and return response and deserializes JSON response into the given type /// @@ -101,7 +98,7 @@ public class JsonWebHelperHttpClient(HttpClient httpClient) /// Destination Type protected virtual async Task GetJsonDataAsync(string url, CancellationToken cancellationToken, Action setup = null) { - using var req = CreateRequest(url); + using var req = await CreateRequestAsync(url, HttpMethod.Get, cancellationToken).ConfigureAwait(false); if (setup != null) setup(req); @@ -132,7 +129,7 @@ public class JsonWebHelperHttpClient(HttpClient httpClient) } ).ConfigureAwait(false); } - + /// /// Executes a web request and json-deserializes the results as the specified type /// @@ -142,7 +139,7 @@ public class JsonWebHelperHttpClient(HttpClient httpClient) /// The type of data to return. public virtual async Task GetJsonDataAsync(string url, CancellationToken cancellationToken) { - + return await GetJsonDataAsync( url, cancellationToken, @@ -161,37 +158,10 @@ public class JsonWebHelperHttpClient(HttpClient httpClient) /// Destination Type protected virtual async Task ReadJsonResponseAsync(HttpRequestMessage req, CancellationToken cancellationToken) { - using var resp = await GetResponseAsync(req, cancellationToken).ConfigureAwait(false); + using var resp = await GetResponseAsync(req, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); return await ReadJsonResponseAsync(resp, cancellationToken).ConfigureAwait(false); } - /// - /// Read the JSON response from the server and deserialize it into the given type - /// - /// Response object - /// Type to cast to - /// - /// Exception when failing to deserialize the JSON to the Type - protected virtual T ReadJsonResponse(HttpResponseMessage response) - { - using var rs = response.Content.ReadAsStream(); - using var ps = new StreamPeekReader(rs); - try - { - using var tr = new StreamReader(ps); - using var jr = new JsonTextReader(tr); - return new JsonSerializer().Deserialize(jr); - } - catch (Exception ex) - { - // If we get invalid JSON, report the peek value - if (ex is JsonReaderException) - throw new IOException($"Invalid JSON data: \"{ps.PeekData()}\"", ex); - // Otherwise, we have no additional help to offer - throw; - } - } - /// /// Read the JSON response from the server and deserialize it into the given type asynchronously /// @@ -224,18 +194,9 @@ public class JsonWebHelperHttpClient(HttpClient httpClient) /// Use this method to register an exception handler, /// which can throw another, more meaningful exception /// - public virtual async Task AttemptParseAndThrowExceptionAsync(Exception ex, HttpResponseMessage responseContext = null, CancellationToken cancellationToken = default) + public virtual Task AttemptParseAndThrowExceptionAsync(Exception ex, HttpResponseMessage responseContext, CancellationToken cancellationToken) { - - } - - /// - /// Use this method to register an exception handler, - /// which can throw another, more meaningful exception - /// - public virtual void AttemptParseAndThrowException(Exception ex, HttpResponseMessage responseContext = null) - { - + return Task.CompletedTask; } /// @@ -244,19 +205,19 @@ public class JsonWebHelperHttpClient(HttpClient httpClient) /// Request object /// Cancellation Token /// - public async Task GetResponseAsync(HttpRequestMessage req, CancellationToken cancellationToken) + public async Task GetResponseAsync(HttpRequestMessage req, HttpCompletionOption httpCompletionOption, CancellationToken cancellationToken) { HttpResponseMessage response = null; try { - response = await _httpClient.SendAsync(req, cancellationToken).ConfigureAwait(false); - + response = await _httpClient.SendAsync(req, httpCompletionOption, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); return response; } catch (Exception ex) { - AttemptParseAndThrowException(ex, response); + await AttemptParseAndThrowExceptionAsync(ex, response, cancellationToken).ConfigureAwait(false); throw; } } diff --git a/Duplicati/Library/Backend/OAuthHelper/OAuthHelperHttpClient.cs b/Duplicati/Library/Backend/OAuthHelper/OAuthHelperHttpClient.cs index 8f35628f0..622c819ed 100644 --- a/Duplicati/Library/Backend/OAuthHelper/OAuthHelperHttpClient.cs +++ b/Duplicati/Library/Backend/OAuthHelper/OAuthHelperHttpClient.cs @@ -36,14 +36,22 @@ public class OAuthHelperHttpClient : JsonWebHelperHttpClient private string _Authid; private DateTime _mTokenExpires = DateTime.UtcNow; private string OAuthLoginUrl { get; } - + /// /// Timeout for authentication requests /// - private const int AUTHENTICATION_TIMEOUT_SECONDS = 25; + private static readonly TimeSpan AUTHENTICATION_TIMEOUT = TimeSpan.FromSeconds(25); + /// + /// Maximum number of retries for authorization + /// private const int MAX_AUTHORIZATION_RETRIES = 5; + /// + /// URL to use for OAuth login + /// + /// The name of the module + /// The URL to use for OAuth login private static string OAUTH_LOGIN_URL(string modulename) { var u = new Utility.Uri(OAuthContextSettings.ServerURL); @@ -84,108 +92,105 @@ public class OAuthHelperHttpClient : JsonWebHelperHttpClient Strings.OAuthHelper.MissingAuthID(OAuthLoginUrl), "MissingAuthID"); } - private HttpRequestMessage CreateRequest(string url, string method, bool noAuthorization, CancellationToken cancellationToken = default) + private async Task CreateRequestAsync(string url, HttpMethod method, bool noAuthorization, CancellationToken cancellationToken) { - var request = new HttpRequestMessage(string.IsNullOrEmpty(method) ? HttpMethod.Get : new HttpMethod(method), url); + var request = new HttpRequestMessage(method, url); request.Headers.Add("User-Agent", UserAgent); - if (!noAuthorization && AutoAuthHeader && !string.Equals(OAuthContextSettings.ServerURL, url)) request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", - GetAccessTokenAsync(cancellationToken).Await()); + if (!noAuthorization && AutoAuthHeader && !string.Equals(OAuthContextSettings.ServerURL, url)) request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", + await GetAccessTokenAsync(cancellationToken).ConfigureAwait(false)); return request; } - public override HttpRequestMessage CreateRequest(string url, string method = null) - { - return CreateRequest(url, method, false); - } - - private async Task GetAccessTokenAsync(CancellationToken cancellationToken) - { - if (AccessTokenOnly) - return _Authid; + public override Task CreateRequestAsync(string url, HttpMethod method, CancellationToken cancellationToken) + => CreateRequestAsync(url, method, false, cancellationToken); - if (_Token == null || _mTokenExpires < DateTime.UtcNow) + private async Task GetAccessTokenAsync(CancellationToken cancellationToken) + { + if (AccessTokenOnly) + return _Authid; + + if (_Token == null || _mTokenExpires < DateTime.UtcNow) + { + var retries = 0; + + while (true) { - var retries = 0; - - while (true) + HttpResponseMessage response = null; + try { - HttpResponseMessage response = null; - try - { - using var timeoutToken = new CancellationTokenSource(); - timeoutToken.CancelAfter(TimeSpan.FromSeconds(AUTHENTICATION_TIMEOUT_SECONDS)); - using var combinedCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken); - - using var request = CreateRequest(OAuthContextSettings.ServerURL, "GET", false, combinedCancellationToken.Token); + using var request = await CreateRequestAsync(OAuthContextSettings.ServerURL, HttpMethod.Get, false, cancellationToken).ConfigureAwait(false); + return await Utility.Utility.WithTimeout(AUTHENTICATION_TIMEOUT, cancellationToken, async ct => + { request.Headers.TryAddWithoutValidation("X-AuthID", _Authid); - response = await _httpClient.SendAsync(request, combinedCancellationToken.Token).ConfigureAwait(false); + response = await _httpClient.SendAsync(request, ct).ConfigureAwait(false); response.EnsureSuccessStatusCode(); - var res = await ReadJsonResponseAsync(response, combinedCancellationToken.Token).ConfigureAwait(false); + var res = await ReadJsonResponseAsync(response, ct).ConfigureAwait(false); _mTokenExpires = DateTime.UtcNow.AddSeconds(res.expires - 30); if (AutoV2 && !string.IsNullOrWhiteSpace(res.v2_authid)) _Authid = res.v2_authid; return _Token = res.access_token; - } - catch (Exception ex) - { - var clientError = false; - - try - { - // Only retry once on client errors - if (ex is HttpRequestException { StatusCode: not null } exception) - { - var sc = (int)exception.StatusCode; - clientError = sc is >= 400 and <= 499; - } - } - catch - { - // ignored - } - - if (response != null && response.Headers.Contains("X-Reason")) - { - var msg = response.Headers.GetValues("X-Reason").FirstOrDefault(); - - if (string.IsNullOrWhiteSpace(msg)) - msg = response.StatusCode.ToString(); - - if (response.StatusCode == HttpStatusCode.ServiceUnavailable) - { - string errorKey = msg == response.StatusCode.ToString() - ? "OAuthOverQuotaError" - : "OAuthLoginError"; - - string errorMessage = errorKey == "OAuthOverQuotaError" - ? Strings.OAuthHelper.OverQuotaError - : Strings.OAuthHelper.AuthorizationFailure(msg, OAuthLoginUrl); - - throw new Interface.UserInformationException(errorMessage, errorKey, - errorKey == "OAuthLoginError" ? ex : null); - } - } - - if (retries >= (clientError ? 1 : MAX_AUTHORIZATION_RETRIES)) - { - await AttemptParseAndThrowExceptionAsync(ex, response, cancellationToken).ConfigureAwait(false); - throw; - } - - Thread.Sleep(TimeSpan.FromSeconds(Math.Pow(2, retries))); - retries++; - } - finally - { - response?.Dispose(); - } - + }).ConfigureAwait(false); } - } + catch (Exception ex) + { + var clientError = false; - return _Token; + try + { + // Only retry once on client errors + if (ex is HttpRequestException { StatusCode: not null } exception) + { + var sc = (int)exception.StatusCode; + clientError = sc is >= 400 and <= 499; + } + } + catch + { + // ignored + } + + if (response != null && response.Headers.Contains("X-Reason")) + { + var msg = response.Headers.GetValues("X-Reason").FirstOrDefault(); + + if (string.IsNullOrWhiteSpace(msg)) + msg = response.StatusCode.ToString(); + + if (response.StatusCode == HttpStatusCode.ServiceUnavailable) + { + string errorKey = msg == response.StatusCode.ToString() + ? "OAuthOverQuotaError" + : "OAuthLoginError"; + + string errorMessage = errorKey == "OAuthOverQuotaError" + ? Strings.OAuthHelper.OverQuotaError + : Strings.OAuthHelper.AuthorizationFailure(msg, OAuthLoginUrl); + + throw new Interface.UserInformationException(errorMessage, errorKey, + errorKey == "OAuthLoginError" ? ex : null); + } + } + + if (retries >= (clientError ? 1 : MAX_AUTHORIZATION_RETRIES)) + { + await AttemptParseAndThrowExceptionAsync(ex, response, cancellationToken).ConfigureAwait(false); + throw; + } + + Thread.Sleep(TimeSpan.FromSeconds(Math.Pow(2, retries))); + retries++; + } + finally + { + response?.Dispose(); + } + + } + } + + return _Token; } } \ No newline at end of file From 4036d185665df2f4a1fb003d732fa19a1adee2ce Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 24 Mar 2025 17:18:41 +0100 Subject: [PATCH 44/48] Fixed non-translateable strings --- Duplicati/Library/Modules/Builtin/Strings.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Duplicati/Library/Modules/Builtin/Strings.cs b/Duplicati/Library/Modules/Builtin/Strings.cs index eec8f6f92..b83acc326 100644 --- a/Duplicati/Library/Modules/Builtin/Strings.cs +++ b/Duplicati/Library/Modules/Builtin/Strings.cs @@ -136,7 +136,7 @@ To enable SMTP over SSL, use the format smtps://example.com. To enable SMTP STAR public static string SendMailLog(string message) { return LC.L(@"Whole SMTP communication: {0}", message); } public static string SendMailFailedRetryError(string failedserver, string message, string retryserver) { return LC.L(@"Failed to send email with server: {0}, message: {1}, retrying with {2}", failedserver, message, retryserver); } public static string SendMailSuccess(string server) { return LC.L(@"Email sent successfully using server: {0}", server); } - public static string SendmailextraparametersLong { get { return LC.L(@"Use this option to set extra parameters for the message body. This parameter can either be a querystring (e.g. ""parameter1=value1¶meter2=value2"") or a JSON key/value object (e.g. ""{\""parameter1\"": \""value1\"", \""parameter2\"": \""value2\""}"")."); } } + public static string SendmailextraparametersLong { get { return LC.L(@"Use this option to set extra parameters for the message body. This parameter can either be a querystring (e.g. 'parameter1=value1¶meter2=value2') or a JSON key/value object (e.g. '{""parameter1"": ""value1"", ""parameter2"": ""value2""}')."); } } public static string SendmailextraparametersShort { get { return LC.L(@"Extra parameters for the message sent"); } } } internal static class SendJabberMessage @@ -165,7 +165,7 @@ You can supply multiple options with a comma separator, e.g. ""{0},{1}"". The sp public static string SendxmppanyoperationLong { get { return LC.L(@"By default, messages will only be sent after a backup operation. Use this option to send messages for all operations."); } } public static string SendxmppanyoperationShort { get { return LC.L(@"Send messages for all operations"); } } public static string LoginTimeoutError { get { return LC.L(@"Timeout occurred while logging in to jabber server"); } } - public static string SendxmppextraparametersLong { get { return LC.L(@"Use this option to set extra parameters for the message body. This parameter can either be a querystring (e.g. ""parameter1=value1¶meter2=value2"") or a JSON key/value object (e.g. ""{\""parameter1\"": \""value1\"", \""parameter2\"": \""value2\""}"")."); } } + public static string SendxmppextraparametersLong { get { return LC.L(@"Use this option to set extra parameters for the message body. This parameter can either be a querystring (e.g. 'parameter1=value1¶meter2=value2') or a JSON key/value object (e.g. '{""parameter1"": ""value1"", ""parameter2"": ""value2""}')."); } } public static string SendxmppextraparametersShort { get { return LC.L(@"Extra parameters for the message sent"); } } } @@ -198,7 +198,7 @@ You can supply multiple options with a comma separator, e.g. ""{0},{1}"". The sp public static string SendTelegramLevelShort => LC.L(@"The messages to send"); public static string SendTelegramManyOperationLong => LC.L(@"By default, messages will only be sent after a backup operation. Use this option to send messages for all operations."); public static string SendTelegramManyOperationShort => LC.L(@"Send messages for all operations"); - public static string SendTelegramExtraParametersLong { get { return LC.L(@"Use this option to set extra parameters for the message body. This parameter can either be a querystring (e.g. ""parameter1=value1¶meter2=value2"") or a JSON key/value object (e.g. ""{\""parameter1\"": \""value1\"", \""parameter2\"": \""value2\""}"")."); } } + public static string SendTelegramExtraParametersLong { get { return LC.L(@"Use this option to set extra parameters for the message body. This parameter can either be a querystring (e.g. 'parameter1=value1¶meter2=value2') or a JSON key/value object (e.g. '{""parameter1"": ""value1"", ""parameter2"": ""value2""}')."); } } public static string SendTelegramExtraParametersShort { get { return LC.L(@"Extra parameters for the message sent"); } } } @@ -220,7 +220,7 @@ All command line options are also reported within %value%, e.g. %volsize%. Any u public static string SendhttpmessageShort { get { return LC.L(@"The message template"); } } public static string SendhttpmessageparameternameLong { get { return LC.L(@"Use this option to set a name of the parameter to send the message as."); } } public static string SendhttpmessageparameternameShort { get { return LC.L(@"The name of the parameter to send the message as"); } } - public static string SendhttpextraparametersLong { get { return LC.L(@"Use this option to set extra parameters for the message body. This parameter can either be a querystring (e.g. ""parameter1=value1¶meter2=value2"") or a JSON key/value object (e.g. ""{\""parameter1\"": \""value1\"", \""parameter2\"": \""value2\""}"")."); } } + public static string SendhttpextraparametersLong { get { return LC.L(@"Use this option to set extra parameters for the message body. This parameter can either be a querystring (e.g. 'parameter1=value1¶meter2=value2') or a JSON key/value object (e.g. '{""parameter1"": ""value1"", ""parameter2"": ""value2""}')."); } } public static string SendhttpextraparametersShort { get { return LC.L(@"Extra parameters for the message sent"); } } public static string SendhttplevelLong(string success, string warning, string error, string fatal, string all) { return LC.L(@"You can specify one of ""{0}"", ""{1}"", ""{2}"", ""{3}"". You can supply multiple options with a comma separator, e.g. ""{0},{1}"". The special value ""{4}"" is a shorthand for ""{0},{1},{2},{3}"" and will cause all backup operations to send a message.", success, warning, error, fatal, all); } From 6d2137d9e4ed7ac9071eca635fac68d3a50e5134 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 24 Mar 2025 17:23:25 +0100 Subject: [PATCH 45/48] Removed JSON example as it trips up the localization on Linux --- Duplicati/Library/Modules/Builtin/Strings.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Duplicati/Library/Modules/Builtin/Strings.cs b/Duplicati/Library/Modules/Builtin/Strings.cs index b83acc326..b89368802 100644 --- a/Duplicati/Library/Modules/Builtin/Strings.cs +++ b/Duplicati/Library/Modules/Builtin/Strings.cs @@ -136,7 +136,7 @@ To enable SMTP over SSL, use the format smtps://example.com. To enable SMTP STAR public static string SendMailLog(string message) { return LC.L(@"Whole SMTP communication: {0}", message); } public static string SendMailFailedRetryError(string failedserver, string message, string retryserver) { return LC.L(@"Failed to send email with server: {0}, message: {1}, retrying with {2}", failedserver, message, retryserver); } public static string SendMailSuccess(string server) { return LC.L(@"Email sent successfully using server: {0}", server); } - public static string SendmailextraparametersLong { get { return LC.L(@"Use this option to set extra parameters for the message body. This parameter can either be a querystring (e.g. 'parameter1=value1¶meter2=value2') or a JSON key/value object (e.g. '{""parameter1"": ""value1"", ""parameter2"": ""value2""}')."); } } + public static string SendmailextraparametersLong { get { return LC.L(@"Use this option to set extra parameters for the message body. This parameter can either be a querystring (e.g. 'parameter1=value1¶meter2=value2') or a JSON key/value object."); } } public static string SendmailextraparametersShort { get { return LC.L(@"Extra parameters for the message sent"); } } } internal static class SendJabberMessage @@ -165,7 +165,7 @@ You can supply multiple options with a comma separator, e.g. ""{0},{1}"". The sp public static string SendxmppanyoperationLong { get { return LC.L(@"By default, messages will only be sent after a backup operation. Use this option to send messages for all operations."); } } public static string SendxmppanyoperationShort { get { return LC.L(@"Send messages for all operations"); } } public static string LoginTimeoutError { get { return LC.L(@"Timeout occurred while logging in to jabber server"); } } - public static string SendxmppextraparametersLong { get { return LC.L(@"Use this option to set extra parameters for the message body. This parameter can either be a querystring (e.g. 'parameter1=value1¶meter2=value2') or a JSON key/value object (e.g. '{""parameter1"": ""value1"", ""parameter2"": ""value2""}')."); } } + public static string SendxmppextraparametersLong { get { return LC.L(@"Use this option to set extra parameters for the message body. This parameter can either be a querystring (e.g. 'parameter1=value1¶meter2=value2') or a JSON key/value object."); } } public static string SendxmppextraparametersShort { get { return LC.L(@"Extra parameters for the message sent"); } } } @@ -198,7 +198,7 @@ You can supply multiple options with a comma separator, e.g. ""{0},{1}"". The sp public static string SendTelegramLevelShort => LC.L(@"The messages to send"); public static string SendTelegramManyOperationLong => LC.L(@"By default, messages will only be sent after a backup operation. Use this option to send messages for all operations."); public static string SendTelegramManyOperationShort => LC.L(@"Send messages for all operations"); - public static string SendTelegramExtraParametersLong { get { return LC.L(@"Use this option to set extra parameters for the message body. This parameter can either be a querystring (e.g. 'parameter1=value1¶meter2=value2') or a JSON key/value object (e.g. '{""parameter1"": ""value1"", ""parameter2"": ""value2""}')."); } } + public static string SendTelegramExtraParametersLong { get { return LC.L(@"Use this option to set extra parameters for the message body. This parameter can either be a querystring (e.g. 'parameter1=value1¶meter2=value2') or a JSON key/value object."); } } public static string SendTelegramExtraParametersShort { get { return LC.L(@"Extra parameters for the message sent"); } } } @@ -220,7 +220,7 @@ All command line options are also reported within %value%, e.g. %volsize%. Any u public static string SendhttpmessageShort { get { return LC.L(@"The message template"); } } public static string SendhttpmessageparameternameLong { get { return LC.L(@"Use this option to set a name of the parameter to send the message as."); } } public static string SendhttpmessageparameternameShort { get { return LC.L(@"The name of the parameter to send the message as"); } } - public static string SendhttpextraparametersLong { get { return LC.L(@"Use this option to set extra parameters for the message body. This parameter can either be a querystring (e.g. 'parameter1=value1¶meter2=value2') or a JSON key/value object (e.g. '{""parameter1"": ""value1"", ""parameter2"": ""value2""}')."); } } + public static string SendhttpextraparametersLong { get { return LC.L(@"Use this option to set extra parameters for the message body. This parameter can either be a querystring (e.g. 'parameter1=value1¶meter2=value2') or a JSON key/value object."); } } public static string SendhttpextraparametersShort { get { return LC.L(@"Extra parameters for the message sent"); } } public static string SendhttplevelLong(string success, string warning, string error, string fatal, string all) { return LC.L(@"You can specify one of ""{0}"", ""{1}"", ""{2}"", ""{3}"". You can supply multiple options with a comma separator, e.g. ""{0},{1}"". The special value ""{4}"" is a shorthand for ""{0},{1},{2},{3}"" and will cause all backup operations to send a message.", success, warning, error, fatal, all); } From a3483243cb16cf92888584a6e41dc5c55d93835a Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 24 Mar 2025 21:39:54 +0100 Subject: [PATCH 46/48] Support password-less WebDAV --- Duplicati/Library/Backend/WEBDAV/Strings.cs | 2 +- Duplicati/Library/Backend/WEBDAV/WEBDAV.cs | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Duplicati/Library/Backend/WEBDAV/Strings.cs b/Duplicati/Library/Backend/WEBDAV/Strings.cs index abacd8593..d7e98ae5e 100644 --- a/Duplicati/Library/Backend/WEBDAV/Strings.cs +++ b/Duplicati/Library/Backend/WEBDAV/Strings.cs @@ -36,6 +36,6 @@ This can be because the file is deleted or unavailable, but it can also be becau Error message: {3}", foldername, filename, extension, errormessage); } public static string DescriptionDebugPropfindLong { get { return LC.L(@"To aid in debugging issues, it is possible to set a path to a file that will be overwritten with the PROPFIND response."); } } public static string DescriptionDebugPropfindShort { get { return LC.L(@"Dump the PROPFIND response"); } } - public static string UsernameAndPasswordRequired { get { return LC.L(@"Non-integrated authentication requires both a username and a password"); } } + public static string UsernameRequired { get { return LC.L(@"Digest authentication requires a username to be set"); } } } } diff --git a/Duplicati/Library/Backend/WEBDAV/WEBDAV.cs b/Duplicati/Library/Backend/WEBDAV/WEBDAV.cs index e0006bca8..627a01890 100644 --- a/Duplicati/Library/Backend/WEBDAV/WEBDAV.cs +++ b/Duplicati/Library/Backend/WEBDAV/WEBDAV.cs @@ -137,8 +137,9 @@ namespace Duplicati.Library.Backend m_useIntegratedAuthentication = Utility.Utility.ParseBoolOption(options, "integrated-authentication"); m_forceDigestAuthentication = Utility.Utility.ParseBoolOption(options, "force-digest-authentication"); - if (!m_useIntegratedAuthentication && m_userInfo == null) - throw new UserInformationException(Strings.WEBDAV.UsernameAndPasswordRequired, "UsernameAndPasswordRequired"); + // Explicitly support setups with no username and password + if (m_forceDigestAuthentication && m_userInfo == null) + throw new UserInformationException(Strings.WEBDAV.UsernameRequired, "UsernameRequired"); m_url = u.SetScheme(m_certificateOptions.UseSSL ? "https" : "http").SetCredentials(null, null).SetQuery(null).ToString(); m_url = Util.AppendDirSeparator(m_url, "/"); @@ -185,14 +186,13 @@ namespace Duplicati.Library.Backend m_httpClient = HttpClientHelper.CreateClient(httpHandler); m_httpClient.Timeout = Timeout.InfiniteTimeSpan; - if (!m_useIntegratedAuthentication && !m_forceDigestAuthentication) + if (!m_useIntegratedAuthentication && !m_forceDigestAuthentication && m_userInfo != null) { m_httpClient.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue( "Basic", - Convert.ToBase64String(System.Text.Encoding.ASCII.GetBytes($"{m_userInfo!.UserName}:{m_userInfo.Password}")) + Convert.ToBase64String(System.Text.Encoding.ASCII.GetBytes($"{m_userInfo.UserName}:{m_userInfo.Password}")) ); } - } return m_httpClient; From 2f8583f0e620cb7d2dd977770544fd0f1d49bf0d Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 24 Mar 2025 22:11:22 +0100 Subject: [PATCH 47/48] Added option to avoid setting username on WebDAV. This fixes #3367 --- .../ngax/scripts/services/EditUriBackendConfig.js | 8 ++++++++ .../webroot/ngax/scripts/services/EditUriBuiltins.js | 10 +++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/Duplicati/Server/webroot/ngax/scripts/services/EditUriBackendConfig.js b/Duplicati/Server/webroot/ngax/scripts/services/EditUriBackendConfig.js index 8b4f3b69a..032f46af2 100644 --- a/Duplicati/Server/webroot/ngax/scripts/services/EditUriBackendConfig.js +++ b/Duplicati/Server/webroot/ngax/scripts/services/EditUriBackendConfig.js @@ -131,4 +131,12 @@ backupApp.service('EditUriBackendConfig', function(AppService, AppUtils, SystemI return true; }; + this.recommend_username = function(scope, continuation) { + if ((scope.Username || '').trim().length == 0) + return self.show_warning_dialog(gettextCatalog.getString('Most servers require a username, so you will likely need to enter one.\nAre you sure want to continue without a username?'), continuation); + else + continuation(); + }; + + }); diff --git a/Duplicati/Server/webroot/ngax/scripts/services/EditUriBuiltins.js b/Duplicati/Server/webroot/ngax/scripts/services/EditUriBuiltins.js index c6af89a3a..c72b18138 100644 --- a/Duplicati/Server/webroot/ngax/scripts/services/EditUriBuiltins.js +++ b/Duplicati/Server/webroot/ngax/scripts/services/EditUriBuiltins.js @@ -1047,12 +1047,12 @@ backupApp.service('EditUriBuiltins', function (AppService, AppUtils, SystemInfo, }; EditUriBackendConfig.validaters['webdav'] = function (scope, continuation) { - var res = - EditUriBackendConfig.require_server(scope) && - EditUriBackendConfig.require_username(scope); - + var res = EditUriBackendConfig.require_server(scope); + if (res) - EditUriBackendConfig.recommend_path(scope, continuation); + EditUriBackendConfig.recommend_username(scope, () => { + EditUriBackendConfig.recommend_path(scope, continuation); + }); }; EditUriBackendConfig.validaters['webdav'] = EditUriBackendConfig.validaters['webdav']; From 0841e884db67bf81a0704e25c83df4dce5b9f27d Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 24 Mar 2025 22:49:05 +0100 Subject: [PATCH 48/48] Better error message on manifest version mismatch. This changes the error message reported if the manifest or encoding fileds contain values that are not supported. This fixes #2065 --- .../Main/Volumes/InvalidManifestException.cs | 58 ++++++++++--------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/Duplicati/Library/Main/Volumes/InvalidManifestException.cs b/Duplicati/Library/Main/Volumes/InvalidManifestException.cs index 9b7543f51..f079e7238 100644 --- a/Duplicati/Library/Main/Volumes/InvalidManifestException.cs +++ b/Duplicati/Library/Main/Volumes/InvalidManifestException.cs @@ -1,40 +1,46 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; +using Duplicati.Library.Interface; namespace Duplicati.Library.Main.Volumes { [Serializable] - public class InvalidManifestException : Exception + public class InvalidManifestException : UserInformationException { + private static string GetMessage(string fieldname, string value, string expected) + { + // Make a custom message if this is a version mismatch + if (string.Equals(fieldname, "version", StringComparison.OrdinalIgnoreCase) || string.Equals(fieldname, "encoding", StringComparison.OrdinalIgnoreCase)) + return $"Invalid manifest detected, the field {fieldname} has value {value} but the value {expected} was expected.\nThe most likely cause for this issue is that you are attempting to read a backup created with a newer version of Duplicati.\nPlease upgrade your Duplicati installation to the latest version to resolve this issue."; + + return $"Invalid manifest detected, the field {fieldname} has value {value} but the value {expected} was expected. This could be a configuration issue, or be caused by mixing files from different backups."; + } public InvalidManifestException(string fieldname, string value, string expected) - : base(string.Format("Invalid manifest detected, the field {0} has value {1} but the value {2} was expected", fieldname, value, expected)) + : base(GetMessage(fieldname, value, expected), "InvalidManifest") { } public InvalidManifestException(string message) - : base(message) + : base(message, "InvalidManifest") { } }