Fixed not using the handler

Duplicati/Library/Backend/WEBDAV/WEBDAV.cs

@@ -21,10 +21,12 @@
 
 using Duplicati.Library.Common.IO;
 using Duplicati.Library.Interface;
+using Duplicati.Library.Modules.Builtin;
 using Duplicati.Library.Utility;
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Net;
 using System.Net.Http;
 using System.Threading;
@@ -82,17 +84,9 @@ namespace Duplicati.Library.Backend
         //  "An empty PROPFIND request body MUST be treated as a request for the names and values of all properties."
         //
         //private static readonly byte[] PROPFIND_BODY = System.Text.Encoding.UTF8.GetBytes("<?xml version=\"1.0\"?><D:propfind xmlns:D=\"DAV:\"><D:allprop/></D:propfind>");
-        private static readonly byte[] PROPFIND_BODY = new byte[0];
+        private static readonly byte[] PROPFIND_BODY = [];
 
-        /// <summary>
-        /// The default timeout in seconds for PUT/GET file operations
-        /// </summary>
-        private const int LONG_OPERATION_TIMEOUT_SECONDS = 30000;
-
-        /// <summary>
-        /// The default timeout in seconds for LIST/CreateFolder operations
-        /// </summary>
-        private const int SHORT_OPERATION_TIMEOUT_SECONDS = 30;
+        private readonly HttpOptions.HttpModuleSettings m_httpModuleSettings;
 
         public WEBDAV()
         {
@@ -150,6 +144,7 @@ namespace Duplicati.Library.Backend
             m_rawurlPort = new Utility.Uri(m_useSSL ? "https" : "http", u.Host, m_path, null, null, null, port).ToString();
             m_sanitizedUrl = new Utility.Uri(m_useSSL ? "https" : "http", u.Host, m_path).ToString();
             m_reverseProtocolUrl = new Utility.Uri(m_useSSL ? "http" : "https", u.Host, m_path).ToString();
+            m_httpModuleSettings = HttpOptions.ParseSettings(options);
         }
 
         #region IBackend Members
@@ -179,7 +174,7 @@ namespace Duplicati.Library.Backend
         private IEnumerable<IFileEntry> ListWithouExceptionCatch()
         {
             using var timeoutToken = new CancellationTokenSource();
-            timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS));
+            timeoutToken.CancelAfter(m_httpModuleSettings.ShortOperationTimeout);
 
             using var requestResources = CreateRequest(string.Empty, new HttpMethod("PROPFIND"));
             requestResources.RequestMessage.Headers.Add("Depth", "1");
@@ -285,7 +280,7 @@ namespace Duplicati.Library.Backend
             try
             {
                 using var timeoutToken = new CancellationTokenSource();
-                timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS));
+                timeoutToken.CancelAfter(m_httpModuleSettings.ShortOperationTimeout);
 
                 using var requestResources = CreateRequest(remotename);
                 requestResources.RequestMessage.Method = HttpMethod.Delete;
@@ -308,13 +303,13 @@ namespace Duplicati.Library.Backend
         {
             get
             {
-                return new List<ICommandLineArgument>(new ICommandLineArgument[] {
+                return new[] {
                     new CommandLineArgument("auth-password", CommandLineArgument.ArgumentType.Password, Strings.WEBDAV.DescriptionAuthPasswordShort, Strings.WEBDAV.DescriptionAuthPasswordLong),
                     new CommandLineArgument("auth-username", CommandLineArgument.ArgumentType.String, Strings.WEBDAV.DescriptionAuthUsernameShort, Strings.WEBDAV.DescriptionAuthUsernameLong),
                     new CommandLineArgument("integrated-authentication", CommandLineArgument.ArgumentType.Boolean, Strings.WEBDAV.DescriptionIntegratedAuthenticationShort, Strings.WEBDAV.DescriptionIntegratedAuthenticationLong),
                     new CommandLineArgument("force-digest-authentication", CommandLineArgument.ArgumentType.Boolean, Strings.WEBDAV.DescriptionForceDigestShort, Strings.WEBDAV.DescriptionForceDigestLong),
                     new CommandLineArgument("use-ssl", CommandLineArgument.ArgumentType.Boolean, Strings.WEBDAV.DescriptionUseSSLShort, Strings.WEBDAV.DescriptionUseSSLLong),
-                 });
+                }.Concat(HttpOptions.GetHttpArguments(includeSsl: true, includeReadWrite: true, includeShortTimeout: true)).ToList();
             }
         }
 
@@ -337,7 +332,7 @@ namespace Duplicati.Library.Backend
         {
 
             using var timeoutToken = new CancellationTokenSource();
-            timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS));
+            timeoutToken.CancelAfter(m_httpModuleSettings.ShortOperationTimeout);
 
             using var requestResources = CreateRequest(string.Empty, new HttpMethod("MKCOL"));
 
@@ -363,14 +358,14 @@ namespace Duplicati.Library.Backend
 
             if (m_useIntegratedAuthentication)
             {
-                httpClient = HttpClientHelper.CreateClient(new HttpClientHandler
+                httpClient = HttpOptions.CreateHttpClient(m_httpModuleSettings, new HttpClientHandler
                 {
                     UseDefaultCredentials = true
                 });
             }
             else if (m_forceDigestAuthentication)
             {
-                httpClient = HttpClientHelper.CreateClient(new HttpClientHandler
+                httpClient = HttpOptions.CreateHttpClient(m_httpModuleSettings, new HttpClientHandler
                 {
                     Credentials = new CredentialCache
                     {
@@ -380,15 +375,13 @@ namespace Duplicati.Library.Backend
             }
             else
             {
-                httpClient = HttpClientHelper.CreateClient();
+                httpClient = HttpOptions.CreateHttpClient(m_httpModuleSettings);
                 httpClient.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue(
                     "Basic",
                     Convert.ToBase64String(System.Text.Encoding.ASCII.GetBytes($"{m_userInfo.UserName}:{m_userInfo.Password}"))
                 );
             }
 
-            httpClient.Timeout = Timeout.InfiniteTimeSpan;
-
             var request = new HttpRequestMessage(HttpMethod.Get, $"{m_url}{Utility.Uri.UrlEncode(remotename).Replace("+", "%20")}");
             request.Headers.Add(HttpRequestHeader.UserAgent.ToString(), "Duplicati WEBDAV Client v" + System.Reflection.Assembly.GetExecutingAssembly().GetName().Version);
             request.Headers.ConnectionClose = true; // Equivalent to KeepAlive = false
@@ -406,17 +399,14 @@ namespace Duplicati.Library.Backend
         {
             try
             {
-                using var timeoutToken = new CancellationTokenSource();
-                timeoutToken.CancelAfter(TimeSpan.FromSeconds(LONG_OPERATION_TIMEOUT_SECONDS));
-                using var combinedTokens = CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancelToken);
-
                 using var requestResources = CreateRequest(remotename, HttpMethod.Put);
-
-                requestResources.RequestMessage.Content = new StreamContent(stream);
+                using var timeoutStream = new TimeoutObservingStream(stream) { ReadTimeout = m_httpModuleSettings.ReadWriteTimeoutMilliseconds };
+                requestResources.RequestMessage.Content = new StreamContent(timeoutStream);
                 requestResources.RequestMessage.Content.Headers.ContentLength = stream.Length;
                 requestResources.RequestMessage.Content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/octet-stream");
                 requestResources.RequestMessage.Version = HttpVersion.Version11;
 
+                using var combinedTokens = CancellationTokenSource.CreateLinkedTokenSource(cancelToken, timeoutStream.TimeoutToken);
                 using var response = await requestResources.HttpClient.SendAsync(requestResources.RequestMessage, HttpCompletionOption.ResponseHeadersRead, combinedTokens.Token);
 
                 response.EnsureSuccessStatusCode(); // This replaces the if needed when Mono was used.
@@ -434,12 +424,10 @@ namespace Duplicati.Library.Backend
         {
             try
             {
-                using var timeoutToken = new CancellationTokenSource();
-                timeoutToken.CancelAfter(TimeSpan.FromSeconds(LONG_OPERATION_TIMEOUT_SECONDS));
-
                 using var requestResources = CreateRequest(remotename, HttpMethod.Get);
 
-                requestResources.HttpClient.DownloadFile(requestResources.RequestMessage, stream, null, timeoutToken.Token).ConfigureAwait(false).GetAwaiter().GetResult();
+                using var timeoutStream = new TimeoutObservingStream(stream) { WriteTimeout = m_httpModuleSettings.ReadWriteTimeoutMilliseconds };
+                requestResources.HttpClient.DownloadFile(requestResources.RequestMessage, stream, null, timeoutStream.TimeoutToken).ConfigureAwait(false).GetAwaiter().GetResult();
 
             }
             catch (HttpRequestException wex)

Duplicati/Library/Modules/Builtin/HttpOptions.cs

@@ -21,8 +21,11 @@
 
 using System;
 using System.Collections.Generic;
-using System.Text;
 using System.Linq;
+using System.Net;
+using System.Net.Http;
+using System.Threading;
+using Duplicati.Library.Utility;
 
 namespace Duplicati.Library.Modules.Builtin
 {
@@ -36,24 +39,25 @@ namespace Duplicati.Library.Modules.Builtin
         private const string OPTION_SSL_VERSIONS = "allowed-ssl-versions";
 
         private const string OPTION_BUFFER_REQUESTS = "http-enable-buffering";
-		private const string OPTION_OPERATION_TIMEOUT = "http-operation-timeout";
-		private const string OPTION_READWRITE_TIMEOUT = "http-readwrite-timeout";
+        private const string OPTION_OPERATION_TIMEOUT = "http-operation-timeout";
+        private const string OPTION_OPERATION_SHORT_TIMEOUT = "http-operation-short-timeout";
+        private const string OPTION_READWRITE_TIMEOUT = "http-readwrite-timeout";
 
 
         private bool m_useNagle;
         private bool m_useExpect;
         private System.Net.SecurityProtocolType m_securityProtocol;
 
-		private bool m_dispose;
+        private bool m_dispose;
 
         private bool m_resetNagle;
         private bool m_resetExpect;
         private bool m_resetSecurity;
 
-		/// <summary>
-		/// The handle to the call-context http settings
-		/// </summary>
-		private IDisposable m_httpsettings;
+        /// <summary>
+        /// The handle to the call-context http settings
+        /// </summary>
+        private IDisposable m_httpsettings;
 
         /// <summary>
         /// The handle to the call-context oauth settings
@@ -76,7 +80,7 @@ namespace Duplicati.Library.Modules.Builtin
         {
             var ptr = SecurityProtocols;
             var res = 0;
-            foreach (var s in names.Split(new char[] {','}, StringSplitOptions.RemoveEmptyEntries).Select(x => x.Trim()))
+            foreach (var s in names.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries).Select(x => x.Trim()))
                 if (ptr.ContainsKey(s))
                     res = res | ptr[s];
 
@@ -105,27 +109,124 @@ namespace Duplicati.Library.Modules.Builtin
             get { return true; }
         }
 
-        public IList<Duplicati.Library.Interface.ICommandLineArgument> SupportedCommands
+        private static Interface.ICommandLineArgument[] GetArguments()
         {
-            get { 
-                var sslnames = SecurityProtocols.Select(x => x.Key).ToArray();
-                var defaultssl = System.Net.SecurityProtocolType.SystemDefault.ToString();
+            var sslnames = SecurityProtocols.Select(x => x.Key).ToArray();
+            var defaultssl = System.Net.SecurityProtocolType.SystemDefault.ToString();
 
-                return new List<Duplicati.Library.Interface.ICommandLineArgument>( new Duplicati.Library.Interface.ICommandLineArgument[] {
-                    new Duplicati.Library.Interface.CommandLineArgument(OPTION_DISABLE_EXPECT100, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.HttpOptions.DisableExpect100Short, Strings.HttpOptions.DisableExpect100Long, "false"),
-                    new Duplicati.Library.Interface.CommandLineArgument(OPTION_DISABLE_NAGLING, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.HttpOptions.DisableNagleShort, Strings.HttpOptions.DisableNagleLong, "false"),
-                    new Duplicati.Library.Interface.CommandLineArgument(OPTION_ACCEPT_SPECIFIED_CERTIFICATE, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.String, Strings.HttpOptions.DescriptionAcceptHashShort, Strings.HttpOptions.DescriptionAcceptHashLong2),
-                    new Duplicati.Library.Interface.CommandLineArgument(OPTION_ACCEPT_ANY_CERTIFICATE, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.HttpOptions.DescriptionAcceptAnyCertificateShort, Strings.HttpOptions.DescriptionAcceptAnyCertificateLong),
-                    new Duplicati.Library.Interface.CommandLineArgument(OPTION_OAUTH_URL, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.String, Strings.HttpOptions.OauthurlShort, Strings.HttpOptions.OauthurlLong, OAuthHelper.DUPLICATI_OAUTH_SERVICE),
-                    new Duplicati.Library.Interface.CommandLineArgument(OPTION_SSL_VERSIONS, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Flags, Strings.HttpOptions.SslversionsShort, Strings.HttpOptions.SslversionsLong, defaultssl, null, sslnames),
+            return [
+                new Duplicati.Library.Interface.CommandLineArgument(OPTION_DISABLE_EXPECT100, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.HttpOptions.DisableExpect100Short, Strings.HttpOptions.DisableExpect100Long, "false"),
+                new Duplicati.Library.Interface.CommandLineArgument(OPTION_DISABLE_NAGLING, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.HttpOptions.DisableNagleShort, Strings.HttpOptions.DisableNagleLong, "false", null, null, "This option is deprecated and no longer has any effect."),
+                new Duplicati.Library.Interface.CommandLineArgument(OPTION_ACCEPT_SPECIFIED_CERTIFICATE, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.String, Strings.HttpOptions.DescriptionAcceptHashShort, Strings.HttpOptions.DescriptionAcceptHashLong2),
+                new Duplicati.Library.Interface.CommandLineArgument(OPTION_ACCEPT_ANY_CERTIFICATE, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.HttpOptions.DescriptionAcceptAnyCertificateShort, Strings.HttpOptions.DescriptionAcceptAnyCertificateLong),
+                new Duplicati.Library.Interface.CommandLineArgument(OPTION_OAUTH_URL, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.String, Strings.HttpOptions.OauthurlShort, Strings.HttpOptions.OauthurlLong, OAuthHelper.DUPLICATI_OAUTH_SERVICE),
+                new Duplicati.Library.Interface.CommandLineArgument(OPTION_SSL_VERSIONS, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Flags, Strings.HttpOptions.SslversionsShort, Strings.HttpOptions.SslversionsLong, defaultssl, null, sslnames),
 
-                    new Duplicati.Library.Interface.CommandLineArgument(OPTION_OPERATION_TIMEOUT, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Timespan, Strings.HttpOptions.OperationtimeoutShort, Strings.HttpOptions.OperationtimeoutLong),
-                    new Duplicati.Library.Interface.CommandLineArgument(OPTION_READWRITE_TIMEOUT, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Timespan, Strings.HttpOptions.ReadwritetimeoutShort, Strings.HttpOptions.ReadwritetimeoutLong),
-                    new Duplicati.Library.Interface.CommandLineArgument(OPTION_BUFFER_REQUESTS, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.HttpOptions.BufferrequestsShort, Strings.HttpOptions.BufferrequestsLong, "false"),
-                }); 
-            }
+                new Duplicati.Library.Interface.CommandLineArgument(OPTION_OPERATION_TIMEOUT, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Timespan, Strings.HttpOptions.OperationtimeoutShort, Strings.HttpOptions.OperationtimeoutLong),
+                new Duplicati.Library.Interface.CommandLineArgument(OPTION_OPERATION_SHORT_TIMEOUT, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Timespan, Strings.HttpOptions.OperationshorttimeoutShort, Strings.HttpOptions.OperationshorttimeoutLong),
+                new Duplicati.Library.Interface.CommandLineArgument(OPTION_READWRITE_TIMEOUT, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Timespan, Strings.HttpOptions.ReadwritetimeoutShort, Strings.HttpOptions.ReadwritetimeoutLong),
+                new Duplicati.Library.Interface.CommandLineArgument(OPTION_BUFFER_REQUESTS, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.HttpOptions.BufferrequestsShort, Strings.HttpOptions.BufferrequestsLong, "false"),
+            ];
         }
 
+        private static readonly Duplicati.Library.Interface.ICommandLineArgument[] ARGUMENTS = GetArguments();
+
+        public static IEnumerable<Duplicati.Library.Interface.ICommandLineArgument> GetHttpArguments(bool includeSsl = false, bool includeBuffer = false, bool includeReadWrite = false, bool includeOAuth = false, bool includeShortTimeout = false)
+        {
+            var res = new List<string>();
+            res.AddRange([OPTION_DISABLE_EXPECT100, OPTION_OPERATION_TIMEOUT]);
+            if (includeShortTimeout)
+                res.Add(OPTION_OPERATION_SHORT_TIMEOUT);
+            if (includeReadWrite)
+                res.Add(OPTION_READWRITE_TIMEOUT);
+            if (includeBuffer)
+                res.Add(OPTION_BUFFER_REQUESTS);
+            if (includeSsl)
+                res.AddRange([OPTION_ACCEPT_ANY_CERTIFICATE, OPTION_ACCEPT_SPECIFIED_CERTIFICATE, OPTION_SSL_VERSIONS]);
+            if (includeOAuth)
+                res.Add(OPTION_OAUTH_URL);
+
+            return ARGUMENTS.Where(x => res.Contains(x.Name));
+        }
+
+        public IList<Duplicati.Library.Interface.ICommandLineArgument> SupportedCommands => ARGUMENTS;
+
+        public sealed record HttpModuleSettings(
+            TimeSpan OperationTimeout,
+            TimeSpan ShortOperationTimeout,
+            TimeSpan ReadWriteTimeout,
+            bool BufferRequests,
+            bool AcceptAllCertificates,
+            string[] AcceptCertificates,
+            bool EnableExpect100,
+            System.Net.SecurityProtocolType SecurityProtocols
+        )
+        {
+            public int ReadWriteTimeoutMilliseconds
+                => ReadWriteTimeout == Timeout.InfiniteTimeSpan
+                    ? Timeout.Infinite
+                    : (int)ReadWriteTimeout.TotalMilliseconds;
+
+            public int OperationTimeoutMilliseconds
+                => OperationTimeout == Timeout.InfiniteTimeSpan
+                    ? Timeout.Infinite
+                    : (int)OperationTimeout.TotalMilliseconds;
+
+            public int ShortOperationTimeoutMilliseconds
+                => ShortOperationTimeout == Timeout.InfiniteTimeSpan
+                    ? Timeout.Infinite
+                    : (int)ShortOperationTimeout.TotalMilliseconds;
+        }
+
+        public static HttpModuleSettings ParseSettings(IDictionary<string, string> options)
+        {
+            var operationTimeout = options.TryGetValue(OPTION_OPERATION_TIMEOUT, out var timetmp) && !string.IsNullOrWhiteSpace(timetmp)
+                ? Utility.Timeparser.ParseTimeSpan(timetmp)
+                : Timeout.InfiniteTimeSpan;
+
+            var shortOperationTimeout = options.TryGetValue(OPTION_OPERATION_SHORT_TIMEOUT, out timetmp) && !string.IsNullOrWhiteSpace(timetmp)
+                ? Utility.Timeparser.ParseTimeSpan(timetmp)
+                : operationTimeout;
+
+            var readwriteTimeout = options.TryGetValue(OPTION_READWRITE_TIMEOUT, out timetmp) && !string.IsNullOrWhiteSpace(timetmp)
+                ? Utility.Timeparser.ParseTimeSpan(timetmp)
+                : Timeout.InfiniteTimeSpan;
+
+            options.TryGetValue(OPTION_ACCEPT_SPECIFIED_CERTIFICATE, out var certHash);
+            options.TryGetValue(OPTION_SSL_VERSIONS, out var sslprotocol);
+
+            return new HttpModuleSettings(
+                operationTimeout,
+                shortOperationTimeout,
+                readwriteTimeout,
+                Utility.Utility.ParseBoolOption(options, OPTION_BUFFER_REQUESTS),
+                Utility.Utility.ParseBoolOption(options, OPTION_ACCEPT_ANY_CERTIFICATE),
+                certHash == null ? null : certHash.Split(new string[] { ",", ";" }, StringSplitOptions.RemoveEmptyEntries),
+                // Legacy handling, the "default" was enabled, but now the default is "disabled"
+                options.ContainsKey(OPTION_DISABLE_EXPECT100) ? !Utility.Utility.ParseBoolOption(options, OPTION_DISABLE_EXPECT100) : false,
+                string.IsNullOrWhiteSpace(sslprotocol) ? System.Net.SecurityProtocolType.SystemDefault : ParseSSLProtocols(sslprotocol)
+            );
+        }
+
+        public static HttpClient CreateHttpClient(HttpModuleSettings settings, HttpClientHandler handler = null)
+        {
+            handler ??= new HttpClientHandler();
+            if (settings.AcceptAllCertificates)
+                handler.ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator;
+            else if (settings.AcceptCertificates != null && settings.AcceptCertificates.Length > 0)
+                handler.ServerCertificateCustomValidationCallback = (message, cert, chain, errors) => settings.AcceptCertificates.Contains(cert.GetCertHashString());
+
+            var client = HttpClientHelper.CreateClient(handler);
+            client.Timeout = settings.OperationTimeout == Timeout.InfiniteTimeSpan ? Timeout.InfiniteTimeSpan : settings.OperationTimeout;
+            client.DefaultRequestHeaders.Add(HttpRequestHeader.UserAgent.ToString(), "Duplicati v" + System.Reflection.Assembly.GetExecutingAssembly().GetName().Version);
+            client.DefaultRequestHeaders.ExpectContinue = settings.EnableExpect100;
+
+            return client;
+        }
+
+        // TODO: The configure method below is legacy code and should be removed
+        // once all code is updated to use the HttpClientFactory pattern
+
         public void Configure(IDictionary<string, string> commandlineOptions)
         {
             m_dispose = true;
@@ -138,13 +239,13 @@ namespace Duplicati.Library.Modules.Builtin
                 operationTimeout = Utility.Timeparser.ParseTimeSpan(timetmp);
 
             commandlineOptions.TryGetValue(OPTION_READWRITE_TIMEOUT, out timetmp);
-			if (!string.IsNullOrWhiteSpace(timetmp))
+            if (!string.IsNullOrWhiteSpace(timetmp))
                 readwriteTimeout = Utility.Timeparser.ParseTimeSpan(timetmp);
 
-			bool accepAllCertificates = Utility.Utility.ParseBoolOption(commandlineOptions, OPTION_ACCEPT_ANY_CERTIFICATE);
+            bool accepAllCertificates = Utility.Utility.ParseBoolOption(commandlineOptions, OPTION_ACCEPT_ANY_CERTIFICATE);
 
-			string certHash;
-			commandlineOptions.TryGetValue(OPTION_ACCEPT_SPECIFIED_CERTIFICATE, out certHash);
+            string certHash;
+            commandlineOptions.TryGetValue(OPTION_ACCEPT_SPECIFIED_CERTIFICATE, out certHash);
 
             m_httpsettings = Duplicati.Library.Utility.HttpContextSettings.StartSession(
                 operationTimeout,

Duplicati/Library/Modules/Builtin/Strings.cs

@@ -53,6 +53,8 @@ namespace Duplicati.Library.Modules.Builtin.Strings
         public static string SslversionsShort { get { return LC.L(@"Set allowed SSL versions"); } }
         public static string OperationtimeoutLong { get { return LC.L(@"This option changes the default timeout for any HTTP request, the time covers the entire operation from initial packet to shutdown."); } }
         public static string OperationtimeoutShort { get { return LC.L(@"Set the default operation timeout"); } }
+        public static string OperationshorttimeoutLong { get { return LC.L(@"This option changes the default timeout for HTTP requests that are expected to finish quickly, the time covers the entire operation from initial packet to shutdown."); } }
+        public static string OperationshorttimeoutShort { get { return LC.L(@"Set the default operation timeout for short HTTP requests"); } }
         public static string ReadwritetimeoutLong { get { return LC.L(@"This option changes the default read-write timeout. Read-write timeouts are used to detect a stalled requests, and this option configures the maximum time between activity on a connection."); } }
         public static string ReadwritetimeoutShort { get { return LC.L(@"Set readwrite"); } }
         public static string BufferrequestsLong { get { return LC.L(@"This option sets the HTTP buffering. Setting this to ""{0}"" can cause memory leaks, but can also improve performance in some cases.", "true"); } }

Duplicati/Library/Utility/HttpClientExtensions.cs

@@ -52,7 +52,7 @@ public static class HttpClientExtensions
         if (progressReportingAction != null)
         {
             using var ProgressReportingStream = new ProgressReportingStream(stream, progressReportingAction);
-            await ProgressReportingStream.CopyToAsync(fileStream,cancellationToken);
+            await ProgressReportingStream.CopyToAsync(fileStream, cancellationToken);
         }
         else
         {
@@ -92,8 +92,8 @@ public static class HttpClientExtensions
     /// <param name="client">The Http client reference</param>
     /// <param name="request">A prepared HttpRequestMessage (Presumably with a stream)</param>
     /// <param name="cancellationToken">Cancelation token</param>
-    public static async Task<HttpResponseMessage> UploadStream(this HttpClient client, HttpRequestMessage request,CancellationToken cancellationToken = default)
+    public static async Task<HttpResponseMessage> UploadStream(this HttpClient client, HttpRequestMessage request, CancellationToken cancellationToken = default)
     {
-        return await client.SendAsync(request,  HttpCompletionOption.ResponseContentRead, cancellationToken);    
+        return await client.SendAsync(request, HttpCompletionOption.ResponseContentRead, cancellationToken);
     }
 }

Duplicati/Library/Utility/ShaderStream.cs

@@ -19,58 +19,43 @@
 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
 // DEALINGS IN THE SOFTWARE.
 
-using System;
+using System.IO;
 
 namespace Duplicati.Library.Utility
 {
     /// <summary>
     /// A small utility stream that allows to keep streams open and counts the bytes sent through.
     /// </summary>
-    public class ShaderStream : System.IO.Stream
+    public class ShaderStream : WrappingStream
     {
-        private readonly System.IO.Stream m_baseStream;
         private readonly bool m_keepBaseOpen;
         private long m_read = 0;
         private long m_written = 0;
 
-        public ShaderStream(System.IO.Stream baseStream, bool keepBaseOpen)
+        public ShaderStream(Stream baseStream, bool keepBaseOpen)
+            : base(baseStream)
         {
-            if (baseStream == null)
-                throw new ArgumentNullException(nameof(baseStream));
-            this.m_baseStream = baseStream;
-            this.m_keepBaseOpen = keepBaseOpen;
+            m_keepBaseOpen = keepBaseOpen;
         }
 
         public long TotalBytesRead { get { return m_read; } }
         public long TotalBytesWritten { get { return m_written; } }
 
-        public override bool CanRead { get { return m_baseStream.CanRead; } }
-        public override bool CanSeek { get { return m_baseStream.CanSeek; } }
-        public override bool CanWrite { get { return m_baseStream.CanWrite; } }
-        public override long Length { get { return m_baseStream.Length; } }
-        public override long Position
-        {
-            get { return m_baseStream.Position; }
-            set { m_baseStream.Position = value; }
-        }
-        public override void Flush() { m_baseStream.Flush(); }
-        public override long Seek(long offset, System.IO.SeekOrigin origin) { return m_baseStream.Seek(offset, origin); }
-        public override void SetLength(long value) { m_baseStream.SetLength(value); }
         public override int Read(byte[] buffer, int offset, int count)
         {
-            int r = m_baseStream.Read(buffer, offset, count);
+            int r = BaseStream.Read(buffer, offset, count);
             m_read += r;
             return r;
         }
         public override void Write(byte[] buffer, int offset, int count)
         {
-            m_baseStream.Write(buffer, offset, count);
+            BaseStream.Write(buffer, offset, count);
             m_written += count;
         }
         protected override void Dispose(bool disposing)
         {
             if (disposing && !m_keepBaseOpen)
-                m_baseStream.Close();
+                BaseStream.Close();
             base.Dispose(disposing);
         }
     }

Duplicati/Library/Utility/TimeoutObservingStream.cs

@@ -0,0 +1,202 @@
+// Copyright (C) 2024, The Duplicati Team
+// https://duplicati.com, hello@duplicati.com
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a 
+// copy of this software and associated documentation files (the "Software"), 
+// to deal in the Software without restriction, including without limitation 
+// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+// and/or sell copies of the Software, and to permit persons to whom the 
+// Software is furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in 
+// all copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+// DEALINGS IN THE SOFTWARE.
+
+using System;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Duplicati.Library.Utility;
+
+/// <summary>
+/// A stream that can observe timeouts for read and write operations.
+/// </summary>
+public sealed class TimeoutObservingStream : WrappingAsyncStream
+{
+    /// <summary>
+    /// The read timeout.
+    /// </summary>
+    private int _readTimeout = Timeout.Infinite;
+    /// <summary>
+    /// The write timeout.
+    /// </summary>
+    private int _writeTimeout = Timeout.Infinite;
+
+    /// <summary>
+    /// The cancellation token source for the timeout.
+    /// </summary>
+    private readonly CancellationTokenSource _timeoutCts = new();
+
+    /// <summary>
+    /// The timer for the read timeout.
+    /// </summary>
+    private readonly Timer _readTimer;
+    /// <summary>
+    /// The timer for the write timeout.
+    /// </summary>
+    private readonly Timer _writeTimer;
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="TimeoutObservingStream"/> class.
+    /// </summary>
+    /// <param name="stream">The stream to wrap.</param>
+    public TimeoutObservingStream(Stream stream)
+        : base(stream)
+    {
+        _readTimer = new(_ => _timeoutCts.Cancel());
+        _writeTimer = new(_ => _timeoutCts.Cancel());
+    }
+
+    /// <summary>
+    /// The cancellation token for the timeout.
+    /// </summary>
+    public CancellationToken TimeoutToken => _timeoutCts.Token;
+
+    /// <inheritdoc/>
+    override public bool CanTimeout => true;
+
+    /// <inheritdoc/>
+    public override int ReadTimeout
+    {
+        get => _readTimeout;
+        set
+        {
+            if (value <= 0 && value != Timeout.Infinite)
+                throw new ArgumentOutOfRangeException(nameof(value));
+            _readTimeout = value;
+            _readTimer.Change(value, Timeout.Infinite);
+        }
+    }
+
+    /// <inheritdoc/>
+    public override int WriteTimeout
+    {
+        get => _writeTimeout;
+        set
+        {
+            if (value <= 0 && value != Timeout.Infinite)
+                throw new ArgumentOutOfRangeException(nameof(value));
+            _writeTimeout = value;
+            _writeTimer.Change(value, Timeout.Infinite);
+        }
+    }
+
+    /// <summary>
+    /// Sets the timeout for both read and write operations to infinite.
+    /// </summary>
+    public void CancelTimeout()
+        => WriteTimeout = ReadTimeout = Timeout.Infinite;
+
+    /// <inheritdoc/>
+    override protected async Task<int> ReadImplAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
+    {
+        // If the timer is disable, it is already stopped, otherwise restart it
+        if (_readTimeout != Timeout.Infinite)
+            _readTimer.Change(_readTimeout, Timeout.Infinite);
+
+        // If there is no timeout and no cancellation token, we can just call the base stream
+        if (_readTimeout == Timeout.Infinite && !cancellationToken.CanBeCanceled)
+            return await BaseStream.ReadAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false);
+
+        // We need a cts here to handle cancellation when not handled by the callee, 
+        // but in case the callee *does* observe it, we also link it to the timeout cts
+        using var cts = cancellationToken.CanBeCanceled && cancellationToken != TimeoutToken
+            ? CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _timeoutCts.Token)
+            : null;
+
+        // Get the token to use
+        var tk = cts == null ? _timeoutCts.Token : cts.Token;
+
+        var task = BaseStream.ReadAsync(buffer, offset, count, tk);
+
+        // If the task is already completed, we can await it without a timeout
+        if (task.IsCompleted)
+            return await task.ConfigureAwait(false);
+
+        // Run the task and observe the cancellation token
+        var res = await Task.WhenAny(Task.Run(() => task, tk)).ConfigureAwait(false);
+
+        // Check if we should throw a timeout exception
+        // In case there is a race here, we prefer timeout over any other error for performance reasons
+        if (!cancellationToken.IsCancellationRequested && _timeoutCts.IsCancellationRequested)
+            throw new TimeoutException();
+
+        // Any exceptions from the task are rethrown here
+        return await res.ConfigureAwait(false);
+    }
+
+    /// <inheritdoc/>
+    override protected async Task WriteImplAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
+    {
+        // If the timer is disable, it is already stopped, otherwise restart it
+        if (_writeTimeout != Timeout.Infinite)
+            _writeTimer.Change(_writeTimeout, Timeout.Infinite);
+
+        // If there is no timeout and no cancellation token, we can just call the base stream
+        if (_writeTimeout == Timeout.Infinite && !cancellationToken.CanBeCanceled)
+        {
+            await BaseStream.WriteAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false);
+            return;
+        }
+
+        // We need a cts here to handle cancellation when not handled by the callee, 
+        // but in case the callee *does* observe it, we also link it to the timeout cts
+        using var cts = cancellationToken.CanBeCanceled && cancellationToken != TimeoutToken
+            ? CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _timeoutCts.Token)
+            : null;
+
+        // Get the token to use
+        var tk = cts == null ? _timeoutCts.Token : cts.Token;
+
+        var task = BaseStream.WriteAsync(buffer, offset, count, tk);
+
+        // If the task is already completed, we can await it without a timeout
+        if (task.IsCompleted)
+        {
+            await task.ConfigureAwait(false);
+            return;
+        }
+
+        // Run the task and observe the cancellation token
+        var res = await Task.WhenAny(Task.Run(() => task, tk)).ConfigureAwait(false);
+
+        // Check if we should throw a timeout exception
+        // In case there is a race here, we prefer timeout over any other error for performance reasons
+        if (!cancellationToken.IsCancellationRequested && _timeoutCts.IsCancellationRequested)
+            throw new TimeoutException();
+
+        // Any exceptions from the task are rethrown here
+        await task.ConfigureAwait(false);
+    }
+
+    /// <inheritdoc/>
+    protected override void Dispose(bool disposing)
+    {
+        if (disposing)
+        {
+            _readTimer.Dispose();
+            _writeTimer.Dispose();
+            _timeoutCts.Dispose();
+        }
+
+        base.Dispose(disposing);
+    }
+}

Duplicati/Library/Utility/WrappingAsyncStream.cs

@@ -0,0 +1,72 @@
+// Copyright (C) 2024, The Duplicati Team
+// https://duplicati.com, hello@duplicati.com
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a 
+// copy of this software and associated documentation files (the "Software"), 
+// to deal in the Software without restriction, including without limitation 
+// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+// and/or sell copies of the Software, and to permit persons to whom the 
+// Software is furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in 
+// all copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+// DEALINGS IN THE SOFTWARE.
+
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Duplicati.Library.Utility;
+
+/// <summary>
+/// Wraps a <see cref="Stream"/> and delegates all calls to it,
+/// mapping the usual read/write calls to the async variants.
+/// </summary>
+public abstract class WrappingAsyncStream : WrappingStream
+{
+    /// <summary>
+    /// Initializes a new instance of the <see cref="WrappingAsyncStream"/> class.
+    /// </summary>
+    /// <param name="stream">The stream to wrap.</param>
+    public WrappingAsyncStream(Stream stream)
+        : base(stream) { }
+
+    /// <inheritdoc/>
+    public override int Read(byte[] buffer, int offset, int count)
+    {
+        return ReadImplAsync(buffer, offset, count, default).ConfigureAwait(false).GetAwaiter().GetResult();
+    }
+
+    /// <inheritdoc/>
+    public override void Write(byte[] buffer, int offset, int count)
+    {
+        WriteImplAsync(buffer, offset, count, default).ConfigureAwait(false).GetAwaiter().GetResult();
+    }
+
+    /// <summary>
+    /// Reads a sequence of bytes from the current stream and advances the position within the stream by the number of bytes read.
+    /// </summary>
+    /// <param name="buffer">An array of bytes. When this method returns, the buffer contains the specified byte array with the values between offset and (offset + count - 1) replaced by the bytes read from the current source.</param>
+    /// <param name="offset">The zero-based byte offset in buffer at which to begin storing the data read from the current stream.</param>
+    /// <param name="count">The maximum number of bytes to be read from the current stream.</param>
+    /// <param name="cancellationToken">The token to monitor for cancellation requests.</param>
+    /// <returns>A task that represents the asynchronous read operation. The value of its <see cref="Task{TResult}.Result"/> property contains the total number of bytes read into the buffer. The result value can be less than the number of bytes requested if the number of bytes currently available is less than the requested number, or it can be 0 (zero) if the end of the stream has been reached.</returns>
+    protected abstract Task<int> ReadImplAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken);
+
+    /// <summary>
+    /// Writes a sequence of bytes to the current stream and advances the current position within this stream by the number of bytes written.
+    /// </summary>
+    /// <param name="buffer">An array of bytes. This method copies count bytes from buffer to the current stream.</param>
+    /// <param name="offset">The zero-based byte offset in buffer at which to begin copying bytes to the current stream.</param>
+    /// <param name="count">The number of bytes to be written to the current stream.</param>
+    /// <param name="cancellationToken">The token to monitor for cancellation requests.</param>
+    /// <returns>A task that represents the asynchronous write operation.</returns>
+    protected abstract Task WriteImplAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken);
+}

Duplicati/Library/Utility/WrappingStream.cs

@@ -0,0 +1,93 @@
+// Copyright (C) 2024, The Duplicati Team
+// https://duplicati.com, hello@duplicati.com
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a 
+// copy of this software and associated documentation files (the "Software"), 
+// to deal in the Software without restriction, including without limitation 
+// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+// and/or sell copies of the Software, and to permit persons to whom the 
+// Software is furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in 
+// all copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+// DEALINGS IN THE SOFTWARE.
+
+
+using System;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Duplicati.Library.Utility;
+
+/// <summary>
+/// Wraps a <see cref="Stream"/> and delegates all calls to it.
+/// </summary>
+public abstract class WrappingStream : Stream
+{
+    /// <summary>
+    /// The stream being wrapped.
+    /// </summary>
+    public Stream BaseStream { get; init; }
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="WrappingStream"/> class.
+    /// </summary>
+    /// <param name="stream">The stream to wrap.</param>
+    protected WrappingStream(Stream stream)
+    {
+        if (stream == null)
+            throw new ArgumentNullException(nameof(stream));
+        BaseStream = stream;
+    }
+
+    /// <inheritdoc/>
+    public override bool CanTimeout => BaseStream.CanTimeout;
+    /// <inheritdoc/>
+    public override bool CanRead => BaseStream.CanRead;
+    /// <inheritdoc/>
+    public override bool CanSeek => BaseStream.CanSeek;
+    /// <inheritdoc/>
+    public override bool CanWrite => BaseStream.CanWrite;
+    /// <inheritdoc/>
+    public override long Length => BaseStream.Length;
+    /// <inheritdoc/>
+    public override long Position
+    {
+        get => BaseStream.Position;
+        set => BaseStream.Position = value;
+    }
+
+    /// <inheritdoc/>
+    public override void Flush() => BaseStream.Flush();
+    /// <inheritdoc/>
+    public override long Seek(long offset, SeekOrigin origin) => BaseStream.Seek(offset, origin);
+    /// <inheritdoc/>
+    public override void SetLength(long value) => BaseStream.SetLength(value);
+
+    /// <inheritdoc/>
+    protected override void Dispose(bool disposing)
+    {
+        if (disposing)
+            BaseStream.Dispose();
+    }
+
+    /// <inheritdoc/>
+    public override ValueTask DisposeAsync()
+        => BaseStream.DisposeAsync();
+
+    /// <inheritdoc/>
+    public override void Close()
+        => BaseStream.Close();
+
+    /// <inheritdoc/>
+    public override Task FlushAsync(CancellationToken cancellationToken)
+        => BaseStream.FlushAsync(cancellationToken);
+}