Merge pull request #6044 from duplicati/feature/retries-for-http-message

Added retries for HTTP messages

Duplicati/Library/Modules/Builtin/SendHttpMessage.cs

@@ -28,6 +28,7 @@ using System.Text;
 using System.Threading.Tasks;
 using Duplicati.Library.Utility;
 using Uri = System.Uri;
+using System.Threading;
 
 namespace Duplicati.Library.Modules.Builtin
 {
@@ -113,6 +114,15 @@ namespace Duplicati.Library.Modules.Builtin
         /// </summary>
         private const string OPTION_ACCEPT_ANY_CERTIFICATE = "send-http-accept-any-ssl-certificate";
 
+        /// <summary>
+        /// The option used to specify the number of retries for sending the HTTP request
+        /// </summary>
+        private const string OPTION_SEND_HTTP_RETRIES = "send-http-retries";
+        /// <summary>
+        /// The option used to specify the delay between retries for sending the HTTP request
+        /// </summary>
+        private const string OPTION_SEND_HTTP_RETRY_DELAY = "send-http-retry-delay";
+
         #endregion
 
         #region Option defaults
@@ -128,6 +138,14 @@ namespace Duplicati.Library.Modules.Builtin
         /// Don't use the subject for HTTP
         /// </summary>
         protected override string DEFAULT_SUBJECT => string.Empty;
+        /// <summary>
+        /// The default number of retries
+        /// </summary>
+        private const int DEFAULT_RETRIES = 3;
+        /// <summary>
+        /// The default delay between retries
+        /// </summary>
+        private const string DEFAULT_RETRY_DELAY = "1s";
         #endregion
 
         #region Private variables
@@ -154,6 +172,15 @@ namespace Duplicati.Library.Modules.Builtin
         /// </summary>
         private string[] m_acceptSpecificCertificates;
 
+        /// <summary>
+        /// The number of retries to attempt
+        /// </summary>
+        private int m_retries;
+        /// <summary>
+        /// The delay between retries
+        /// </summary>
+        private TimeSpan m_retryDelay;
+
         #endregion
 
 
@@ -208,6 +235,9 @@ namespace Duplicati.Library.Modules.Builtin
 
                     new CommandLineArgument(OPTION_ACCEPT_ANY_CERTIFICATE, CommandLineArgument.ArgumentType.Boolean, Strings.SendHttpMessage.AcceptAnyCertificateShort, Strings.SendHttpMessage.AcceptAnyCertificateLong),
                     new CommandLineArgument(OPTION_ACCEPT_SPECIFIED_CERTIFICATE, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.AcceptSpecifiedCertificateShort, Strings.SendHttpMessage.AcceptSpecifiedCertificateLong),
+
+                    new CommandLineArgument(OPTION_SEND_HTTP_RETRIES, CommandLineArgument.ArgumentType.Integer, Strings.SendHttpMessage.SendHttpRetriesShort, Strings.SendHttpMessage.SendHttpRetriesLong, DEFAULT_RETRIES.ToString()),
+                    new CommandLineArgument(OPTION_SEND_HTTP_RETRY_DELAY, CommandLineArgument.ArgumentType.Integer, Strings.SendHttpMessage.SendHttpRetryDelayShort, Strings.SendHttpMessage.SendHttpRetryDelayLong, DEFAULT_RETRY_DELAY),
                 });
             }
         }
@@ -266,9 +296,12 @@ namespace Duplicati.Library.Modules.Builtin
                 m_messageParameterName = DEFAULT_MESSAGE_PARAMETER_NAME;
 
             commandlineOptions.TryGetValue(OPTION_EXTRA_PARAMETERS, out m_extraParameters);
-            m_acceptAnyCertificate = commandlineOptions.ContainsKey(OPTION_ACCEPT_ANY_CERTIFICATE) && Utility.Utility.ParseBoolOption(commandlineOptions.AsReadOnly(), OPTION_ACCEPT_ANY_CERTIFICATE);
+            m_acceptAnyCertificate = Utility.Utility.ParseBoolOption(commandlineOptions.AsReadOnly(), OPTION_ACCEPT_ANY_CERTIFICATE);
             m_acceptSpecificCertificates = commandlineOptions.ContainsKey(OPTION_ACCEPT_SPECIFIED_CERTIFICATE) ? commandlineOptions[OPTION_ACCEPT_SPECIFIED_CERTIFICATE].Split([",", ";"], StringSplitOptions.RemoveEmptyEntries) : null;
 
+            m_retries = Utility.Utility.ParseIntOption(commandlineOptions.AsReadOnly(), OPTION_SEND_HTTP_RETRIES, DEFAULT_RETRIES);
+            m_retryDelay = Utility.Utility.ParseTimespanOption(commandlineOptions.AsReadOnly(), OPTION_SEND_HTTP_RETRY_DELAY, DEFAULT_RETRY_DELAY);
+
             return true;
         }
 
@@ -301,7 +334,9 @@ namespace Duplicati.Library.Modules.Builtin
             };
             request.Content.Headers.ContentType = contenttype;
 
-            try
+            Exception lastEx = null;
+
+            await RetryHelper.Retry(async () =>
             {
                 var response = await client.SendAsync(request);
                 var responseContent = await response.Content.ReadAsStringAsync();
@@ -315,18 +350,17 @@ namespace Duplicati.Library.Modules.Builtin
                 );
 
                 response.EnsureSuccessStatusCode();
-            }
-            catch (Exception ex)
+                lastEx = null;
+            }, (ex, retry) =>
             {
-                Logging.Log.WriteWarningMessage(LOGTAG, "HttpResponseError", ex, "HTTP Response request failed for: {0}", target.Url);
-                return ex;
-            }
+                Logging.Log.WriteWarningMessage(LOGTAG, "HttpResponseError", ex, "HTTP Response request attempt {0} of {1} failed for: {2}", retry, m_retries, target.Url);
+                lastEx = ex;
+            }, m_retries, m_retryDelay, CancellationToken.None);
 
-            return null;
+            return lastEx;
         }
 
         private Dictionary<ResultExportFormat, string> m_cachedBodyResults;
-        private string m_form_body = string.Empty;
 
         protected override string ReplaceTemplate(string input, object result, Exception exception, bool subjectline)
         {
@@ -364,7 +398,7 @@ namespace Duplicati.Library.Modules.Builtin
             foreach (var target in m_report_targets)
             {
                 if (m_cachedBodyResults.TryGetValue(target.Format, out var result))
-                    ex ??= SendMessage(client, target, subject, result).ConfigureAwait(false).GetAwaiter().GetResult();
+                    ex ??= SendMessage(client, target, subject, result).Await();
             }
 
             if (ex != null)

Duplicati/Library/Modules/Builtin/SendJabberMessage.cs

@@ -22,9 +22,6 @@ using System;
 using System.Collections.Generic;
 using System.Linq;
 using Duplicati.Library.Interface;
-using Duplicati.Library.Logging;
-using System.Net.NetworkInformation;
-using Duplicati.Library.Modules.Builtin.ResultSerialization;
 using Artalk.Xmpp.Client;
 
 namespace Duplicati.Library.Modules.Builtin

Duplicati/Library/Modules/Builtin/SendMail.cs

@@ -1,39 +1,31 @@
-// Copyright (C) 2025, The Duplicati Team
-// https://duplicati.com, hello@duplicati.com
-// 
-// Permission is hereby granted, free of charge, to any person obtaining a 
-// copy of this software and associated documentation files (the "Software"), 
-// to deal in the Software without restriction, including without limitation 
-// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
-// and/or sell copies of the Software, and to permit persons to whom the 
-// Software is furnished to do so, subject to the following conditions:
-// 
-// The above copyright notice and this permission notice shall be included in 
-// all copies or substantial portions of the Software.
-// 
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+// Copyright (C) 2025, The Duplicati Team
+// https://duplicati.com, hello@duplicati.com
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a 
+// copy of this software and associated documentation files (the "Software"), 
+// to deal in the Software without restriction, including without limitation 
+// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+// and/or sell copies of the Software, and to permit persons to whom the 
+// Software is furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in 
+// all copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
 // DEALINGS IN THE SOFTWARE.
 using System;
-using System.Collections;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
-using System.Net;
 using System.Text;
-using System.Text.RegularExpressions;
 using Duplicati.Library.Interface;
-using Duplicati.Library.Logging;
-using Duplicati.Library.Utility;
-using System.Net.NetworkInformation;
-using System.Net.Sockets;
 using MailKit.Net.Smtp;
 using MimeKit;
-using Duplicati.Library.Modules.Builtin.ResultSerialization;
 using DnsClient;
 
 namespace Duplicati.Library.Modules.Builtin
@@ -277,7 +269,7 @@ namespace Duplicati.Library.Modules.Builtin
                 lastServer = server;
                 try
                 {
-                    using (MemoryStream ms = new MemoryStream())
+                    using (var ms = new MemoryStream())
                     {
                         try
                         {

Duplicati/Library/Modules/Builtin/Strings.cs

@@ -231,6 +231,10 @@ You can supply multiple options with a comma separator, e.g. ""{0},{1}"". The sp
         public static string AcceptAnyCertificateShort { get { return LC.L(@"Accept any server certificate"); } }
         public static string AcceptSpecifiedCertificateLong { get { return LC.L(@"If your server certificate is reported as invalid (e.g. with self-signed certificates), you can supply the certificate hash (SHA1) to approve it anyway. The hash value must be entered in hex format without spaces or colons. You can enter multiple hashes separated by commas."); } }
         public static string AcceptSpecifiedCertificateShort { get { return LC.L(@"Optionally accept a known SSL certificate"); } }
+        public static string SendHttpRetriesLong { get { return LC.L(@"Use this option to set the number of retries to attempt if the HTTP request fails."); } }
+        public static string SendHttpRetriesShort { get { return LC.L(@"Set the number of retries"); } }
+        public static string SendHttpRetryDelayLong { get { return LC.L(@"Use this option to set the delay between retries."); } }
+        public static string SendHttpRetryDelayShort { get { return LC.L(@"Set the retry delay"); } }
     }
 
     internal static class ReportHelper

Duplicati/Library/Utility/RetryHelper.cs

@@ -19,6 +19,8 @@
 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
 // DEALINGS IN THE SOFTWARE.
 
+#nullable enable
+
 using System;
 using System.Threading;
 using System.Threading.Tasks;
@@ -27,7 +29,7 @@ namespace Duplicati.Library.Utility;
 
 public static class RetryHelper
 {
-    public static async Task<T> Retry<T>(Func<Task<T>> action, int maxRetries, TimeSpan delay, CancellationToken token)
+    public static async Task<T> Retry<T>(Func<Task<T>> action, Action<Exception, int>? errorCallback, int maxRetries, TimeSpan delay, CancellationToken token)
     {
         var attempt = 0;
 
@@ -37,21 +39,26 @@ public static class RetryHelper
             {
                 return await action();
             }
-            catch
+            catch (Exception ex)
             {
+                attempt++;
+                errorCallback?.Invoke(ex, attempt);
+
                 if (token.IsCancellationRequested)
                     throw;
                 if (attempt >= maxRetries)
                     throw;
-
-                attempt++;
             }
 
             await Task.Delay(delay, token);
         }
     }
 
+    public static async Task<T> Retry<T>(Func<Task<T>> action, int maxRetries, TimeSpan delay, CancellationToken token)
+        => await Retry(action, null, maxRetries, delay, token);
+    public static async Task Retry(Func<Task> action, Action<Exception, int>? errorCallback, int maxRetries, TimeSpan delay, CancellationToken token)
+        => await Retry(async () => { await action(); return true; }, errorCallback, maxRetries, delay, token);
     public static async Task Retry(Func<Task> action, int maxRetries, TimeSpan delay, CancellationToken token)
-        => await Retry(async () => { await action(); return true; }, maxRetries, delay, token);
+        => await Retry(async () => { await action(); return true; }, null, maxRetries, delay, token);
 
 }

Duplicati/Library/Utility/Timeparser.cs

@@ -1,27 +1,25 @@
-// Copyright (C) 2025, The Duplicati Team
-// https://duplicati.com, hello@duplicati.com
-// 
-// Permission is hereby granted, free of charge, to any person obtaining a 
-// copy of this software and associated documentation files (the "Software"), 
-// to deal in the Software without restriction, including without limitation 
-// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
-// and/or sell copies of the Software, and to permit persons to whom the 
-// Software is furnished to do so, subject to the following conditions:
-// 
-// The above copyright notice and this permission notice shall be included in 
-// all copies or substantial portions of the Software.
-// 
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+// Copyright (C) 2025, The Duplicati Team
+// https://duplicati.com, hello@duplicati.com
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a 
+// copy of this software and associated documentation files (the "Software"), 
+// to deal in the Software without restriction, including without limitation 
+// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+// and/or sell copies of the Software, and to permit persons to whom the 
+// Software is furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in 
+// all copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
 // DEALINGS IN THE SOFTWARE.
 
 using System;
-using System.Collections.Generic;
-using System.Text;
 
 namespace Duplicati.Library.Utility
 {
@@ -34,7 +32,7 @@ namespace Duplicati.Library.Utility
     {
         public static TimeSpan ParseTimeSpan(string datestring)
         {
-            DateTime dt = new DateTime(0, DateTimeKind.Local);
+            var dt = new DateTime(0, DateTimeKind.Local);
             return ParseTimeInterval(datestring, dt) - dt;
         }
 
@@ -48,16 +46,15 @@ namespace Duplicati.Library.Utility
             if (offset.Kind == DateTimeKind.Unspecified)
                 offset = new DateTime(offset.Ticks, DateTimeKind.Local);
 
-            int multiplier = negate ? -1 : 1;
+            var multiplier = negate ? -1 : 1;
 
             if (string.IsNullOrEmpty(datestring))
                 return offset;
 
-            if (String.Equals(datestring.Trim(), "now", StringComparison.OrdinalIgnoreCase))
+            if (string.Equals(datestring.Trim(), "now", StringComparison.OrdinalIgnoreCase))
                 return DateTime.Now;
 
-            long l;
-            if (long.TryParse(datestring, System.Globalization.NumberStyles.Integer, null, out l))
+            if (long.TryParse(datestring, System.Globalization.NumberStyles.Integer, null, out var l))
                 return offset.AddSeconds(l * multiplier);
 
             if (DateTime.TryParse(datestring, System.Globalization.CultureInfo.CurrentCulture, System.Globalization.DateTimeStyles.AssumeLocal, out var t))
@@ -66,16 +63,15 @@ namespace Duplicati.Library.Utility
             if (Utility.TryDeserializeDateTime(datestring, out t))
                 return t;
 
-            char[] separators = new char[] { 's', 'm', 'h', 'D', 'W', 'M', 'Y' };
+            var separators = new char[] { 's', 'm', 'h', 'D', 'W', 'M', 'Y' };
 
-            int index = 0;
-            int previndex = 0;
+            int index;
+            var previndex = 0;
 
             while ((index = datestring.IndexOfAny(separators, previndex)) > 0)
             {
-                string partial = datestring.Substring(previndex, index - previndex).Trim();
-                int factor;
-                if (!int.TryParse(partial, System.Globalization.NumberStyles.Integer, null, out factor))
+                var partial = datestring.Substring(previndex, index - previndex).Trim();
+                if (!int.TryParse(partial, System.Globalization.NumberStyles.Integer, null, out var factor))
                     throw new Exception(Strings.Timeparser.InvalidIntegerError(partial));
 
                 factor *= multiplier;
@@ -134,11 +130,10 @@ namespace Duplicati.Library.Utility
             if (string.IsNullOrEmpty(datestring))
                 return offset;
 
-            if (String.Equals(datestring.Trim(), "now", StringComparison.OrdinalIgnoreCase))
+            if (string.Equals(datestring.Trim(), "now", StringComparison.OrdinalIgnoreCase))
                 return DateTime.UtcNow;
 
-            long l;
-            if (long.TryParse(datestring, System.Globalization.NumberStyles.Integer, null, out l))
+            if (long.TryParse(datestring, System.Globalization.NumberStyles.Integer, null, out var l))
                 return keepTimeOfDay
                     ? timeZoneInfo.DSTAwareAddSeconds(offset, l * multiplier)
                     : timeZoneInfo.DSTAwareAddSeconds(DateTime.UtcNow, l * multiplier);
@@ -149,16 +144,15 @@ namespace Duplicati.Library.Utility
             if (Utility.TryDeserializeDateTime(datestring, out t))
                 return t;
 
-            char[] separators = ['s', 'm', 'h', 'D', 'W', 'M', 'Y'];
+            var separators = new char[] { 's', 'm', 'h', 'D', 'W', 'M', 'Y' };
 
             int index;
-            int previndex = 0;
+            var previndex = 0;
 
             while ((index = datestring.IndexOfAny(separators, previndex)) > 0)
             {
-                string partial = datestring.Substring(previndex, index - previndex).Trim();
-                int factor;
-                if (!int.TryParse(partial, System.Globalization.NumberStyles.Integer, null, out factor))
+                var partial = datestring.Substring(previndex, index - previndex).Trim();
+                if (!int.TryParse(partial, System.Globalization.NumberStyles.Integer, null, out var factor))
                     throw new Exception(Strings.Timeparser.InvalidIntegerError(partial));
 
                 factor *= multiplier;

Duplicati/Library/Utility/Utility.cs

@@ -702,7 +702,7 @@ namespace Duplicati.Library.Utility
         /// </summary>
         /// <param name="options">The set of options to look for the setting in</param>
         /// <param name="value">The value to look for in the settings</param>
-        /// <returns></returns>
+        /// <returns>The parsed value, or the default value (<c>false</c>).</returns>
         public static bool ParseBoolOption(IReadOnlyDictionary<string, string?> options, string value)
         {
             if (options.TryGetValue(value, out var opt))
@@ -711,6 +711,22 @@ namespace Duplicati.Library.Utility
                 return false;
         }
 
+        /// <summary>
+        /// Parses an integer option from the option set, returning the default value if the option is not found or cannot be parsed
+        /// </summary>
+        /// <param name="options">The set of options to look for the setting in</param>
+        /// <param name="value">The value to look for in the settings</param>
+        /// <param name="default">The default value to return if there are no matches.</param>
+        /// <returns>The parsed or default integer value.</returns>
+        public static TimeSpan ParseTimespanOption(IReadOnlyDictionary<string, string?> options, string value, string @default)
+        {
+            var opt = options.GetValueOrDefault(value);
+            if (string.IsNullOrWhiteSpace(opt))
+                opt = @default;
+
+            return Timeparser.ParseTimeSpan(opt);
+        }
+
         /// <summary>
         /// Parses an enum found in the options dictionary
         /// </summary>