From e37f20aedd9660985d8cea0ff9f3b9ec2734d349 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sun, 16 Mar 2025 16:52:30 +0100 Subject: [PATCH] Added retries for HTTP messages This adds a retry count and a retry delay for posting backup reports via HTTP requests. This will make the report delivery more reliable if the network is unstable or server is briefly unreachable. --- .../Modules/Builtin/SendHttpMessage.cs | 54 +++++++++++--- .../Modules/Builtin/SendJabberMessage.cs | 3 - Duplicati/Library/Modules/Builtin/SendMail.cs | 48 +++++------- Duplicati/Library/Modules/Builtin/Strings.cs | 4 + Duplicati/Library/Utility/RetryHelper.cs | 17 +++-- Duplicati/Library/Utility/Timeparser.cs | 74 +++++++++---------- Duplicati/Library/Utility/Utility.cs | 18 ++++- 7 files changed, 131 insertions(+), 87 deletions(-) diff --git a/Duplicati/Library/Modules/Builtin/SendHttpMessage.cs b/Duplicati/Library/Modules/Builtin/SendHttpMessage.cs index 94f97bc56..6bd57fdec 100644 --- a/Duplicati/Library/Modules/Builtin/SendHttpMessage.cs +++ b/Duplicati/Library/Modules/Builtin/SendHttpMessage.cs @@ -28,6 +28,7 @@ using System.Text; using System.Threading.Tasks; using Duplicati.Library.Utility; using Uri = System.Uri; +using System.Threading; namespace Duplicati.Library.Modules.Builtin { @@ -113,6 +114,15 @@ namespace Duplicati.Library.Modules.Builtin /// private const string OPTION_ACCEPT_ANY_CERTIFICATE = "send-http-accept-any-ssl-certificate"; + /// + /// The option used to specify the number of retries for sending the HTTP request + /// + private const string OPTION_SEND_HTTP_RETRIES = "send-http-retries"; + /// + /// The option used to specify the delay between retries for sending the HTTP request + /// + private const string OPTION_SEND_HTTP_RETRY_DELAY = "send-http-retry-delay"; + #endregion #region Option defaults @@ -128,6 +138,14 @@ namespace Duplicati.Library.Modules.Builtin /// Don't use the subject for HTTP /// protected override string DEFAULT_SUBJECT => string.Empty; + /// + /// The default number of retries + /// + private const int DEFAULT_RETRIES = 3; + /// + /// The default delay between retries + /// + private const string DEFAULT_RETRY_DELAY = "1s"; #endregion #region Private variables @@ -154,6 +172,15 @@ namespace Duplicati.Library.Modules.Builtin /// private string[] m_acceptSpecificCertificates; + /// + /// The number of retries to attempt + /// + private int m_retries; + /// + /// The delay between retries + /// + private TimeSpan m_retryDelay; + #endregion @@ -208,6 +235,9 @@ namespace Duplicati.Library.Modules.Builtin new CommandLineArgument(OPTION_ACCEPT_ANY_CERTIFICATE, CommandLineArgument.ArgumentType.Boolean, Strings.SendHttpMessage.AcceptAnyCertificateShort, Strings.SendHttpMessage.AcceptAnyCertificateLong), new CommandLineArgument(OPTION_ACCEPT_SPECIFIED_CERTIFICATE, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.AcceptSpecifiedCertificateShort, Strings.SendHttpMessage.AcceptSpecifiedCertificateLong), + + new CommandLineArgument(OPTION_SEND_HTTP_RETRIES, CommandLineArgument.ArgumentType.Integer, Strings.SendHttpMessage.SendHttpRetriesShort, Strings.SendHttpMessage.SendHttpRetriesLong, DEFAULT_RETRIES.ToString()), + new CommandLineArgument(OPTION_SEND_HTTP_RETRY_DELAY, CommandLineArgument.ArgumentType.Integer, Strings.SendHttpMessage.SendHttpRetryDelayShort, Strings.SendHttpMessage.SendHttpRetryDelayLong, DEFAULT_RETRY_DELAY), }); } } @@ -266,9 +296,12 @@ namespace Duplicati.Library.Modules.Builtin m_messageParameterName = DEFAULT_MESSAGE_PARAMETER_NAME; commandlineOptions.TryGetValue(OPTION_EXTRA_PARAMETERS, out m_extraParameters); - m_acceptAnyCertificate = commandlineOptions.ContainsKey(OPTION_ACCEPT_ANY_CERTIFICATE) && Utility.Utility.ParseBoolOption(commandlineOptions.AsReadOnly(), OPTION_ACCEPT_ANY_CERTIFICATE); + m_acceptAnyCertificate = Utility.Utility.ParseBoolOption(commandlineOptions.AsReadOnly(), OPTION_ACCEPT_ANY_CERTIFICATE); m_acceptSpecificCertificates = commandlineOptions.ContainsKey(OPTION_ACCEPT_SPECIFIED_CERTIFICATE) ? commandlineOptions[OPTION_ACCEPT_SPECIFIED_CERTIFICATE].Split([",", ";"], StringSplitOptions.RemoveEmptyEntries) : null; + m_retries = Utility.Utility.ParseIntOption(commandlineOptions.AsReadOnly(), OPTION_SEND_HTTP_RETRIES, DEFAULT_RETRIES); + m_retryDelay = Utility.Utility.ParseTimespanOption(commandlineOptions.AsReadOnly(), OPTION_SEND_HTTP_RETRY_DELAY, DEFAULT_RETRY_DELAY); + return true; } @@ -301,7 +334,9 @@ namespace Duplicati.Library.Modules.Builtin }; request.Content.Headers.ContentType = contenttype; - try + Exception lastEx = null; + + await RetryHelper.Retry(async () => { var response = await client.SendAsync(request); var responseContent = await response.Content.ReadAsStringAsync(); @@ -315,18 +350,17 @@ namespace Duplicati.Library.Modules.Builtin ); response.EnsureSuccessStatusCode(); - } - catch (Exception ex) + lastEx = null; + }, (ex, retry) => { - Logging.Log.WriteWarningMessage(LOGTAG, "HttpResponseError", ex, "HTTP Response request failed for: {0}", target.Url); - return ex; - } + Logging.Log.WriteWarningMessage(LOGTAG, "HttpResponseError", ex, "HTTP Response request attempt {0} of {1} failed for: {2}", retry, m_retries, target.Url); + lastEx = ex; + }, m_retries, m_retryDelay, CancellationToken.None); - return null; + return lastEx; } private Dictionary m_cachedBodyResults; - private string m_form_body = string.Empty; protected override string ReplaceTemplate(string input, object result, Exception exception, bool subjectline) { @@ -364,7 +398,7 @@ namespace Duplicati.Library.Modules.Builtin foreach (var target in m_report_targets) { if (m_cachedBodyResults.TryGetValue(target.Format, out var result)) - ex ??= SendMessage(client, target, subject, result).ConfigureAwait(false).GetAwaiter().GetResult(); + ex ??= SendMessage(client, target, subject, result).Await(); } if (ex != null) diff --git a/Duplicati/Library/Modules/Builtin/SendJabberMessage.cs b/Duplicati/Library/Modules/Builtin/SendJabberMessage.cs index bbb565017..81c54e07b 100644 --- a/Duplicati/Library/Modules/Builtin/SendJabberMessage.cs +++ b/Duplicati/Library/Modules/Builtin/SendJabberMessage.cs @@ -22,9 +22,6 @@ using System; using System.Collections.Generic; using System.Linq; using Duplicati.Library.Interface; -using Duplicati.Library.Logging; -using System.Net.NetworkInformation; -using Duplicati.Library.Modules.Builtin.ResultSerialization; using Artalk.Xmpp.Client; namespace Duplicati.Library.Modules.Builtin diff --git a/Duplicati/Library/Modules/Builtin/SendMail.cs b/Duplicati/Library/Modules/Builtin/SendMail.cs index beac39fee..9dc2b9e03 100644 --- a/Duplicati/Library/Modules/Builtin/SendMail.cs +++ b/Duplicati/Library/Modules/Builtin/SendMail.cs @@ -1,39 +1,31 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; -using System.Collections; using System.Collections.Generic; using System.IO; using System.Linq; -using System.Net; using System.Text; -using System.Text.RegularExpressions; using Duplicati.Library.Interface; -using Duplicati.Library.Logging; -using Duplicati.Library.Utility; -using System.Net.NetworkInformation; -using System.Net.Sockets; using MailKit.Net.Smtp; using MimeKit; -using Duplicati.Library.Modules.Builtin.ResultSerialization; using DnsClient; namespace Duplicati.Library.Modules.Builtin @@ -277,7 +269,7 @@ namespace Duplicati.Library.Modules.Builtin lastServer = server; try { - using (MemoryStream ms = new MemoryStream()) + using (var ms = new MemoryStream()) { try { diff --git a/Duplicati/Library/Modules/Builtin/Strings.cs b/Duplicati/Library/Modules/Builtin/Strings.cs index 5d4750cc5..bcd66b666 100644 --- a/Duplicati/Library/Modules/Builtin/Strings.cs +++ b/Duplicati/Library/Modules/Builtin/Strings.cs @@ -231,6 +231,10 @@ You can supply multiple options with a comma separator, e.g. ""{0},{1}"". The sp public static string AcceptAnyCertificateShort { get { return LC.L(@"Accept any server certificate"); } } public static string AcceptSpecifiedCertificateLong { get { return LC.L(@"If your server certificate is reported as invalid (e.g. with self-signed certificates), you can supply the certificate hash (SHA1) to approve it anyway. The hash value must be entered in hex format without spaces or colons. You can enter multiple hashes separated by commas."); } } public static string AcceptSpecifiedCertificateShort { get { return LC.L(@"Optionally accept a known SSL certificate"); } } + public static string SendHttpRetriesLong { get { return LC.L(@"Use this option to set the number of retries to attempt if the HTTP request fails."); } } + public static string SendHttpRetriesShort { get { return LC.L(@"Set the number of retries"); } } + public static string SendHttpRetryDelayLong { get { return LC.L(@"Use this option to set the delay between retries."); } } + public static string SendHttpRetryDelayShort { get { return LC.L(@"Set the retry delay"); } } } internal static class ReportHelper diff --git a/Duplicati/Library/Utility/RetryHelper.cs b/Duplicati/Library/Utility/RetryHelper.cs index 38554e15a..1bbcaae3e 100644 --- a/Duplicati/Library/Utility/RetryHelper.cs +++ b/Duplicati/Library/Utility/RetryHelper.cs @@ -19,6 +19,8 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +#nullable enable + using System; using System.Threading; using System.Threading.Tasks; @@ -27,7 +29,7 @@ namespace Duplicati.Library.Utility; public static class RetryHelper { - public static async Task Retry(Func> action, int maxRetries, TimeSpan delay, CancellationToken token) + public static async Task Retry(Func> action, Action? errorCallback, int maxRetries, TimeSpan delay, CancellationToken token) { var attempt = 0; @@ -37,21 +39,26 @@ public static class RetryHelper { return await action(); } - catch + catch (Exception ex) { + attempt++; + errorCallback?.Invoke(ex, attempt); + if (token.IsCancellationRequested) throw; if (attempt >= maxRetries) throw; - - attempt++; } await Task.Delay(delay, token); } } + public static async Task Retry(Func> action, int maxRetries, TimeSpan delay, CancellationToken token) + => await Retry(action, null, maxRetries, delay, token); + public static async Task Retry(Func action, Action? errorCallback, int maxRetries, TimeSpan delay, CancellationToken token) + => await Retry(async () => { await action(); return true; }, errorCallback, maxRetries, delay, token); public static async Task Retry(Func action, int maxRetries, TimeSpan delay, CancellationToken token) - => await Retry(async () => { await action(); return true; }, maxRetries, delay, token); + => await Retry(async () => { await action(); return true; }, null, maxRetries, delay, token); } diff --git a/Duplicati/Library/Utility/Timeparser.cs b/Duplicati/Library/Utility/Timeparser.cs index cc505b468..64f9a23cc 100644 --- a/Duplicati/Library/Utility/Timeparser.cs +++ b/Duplicati/Library/Utility/Timeparser.cs @@ -1,27 +1,25 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; -using System.Collections.Generic; -using System.Text; namespace Duplicati.Library.Utility { @@ -34,7 +32,7 @@ namespace Duplicati.Library.Utility { public static TimeSpan ParseTimeSpan(string datestring) { - DateTime dt = new DateTime(0, DateTimeKind.Local); + var dt = new DateTime(0, DateTimeKind.Local); return ParseTimeInterval(datestring, dt) - dt; } @@ -48,16 +46,15 @@ namespace Duplicati.Library.Utility if (offset.Kind == DateTimeKind.Unspecified) offset = new DateTime(offset.Ticks, DateTimeKind.Local); - int multiplier = negate ? -1 : 1; + var multiplier = negate ? -1 : 1; if (string.IsNullOrEmpty(datestring)) return offset; - if (String.Equals(datestring.Trim(), "now", StringComparison.OrdinalIgnoreCase)) + if (string.Equals(datestring.Trim(), "now", StringComparison.OrdinalIgnoreCase)) return DateTime.Now; - long l; - if (long.TryParse(datestring, System.Globalization.NumberStyles.Integer, null, out l)) + if (long.TryParse(datestring, System.Globalization.NumberStyles.Integer, null, out var l)) return offset.AddSeconds(l * multiplier); if (DateTime.TryParse(datestring, System.Globalization.CultureInfo.CurrentCulture, System.Globalization.DateTimeStyles.AssumeLocal, out var t)) @@ -66,16 +63,15 @@ namespace Duplicati.Library.Utility if (Utility.TryDeserializeDateTime(datestring, out t)) return t; - char[] separators = new char[] { 's', 'm', 'h', 'D', 'W', 'M', 'Y' }; + var separators = new char[] { 's', 'm', 'h', 'D', 'W', 'M', 'Y' }; - int index = 0; - int previndex = 0; + int index; + var previndex = 0; while ((index = datestring.IndexOfAny(separators, previndex)) > 0) { - string partial = datestring.Substring(previndex, index - previndex).Trim(); - int factor; - if (!int.TryParse(partial, System.Globalization.NumberStyles.Integer, null, out factor)) + var partial = datestring.Substring(previndex, index - previndex).Trim(); + if (!int.TryParse(partial, System.Globalization.NumberStyles.Integer, null, out var factor)) throw new Exception(Strings.Timeparser.InvalidIntegerError(partial)); factor *= multiplier; @@ -134,11 +130,10 @@ namespace Duplicati.Library.Utility if (string.IsNullOrEmpty(datestring)) return offset; - if (String.Equals(datestring.Trim(), "now", StringComparison.OrdinalIgnoreCase)) + if (string.Equals(datestring.Trim(), "now", StringComparison.OrdinalIgnoreCase)) return DateTime.UtcNow; - long l; - if (long.TryParse(datestring, System.Globalization.NumberStyles.Integer, null, out l)) + if (long.TryParse(datestring, System.Globalization.NumberStyles.Integer, null, out var l)) return keepTimeOfDay ? timeZoneInfo.DSTAwareAddSeconds(offset, l * multiplier) : timeZoneInfo.DSTAwareAddSeconds(DateTime.UtcNow, l * multiplier); @@ -149,16 +144,15 @@ namespace Duplicati.Library.Utility if (Utility.TryDeserializeDateTime(datestring, out t)) return t; - char[] separators = ['s', 'm', 'h', 'D', 'W', 'M', 'Y']; + var separators = new char[] { 's', 'm', 'h', 'D', 'W', 'M', 'Y' }; int index; - int previndex = 0; + var previndex = 0; while ((index = datestring.IndexOfAny(separators, previndex)) > 0) { - string partial = datestring.Substring(previndex, index - previndex).Trim(); - int factor; - if (!int.TryParse(partial, System.Globalization.NumberStyles.Integer, null, out factor)) + var partial = datestring.Substring(previndex, index - previndex).Trim(); + if (!int.TryParse(partial, System.Globalization.NumberStyles.Integer, null, out var factor)) throw new Exception(Strings.Timeparser.InvalidIntegerError(partial)); factor *= multiplier; diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index 92a6ecc24..9264943d7 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -702,7 +702,7 @@ namespace Duplicati.Library.Utility /// /// The set of options to look for the setting in /// The value to look for in the settings - /// + /// The parsed value, or the default value (false). public static bool ParseBoolOption(IReadOnlyDictionary options, string value) { if (options.TryGetValue(value, out var opt)) @@ -711,6 +711,22 @@ namespace Duplicati.Library.Utility return false; } + /// + /// Parses an integer option from the option set, returning the default value if the option is not found or cannot be parsed + /// + /// The set of options to look for the setting in + /// The value to look for in the settings + /// The default value to return if there are no matches. + /// The parsed or default integer value. + public static TimeSpan ParseTimespanOption(IReadOnlyDictionary options, string value, string @default) + { + var opt = options.GetValueOrDefault(value); + if (string.IsNullOrWhiteSpace(opt)) + opt = @default; + + return Timeparser.ParseTimeSpan(opt); + } + /// /// Parses an enum found in the options dictionary ///