diff --git a/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs b/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs index de27a38a6..ea8f7df60 100644 --- a/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs +++ b/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs @@ -56,6 +56,15 @@ namespace Duplicati.Library.Backend.AzureBlob /// The option to specify the Azure access tier /// private const string AZURE_ACCESS_TIER_OPTION = "azure-access-tier"; + /// + /// The option to specify the number of internal retries + /// + private const string AZURE_INTERNAL_RETRIES_OPTION = "azure-internal-retries"; + + /// + /// The default number of internal retries + /// + public const int DEFAULT_INTERNAL_RETRIES = 3; /// /// The default storage classes that are considered archive classes @@ -106,7 +115,8 @@ namespace Duplicati.Library.Backend.AzureBlob ? null // Warning: The cast here is required to avoid implicit casting null to AccessTier : (AccessTier?)new AccessTier(accessTierValue); - _azureBlob = new AzureBlobWrapper(auth.Username!, auth.Password, sasToken, containerName, accessTier, archiveClasses, timeouts); + var internalRetries = Library.Utility.Utility.ParseIntOption(options, AZURE_INTERNAL_RETRIES_OPTION, DEFAULT_INTERNAL_RETRIES); + _azureBlob = new AzureBlobWrapper(auth.Username!, auth.Password, sasToken, containerName, accessTier, archiveClasses, timeouts, internalRetries); } /// @@ -190,9 +200,9 @@ namespace Duplicati.Library.Backend.AzureBlob CommandLineArgument.ArgumentType.Flags, Strings.AzureBlobBackend.ArchiveClassesDescriptionShort, Strings.AzureBlobBackend.ArchiveClassesDescriptionLong, - string.Join(",", DEFAULT_ARCHIVE_CLASSES.Select(x => x.ToString())), + string.Join(",", DEFAULT_ARCHIVE_CLASSES.Select(x => Library.Utility.Utility.FormatInvariantValue(x))), null, - ACCESS_TIERS.Select(x => x.ToString()).ToArray()), + ACCESS_TIERS.Select(x => Library.Utility.Utility.FormatInvariantValue(x)).ToArray()), new CommandLineArgument(AZURE_ACCESS_TIER_OPTION, CommandLineArgument.ArgumentType.String, Strings.AzureBlobBackend.AccessTierDescriptionShort, @@ -200,6 +210,11 @@ namespace Duplicati.Library.Backend.AzureBlob "", null, ACCESS_TIERS.Select(x => x.ToString()).ToArray()), + new CommandLineArgument(AZURE_INTERNAL_RETRIES_OPTION, + CommandLineArgument.ArgumentType.Integer, + Strings.AzureBlobBackend.InternalRetriesDescriptionShort, + Strings.AzureBlobBackend.InternalRetriesDescriptionLong, + Library.Utility.Utility.FormatInvariantValue(DEFAULT_INTERNAL_RETRIES)), .. TimeoutOptionsHelper.GetOptions() ]; } diff --git a/Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs b/Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs index 8d02c35da..1a3bec24d 100644 --- a/Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs +++ b/Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs @@ -27,6 +27,7 @@ using Duplicati.Library.Interface; using Uri = System.Uri; using Duplicati.Library.Utility.Options; using Duplicati.Library.Utility; +using Azure.Core.Pipeline; namespace Duplicati.Library.Backend.AzureBlob { @@ -64,18 +65,42 @@ namespace Duplicati.Library.Backend.AzureBlob /// The access tier assigned to blobs on upload. /// The storage classes that are considered archive classes. /// The timeout options. - public AzureBlobWrapper(string accountName, string? accessKey, string? sasToken, string containerName, AccessTier? accessTier, IReadOnlySet archiveClasses, TimeoutOptionsHelper.Timeouts timeouts) + /// The maximum number of retries for Azure operations. + public AzureBlobWrapper(string accountName, string? accessKey, string? sasToken, string containerName, AccessTier? accessTier, IReadOnlySet archiveClasses, TimeoutOptionsHelper.Timeouts timeouts, int maxRetries) { BlobServiceClient blobServiceClient; + var maxTicks = timeouts.ReadWriteTimeout.Ticks - TimeSpan.FromSeconds(1).Ticks; + + // If not infinite, then share the timeout across retries + var delay = maxTicks <= 0 + ? TimeSpan.FromSeconds(1) + : TimeSpan.FromTicks(maxTicks / Math.Max(1, maxRetries)); + + var maxDelay = maxTicks <= 0 + ? TimeSpan.FromSeconds(10) + : TimeSpan.FromTicks(maxTicks); + + var blobServiceOptions = new BlobClientOptions() + { + Retry = + { + MaxRetries = maxRetries, + Mode = Azure.Core.RetryMode.Exponential, + Delay = delay, + MaxDelay = maxDelay + }, + Transport = new HttpClientTransport(new HttpClient { Timeout = Timeout.InfiniteTimeSpan }) + }; + if (sasToken != null) { var sasUri = new Uri($"https://{accountName}.blob.core.windows.net/?{sasToken}"); - blobServiceClient = new BlobServiceClient(sasUri); + blobServiceClient = new BlobServiceClient(sasUri, blobServiceOptions); } else { var connectionString = $"DefaultEndpointsProtocol=https;AccountName={accountName};AccountKey={accessKey};EndpointSuffix=core.windows.net"; - blobServiceClient = new BlobServiceClient(connectionString); + blobServiceClient = new BlobServiceClient(connectionString, blobServiceOptions); } _accessTier = accessTier; diff --git a/Duplicati/Library/Backend/AzureBlob/Strings.cs b/Duplicati/Library/Backend/AzureBlob/Strings.cs index fe23657ac..e96d4a584 100644 --- a/Duplicati/Library/Backend/AzureBlob/Strings.cs +++ b/Duplicati/Library/Backend/AzureBlob/Strings.cs @@ -41,4 +41,6 @@ internal static class AzureBlobBackend public static string AccessTierDescriptionLong => LC.L(@"Use this option to specify the access tier. If this option is not used, the server will choose a default access tier."); public static string ArchiveClassesDescriptionShort => LC.L(@"The storage classes that are considered archive classes"); public static string ArchiveClassesDescriptionLong => LC.L(@"Use this option to specify what storage classes are considered archive storage classes. With this option it is possible to allow lifecycle policies to move data to cheaper storage classes and prevent Duplicati from accessing archived data."); + public static string InternalRetriesDescriptionShort => LC.L(@"The number of retries for Azure operations"); + public static string InternalRetriesDescriptionLong => LC.L(@"Use this option to specify the number of retries for Azure blob operations. Set to 0 to disable retries."); } \ No newline at end of file