Merge pull request #6632 from duplicati/feature/timeouts-for-azure-blob-storage

Added override for AzureBlobStorage's internal timeout handling.

Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs

@@ -56,6 +56,15 @@ namespace Duplicati.Library.Backend.AzureBlob
         /// The option to specify the Azure access tier
         /// </summary>
         private const string AZURE_ACCESS_TIER_OPTION = "azure-access-tier";
+        /// <summary>
+        /// The option to specify the number of internal retries
+        /// </summary>
+        private const string AZURE_INTERNAL_RETRIES_OPTION = "azure-internal-retries";
+
+        /// <summary>
+        /// The default number of internal retries
+        /// </summary>
+        public const int DEFAULT_INTERNAL_RETRIES = 3;
 
         /// <summary>
         /// The default storage classes that are considered archive classes
@@ -106,7 +115,8 @@ namespace Duplicati.Library.Backend.AzureBlob
                 ? null
                 // Warning: The cast here is required to avoid implicit casting null to AccessTier
                 : (AccessTier?)new AccessTier(accessTierValue);
-            _azureBlob = new AzureBlobWrapper(auth.Username!, auth.Password, sasToken, containerName, accessTier, archiveClasses, timeouts);
+            var internalRetries = Library.Utility.Utility.ParseIntOption(options, AZURE_INTERNAL_RETRIES_OPTION, DEFAULT_INTERNAL_RETRIES);
+            _azureBlob = new AzureBlobWrapper(auth.Username!, auth.Password, sasToken, containerName, accessTier, archiveClasses, timeouts, internalRetries);
         }
 
         /// <summary>
@@ -190,9 +200,9 @@ namespace Duplicati.Library.Backend.AzureBlob
                         CommandLineArgument.ArgumentType.Flags,
                         Strings.AzureBlobBackend.ArchiveClassesDescriptionShort,
                         Strings.AzureBlobBackend.ArchiveClassesDescriptionLong,
-                        string.Join(",", DEFAULT_ARCHIVE_CLASSES.Select(x => x.ToString())),
+                        string.Join(",", DEFAULT_ARCHIVE_CLASSES.Select(x => Library.Utility.Utility.FormatInvariantValue(x))),
                         null,
-                        ACCESS_TIERS.Select(x => x.ToString()).ToArray()),
+                        ACCESS_TIERS.Select(x => Library.Utility.Utility.FormatInvariantValue(x)).ToArray()),
                     new CommandLineArgument(AZURE_ACCESS_TIER_OPTION,
                         CommandLineArgument.ArgumentType.String,
                         Strings.AzureBlobBackend.AccessTierDescriptionShort,
@@ -200,6 +210,11 @@ namespace Duplicati.Library.Backend.AzureBlob
                         "",
                         null,
                         ACCESS_TIERS.Select(x => x.ToString()).ToArray()),
+                    new CommandLineArgument(AZURE_INTERNAL_RETRIES_OPTION,
+                        CommandLineArgument.ArgumentType.Integer,
+                        Strings.AzureBlobBackend.InternalRetriesDescriptionShort,
+                        Strings.AzureBlobBackend.InternalRetriesDescriptionLong,
+                        Library.Utility.Utility.FormatInvariantValue(DEFAULT_INTERNAL_RETRIES)),
                     .. TimeoutOptionsHelper.GetOptions()
                 ];
             }

Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs

@@ -27,6 +27,7 @@ using Duplicati.Library.Interface;
 using Uri = System.Uri;
 using Duplicati.Library.Utility.Options;
 using Duplicati.Library.Utility;
+using Azure.Core.Pipeline;
 
 namespace Duplicati.Library.Backend.AzureBlob
 {
@@ -64,18 +65,42 @@ namespace Duplicati.Library.Backend.AzureBlob
         /// <param name="accessTier">The access tier assigned to blobs on upload.</param>
         /// <param name="archiveClasses">The storage classes that are considered archive classes.</param>
         /// <param name="timeouts">The timeout options.</param>
-        public AzureBlobWrapper(string accountName, string? accessKey, string? sasToken, string containerName, AccessTier? accessTier, IReadOnlySet<AccessTier> archiveClasses, TimeoutOptionsHelper.Timeouts timeouts)
+        /// <param name="maxRetries">The maximum number of retries for Azure operations.</param>
+        public AzureBlobWrapper(string accountName, string? accessKey, string? sasToken, string containerName, AccessTier? accessTier, IReadOnlySet<AccessTier> archiveClasses, TimeoutOptionsHelper.Timeouts timeouts, int maxRetries)
         {
             BlobServiceClient blobServiceClient;
+            var maxTicks = timeouts.ReadWriteTimeout.Ticks - TimeSpan.FromSeconds(1).Ticks;
+
+            // If not infinite, then share the timeout across retries
+            var delay = maxTicks <= 0
+                ? TimeSpan.FromSeconds(1)
+                : TimeSpan.FromTicks(maxTicks / Math.Max(1, maxRetries));
+
+            var maxDelay = maxTicks <= 0
+                ? TimeSpan.FromSeconds(10)
+                : TimeSpan.FromTicks(maxTicks);
+
+            var blobServiceOptions = new BlobClientOptions()
+            {
+                Retry =
+                {
+                    MaxRetries = maxRetries,
+                    Mode = Azure.Core.RetryMode.Exponential,
+                    Delay = delay,
+                    MaxDelay = maxDelay
+                },
+                Transport = new HttpClientTransport(new HttpClient { Timeout = Timeout.InfiniteTimeSpan })
+            };
+
             if (sasToken != null)
             {
                 var sasUri = new Uri($"https://{accountName}.blob.core.windows.net/?{sasToken}");
-                blobServiceClient = new BlobServiceClient(sasUri);
+                blobServiceClient = new BlobServiceClient(sasUri, blobServiceOptions);
             }
             else
             {
                 var connectionString = $"DefaultEndpointsProtocol=https;AccountName={accountName};AccountKey={accessKey};EndpointSuffix=core.windows.net";
-                blobServiceClient = new BlobServiceClient(connectionString);
+                blobServiceClient = new BlobServiceClient(connectionString, blobServiceOptions);
             }
 
             _accessTier = accessTier;

Duplicati/Library/Backend/AzureBlob/Strings.cs

@@ -41,4 +41,6 @@ internal static class AzureBlobBackend
     public static string AccessTierDescriptionLong => LC.L(@"Use this option to specify the access tier. If this option is not used, the server will choose a default access tier.");
     public static string ArchiveClassesDescriptionShort => LC.L(@"The storage classes that are considered archive classes");
     public static string ArchiveClassesDescriptionLong => LC.L(@"Use this option to specify what storage classes are considered archive storage classes. With this option it is possible to allow lifecycle policies to move data to cheaper storage classes and prevent Duplicati from accessing archived data.");
+    public static string InternalRetriesDescriptionShort => LC.L(@"The number of retries for Azure operations");
+    public static string InternalRetriesDescriptionLong => LC.L(@"Use this option to specify the number of retries for Azure blob operations. Set to 0 to disable retries.");
 }