Merge pull request #6008 from marceloduplicati/feature/azure-upgrade-and-fixes

Azure Library Upgrade and Fixes
This commit is contained in:
marceloduplicati
2025-03-10 09:36:32 -03:00
committed by GitHub
4 changed files with 138 additions and 172 deletions
@@ -19,12 +19,9 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Azure;
using Azure.Storage.Blobs.Models;
using Duplicati.Library.Interface;
using Microsoft.Azure.Storage.Shared.Protocol;
using System.Collections.Generic;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
namespace Duplicati.Library.Backend.AzureBlob
{
@@ -77,31 +74,22 @@ namespace Duplicati.Library.Backend.AzureBlob
{
throw new UserInformationException(Strings.AzureBlobBackend.NoAccessKeyOrSasToken, "AzureNoAccessKeyOrSasToken");
}
_azureBlob = new AzureBlobWrapper(storageAccountName, accessKey, sasToken, containerName);
}
public string DisplayName
{
get { return Strings.AzureBlobBackend.DisplayName; }
}
public string DisplayName => Strings.AzureBlobBackend.DisplayName;
public string ProtocolKey
{
get { return "azure"; }
}
public string ProtocolKey => "azure";
public IAsyncEnumerable<IFileEntry> ListAsync(CancellationToken cancelToken)
=> _azureBlob.ListContainerEntriesAsync(cancelToken);
public async Task PutAsync(string remotename, string localname, CancellationToken cancelToken)
{
using (var fs = File.Open(localname,
FileMode.Open, FileAccess.Read, FileShare.Read))
{
await PutAsync(remotename, fs, cancelToken);
}
await using var fs = File.Open(localname,
FileMode.Open, FileAccess.Read, FileShare.Read);
await PutAsync(remotename, fs, cancelToken).ConfigureAwait(false);
}
public Task PutAsync(string remotename, Stream input, CancellationToken cancelToken)
@@ -111,12 +99,10 @@ namespace Duplicati.Library.Backend.AzureBlob
public async Task GetAsync(string remotename, string localname, CancellationToken cancellationToken)
{
using (var fs = File.Open(localname,
await using var fs = File.Open(localname,
FileMode.Create, FileAccess.Write,
FileShare.None))
{
await GetAsync(remotename, fs, cancellationToken).ConfigureAwait(false);
}
FileShare.None);
await GetAsync(remotename, fs, cancellationToken).ConfigureAwait(false);
}
public Task GetAsync(string remotename, Stream output, CancellationToken cancellationToken)
@@ -159,13 +145,7 @@ namespace Duplicati.Library.Backend.AzureBlob
}
}
public string Description
{
get
{
return Strings.AzureBlobBackend.Description_v2;
}
}
public string Description => Strings.AzureBlobBackend.DescriptionV2;
public Task<string[]> GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(_azureBlob.DnsNames);
@@ -176,23 +156,27 @@ namespace Duplicati.Library.Backend.AzureBlob
{
return WrapWithExceptionHandler(_azureBlob.AddContainerAsync(cancellationToken));
}
/// <summary>
/// Wraps the task with exception handling
/// </summary>
private async Task WrapWithExceptionHandler(Task task)
{
try
{
await task.ConfigureAwait(false);
}
catch (Microsoft.WindowsAzure.Storage.StorageException e)
when (e.RequestInformation.HttpStatusCode == 404
|| e.RequestInformation.HttpStatusMessage == StorageErrorCodeStrings.ResourceNotFound)
catch (RequestFailedException e)
when (e.Status == 404
|| e.ErrorCode == BlobErrorCode.BlobNotFound
|| e.ErrorCode == BlobErrorCode.ResourceNotFound)
{
throw new FileMissingException(e.Message, e);
}
catch (Microsoft.WindowsAzure.Storage.StorageException e)
when (e.RequestInformation.HttpStatusMessage == StorageErrorCodeStrings.ContainerNotFound
|| e.RequestInformation.HttpStatusMessage == StorageErrorCodeStrings.ContainerBeingDeleted
|| e.RequestInformation.HttpStatusMessage == StorageErrorCodeStrings.ContainerDisabled)
catch (RequestFailedException e)
when (e.ErrorCode == BlobErrorCode.ContainerNotFound
|| e.ErrorCode == BlobErrorCode.ContainerBeingDeleted
|| e.ErrorCode == BlobErrorCode.ContainerDisabled)
{
throw new FolderMissingException(e.Message, e);
}
@@ -19,16 +19,12 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System.Collections.Generic;
using System.IO;
using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Models;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Blob;
using Uri = System.Uri;
namespace Duplicati.Library.Backend.AzureBlob
{
@@ -37,141 +33,133 @@ namespace Duplicati.Library.Backend.AzureBlob
/// </summary>
public class AzureBlobWrapper
{
private readonly string _containerName;
private readonly CloudBlobContainer _container;
private readonly OperationContext _operationContext;
// Note: May need metadata; need to test with Azure blobs
private const BlobListingDetails ListDetails = BlobListingDetails.None;
private readonly BlobContainerClient _container;
/// <summary>
/// Gets an array of DNS names associated with the blob container.
/// </summary>
/// <returns>An array of DNS hostnames for the primary and secondary URIs of the container.</returns>
public string[] DnsNames
{
get
{
var lst = new List<string>();
if (_container != null)
{
if (_container.Uri != null)
lst.Add(_container.Uri.Host);
if (_container.StorageUri != null)
{
if (_container.StorageUri.PrimaryUri != null)
lst.Add(_container.StorageUri.PrimaryUri.Host);
if (_container.StorageUri.SecondaryUri != null)
lst.Add(_container.StorageUri.SecondaryUri.Host);
}
}
if (_container != null && _container.Uri != null) lst.Add(_container.Uri.Host);
return lst.ToArray();
}
}
/// <summary>
/// Initializes a new instance of the AzureBlobWrapper class.
/// </summary>
/// <param name="accountName">The Azure storage account name.</param>
/// <param name="accessKey">The access key for the storage account.</param>
/// <param name="sasToken">The Shared Access Signature (SAS) token for authentication.</param>
/// <param name="containerName">The name of the blob container.</param>
public AzureBlobWrapper(string accountName, string accessKey, string sasToken, string containerName)
{
_operationContext = new()
{
CustomUserAgent = string.Format(
"APN/1.0 Duplicati/{0} AzureBlob/2.0 {1}",
System.Reflection.Assembly.GetExecutingAssembly().GetName().Version,
Microsoft.WindowsAzure.Storage.Shared.Protocol.Constants.HeaderConstants.UserAgent
)
};
string connectionString;
BlobServiceClient blobServiceClient;
if (sasToken != null)
{
connectionString = string.Format("DefaultEndpointsProtocol=https;AccountName={0};SharedAccessSignature={1}",
accountName, sasToken);
var sasUri = new Uri($"https://{accountName}.blob.core.windows.net/?{sasToken}");
blobServiceClient = new BlobServiceClient(sasUri);
}
else
{
connectionString = string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}",
accountName, accessKey);
var connectionString = $"DefaultEndpointsProtocol=https;AccountName={accountName};AccountKey={accessKey};EndpointSuffix=core.windows.net";
blobServiceClient = new BlobServiceClient(connectionString);
}
var storageAccount = CloudStorageAccount.Parse(connectionString);
var blobClient = storageAccount.CreateCloudBlobClient();
_containerName = containerName;
_container = blobClient.GetContainerReference(containerName);
_container = blobServiceClient.GetBlobContainerClient(containerName);
}
/// <summary>
/// Creates a new blob container asynchronously and sets its access permissions to private.
/// </summary>
/// <param name="cancellationToken">A token to monitor for cancellation requests.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
public async Task AddContainerAsync(CancellationToken cancellationToken)
{
await _container.CreateAsync(default, default, _operationContext, cancellationToken).ConfigureAwait(false);
await _container.SetPermissionsAsync(new BlobContainerPermissions { PublicAccess = BlobContainerPublicAccessType.Off }, default, default, _operationContext, cancellationToken).ConfigureAwait(false);
// Even though PublicAccessType.None is by default, we set it explicitly to highlight it.
await _container.CreateAsync(PublicAccessType.None, cancellationToken: cancellationToken).ConfigureAwait(false);
}
public virtual Task GetFileStreamAsync(string keyName, Stream target, CancellationToken cancellationToken)
/// <summary>
/// Downloads a blob to a stream asynchronously.
/// </summary>
/// <param name="keyName">The name of the blob to download.</param>
/// <param name="target">The stream to download the blob to.</param>
/// <param name="cancellationToken">A token to monitor for cancellation requests.</param>
/// <returns>A task that represents the asynchronous download operation.</returns>
public async Task GetFileStreamAsync(string keyName, Stream target, CancellationToken cancellationToken)
{
return _container.GetBlockBlobReference(keyName).DownloadToStreamAsync(target, default, default, _operationContext, cancellationToken);
var blobClient = _container.GetBlobClient(keyName);
await blobClient.DownloadToAsync(target, cancellationToken).ConfigureAwait(false);
}
public virtual Task AddFileStream(string keyName, Stream source, CancellationToken cancelToken)
/// <summary>
/// Uploads a stream to a blob asynchronously.
/// </summary>
/// <param name="keyName">The name to give the uploaded blob.</param>
/// <param name="source">The stream containing the data to upload.</param>
/// <param name="cancelToken">A token to monitor for cancellation requests.</param>
/// <returns>A task that represents the asynchronous upload operation.</returns>
public async Task AddFileStream(string keyName, Stream source, CancellationToken cancelToken)
{
return _container.GetBlockBlobReference(keyName).UploadFromStreamAsync(source, source.Length, default, default, _operationContext, cancelToken);
var blobClient = _container.GetBlobClient(keyName);
await blobClient.UploadAsync(source, true, cancelToken).ConfigureAwait(false);
}
public Task DeleteObjectAsync(string keyName, CancellationToken cancelToken)
/// <summary>
/// Deletes a blob if it exists asynchronously.
/// </summary>
/// <param name="keyName">The name of the blob to delete.</param>
/// <param name="cancelToken">A token to monitor for cancellation requests.</param>
public async Task DeleteObjectAsync(string keyName, CancellationToken cancelToken)
{
return _container.GetBlockBlobReference(keyName).DeleteIfExistsAsync(default, default, default, _operationContext, cancelToken);
var blobClient = _container.GetBlobClient(keyName);
await blobClient.DeleteIfExistsAsync(cancellationToken: cancelToken).ConfigureAwait(false);
}
private async IAsyncEnumerable<IListBlobItem> ListContainerBlobEntriesAsync([EnumeratorCancellation] CancellationToken cancelToken)
/// <summary>
/// List container files.
/// </summary>
/// <param name="cancelToken">A token to monitor for cancellation requests.</param>
/// <returns></returns>
/// <exception cref="FolderMissingException">Thrown when the container is not found</exception>
public virtual async IAsyncEnumerable<IFileEntry> ListContainerEntriesAsync([EnumeratorCancellation] CancellationToken cancelToken)
{
BlobResultSegment segment;
try
await using var blobEnumerator = _container.GetBlobsAsync().GetAsyncEnumerator(cancelToken);
while (true)
{
segment = await _container.ListBlobsSegmentedAsync(null, false, ListDetails, null, null, null, _operationContext, cancelToken).ConfigureAwait(false);
}
catch (StorageException ex)
{
if (ex.RequestInformation.HttpStatusCode == 404)
bool hasNext;
try
{
hasNext = await blobEnumerator.MoveNextAsync().ConfigureAwait(false);
}
catch (Azure.RequestFailedException ex) when (ex.Status == 404)
{
throw new FolderMissingException(ex);
}
throw;
}
foreach (var item in segment.Results)
yield return item;
while (segment.ContinuationToken != null)
{
segment = await _container.ListBlobsSegmentedAsync(null, false, ListDetails, null, segment.ContinuationToken, null, _operationContext, cancelToken).ConfigureAwait(false);
foreach (var item in segment.Results)
yield return item;
}
}
public virtual async IAsyncEnumerable<IFileEntry> ListContainerEntriesAsync([EnumeratorCancellation] CancellationToken cancelToken)
{
await foreach (var x in ListContainerBlobEntriesAsync(cancelToken).ConfigureAwait(false))
{
var absolutePath = x.StorageUri.PrimaryUri.AbsolutePath;
var containerSegment = string.Concat("/", _containerName, "/");
var blobName = absolutePath.Substring(absolutePath.IndexOf(
containerSegment, System.StringComparison.Ordinal) + containerSegment.Length);
var res = new FileEntry(Uri.UrlDecode(blobName.Replace("+", "%2B")));
try
if (!hasNext) break;
cancelToken.ThrowIfCancellationRequested();
if (blobEnumerator.Current is { } blob)
{
if (x is CloudBlockBlob cb)
{
var lastModified = new System.DateTime();
if (cb.Properties.LastModified != null)
lastModified = new System.DateTime(cb.Properties.LastModified.Value.Ticks, System.DateTimeKind.Utc);
res = new FileEntry(res.Name, cb.Properties.Length, lastModified, lastModified);
}
var blobName = Uri.UnescapeDataString(blob.Name.Replace("+", "%2B"));
var lastModified = blob.Properties.LastModified?.UtcDateTime ?? DateTime.UtcNow;
yield return new FileEntry(
blobName,
blob.Properties.ContentLength ?? 0,
lastModified,
lastModified
);
}
catch
{
// If the metadata fails to parse, return the basic entry
}
yield return res;
}
}
}
@@ -4,18 +4,11 @@
<TargetFramework>net8.0</TargetFramework>
<OutputType>Library</OutputType>
<Copyright>Copyright © 2025 Team Duplicati, MIT license</Copyright>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Azure.KeyVault.Core" Version="3.0.4" />
<PackageReference Include="Microsoft.Azure.KeyVault.WebKey" Version="3.0.4" />
<PackageReference Include="Microsoft.Azure.Storage.Blob" Version="11.0.1" />
<PackageReference Include="Microsoft.Azure.Storage.Common" Version="11.0.1" />
<PackageReference Include="Microsoft.Azure.Storage.File" Version="11.0.1" />
<PackageReference Include="Microsoft.Azure.Storage.Queue" Version="11.0.1" />
<PackageReference Include="Microsoft.Rest.ClientRuntime" Version="2.3.24" />
<PackageReference Include="Microsoft.Rest.ClientRuntime.Azure" Version="3.3.19" />
<PackageReference Include="WindowsAzure.Storage" Version="9.3.3" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="Azure.Storage.Blobs" Version="12.23.0" />
<PackageReference Include="Azure.Storage.Common" Version="12.22.0" />
</ItemGroup>
<ItemGroup>
+21 -20
View File
@@ -20,23 +20,24 @@
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Localization.Short;
namespace Duplicati.Library.Backend.AzureBlob.Strings {
internal static class AzureBlobBackend {
public static string Description_v2 { get { return LC.L(@"This backend can read and write data to Azure blob storage. Allowed format is ""azure://bucketname""."); } }
public static string DisplayName { get { return LC.L(@"Azure blob"); } }
public static string ContainerNameDescriptionLong { get { return LC.L(@"All files will be written to the container specified."); } }
public static string ContainerNameDescriptionShort { get { return LC.L(@"The name of the storage container"); } }
public static string NoStorageAccountName { get { return LC.L(@"No Azure storage account name given"); } }
public static string StorageAccountNameDescriptionLong { get { return LC.L(@"The Azure storage account name which can be obtained by clicking the ""Manage Access Keys"" button on the storage account dashboard."); } }
public static string StorageAccountNameDescriptionShort { get { return LC.L(@"The storage account name"); } }
public static string AccessKeyDescriptionLong { get { return LC.L(@"The Azure access key which can be obtained by clicking the ""Manage Access Keys"" button on the storage account dashboard."); } }
public static string AccessKeyDescriptionShort { get { return LC.L(@"The access key"); } }
public static string SasTokenDescriptionLong { get { return LC.L(@"The Azure shared access signature (SAS) token which can be obtained by selecting the ""Shared access signature"" blade on the storage account dashboard, or inside a container blade."); } }
public static string SasTokenDescriptionShort { get { return LC.L(@"The SAS token"); } }
public static string NoAccessKeyOrSasToken { get { return LC.L(@"No Azure access key or SAS token given"); } }
public static string AuthPasswordDescriptionLong { get { return LC.L(@"The password used to connect to the server. This may also be supplied as the environment variable ""AUTH_PASSWORD""."); } }
public static string AuthPasswordDescriptionShort { get { return LC.L(@"Supply the password used to connect to the server"); } }
public static string AuthUsernameDescriptionLong { get { return LC.L(@"The username used to connect to the server. This may also be supplied as the environment variable ""AUTH_USERNAME""."); } }
public static string AuthUsernameDescriptionShort { get { return LC.L(@"Supply the username used to connect to the server"); } }
}
}
namespace Duplicati.Library.Backend.AzureBlob.Strings;
internal static class AzureBlobBackend {
public static string DescriptionV2 => LC.L(@"This backend can read and write data to Azure blob storage. Allowed format is ""azure://bucketname"".");
public static string DisplayName => LC.L(@"Azure blob");
public static string ContainerNameDescriptionLong => LC.L(@"All files will be written to the container specified.");
public static string ContainerNameDescriptionShort => LC.L(@"The name of the storage container");
public static string NoStorageAccountName => LC.L(@"No Azure storage account name given");
public static string StorageAccountNameDescriptionLong => LC.L(@"The Azure storage account name which can be obtained by clicking the ""Manage Access Keys"" button on the storage account dashboard.");
public static string StorageAccountNameDescriptionShort => LC.L(@"The storage account name");
public static string AccessKeyDescriptionLong => LC.L(@"The Azure access key which can be obtained by clicking the ""Manage Access Keys"" button on the storage account dashboard.");
public static string AccessKeyDescriptionShort => LC.L(@"The access key");
public static string SasTokenDescriptionLong => LC.L(@"The Azure shared access signature (SAS) token which can be obtained by selecting the ""Shared access signature"" blade on the storage account dashboard, or inside a container blade.");
public static string SasTokenDescriptionShort => LC.L(@"The SAS token");
public static string NoAccessKeyOrSasToken => LC.L(@"No Azure access key or SAS token given");
public static string AuthPasswordDescriptionLong => LC.L(@"The password used to connect to the server. This may also be supplied as the environment variable ""AUTH_PASSWORD"".");
public static string AuthPasswordDescriptionShort => LC.L(@"Supply the password used to connect to the server");
public static string AuthUsernameDescriptionLong => LC.L(@"The username used to connect to the server. This may also be supplied as the environment variable ""AUTH_USERNAME"".");
public static string AuthUsernameDescriptionShort => LC.L(@"Supply the username used to connect to the server");
}