Merge branch 'master' into feature/fix-decoding-windows-paths

This commit is contained in:
Kenneth Skovhede
2025-03-25 08:55:06 +01:00
committed by GitHub
167 changed files with 4994 additions and 4918 deletions
+1 -1
View File
@@ -164,7 +164,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.Backend.p
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.Crashlog", "Duplicati\Library\Crashlog\Duplicati.Library.Crashlog.csproj", "{8ACA2736-3C69-4FA5-BCF9-5EDF50CAF332}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.Backend.CIFS", "Duplicati\Library\Backend\CIFS\Duplicati.Library.Backend.CIFS.csproj", "{836E0557-B40C-4DC7-9A2A-5C062F9ACC6B}"
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.Backend.SMB", "Duplicati\Library\Backend\SMB\Duplicati.Library.Backend.SMB.csproj", "{836E0557-B40C-4DC7-9A2A-5C062F9ACC6B}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.SourceProviders", "Duplicati\Library\SourceProviders\Duplicati.Library.SourceProviders.csproj", "{8A2C7A9F-3EC8-4DE5-A9FA-9E4BCF955EF3}"
EndProject
@@ -22,13 +22,10 @@ using Aliyun.OSS;
using Aliyun.OSS.Common;
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using System;
using System.Collections.Generic;
using System.IO;
using Duplicati.Library.Utility;
using Duplicati.Library.Utility.Options;
using System.Net;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
namespace Duplicati.Library.Backend.AliyunOSS
{
@@ -46,42 +43,33 @@ namespace Duplicati.Library.Backend.AliyunOSS
private const string OSS_ACCESS_KEY_ID = "oss-access-key-id";
private const string OSS_ACCESS_KEY_SECRET = "oss-access-key-secret";
private AliyunOSSOptions _ossOptions;
private readonly AliyunOSSOptions _ossOptions;
private readonly TimeoutOptionsHelper.Timeouts _timeouts;
public OSS()
{ }
public OSS(string url, Dictionary<string, string> options)
{
_ossOptions = new AliyunOSSOptions();
_ossOptions = null!;
_timeouts = null!;
}
public OSS(string url, Dictionary<string, string?> options)
{
_timeouts = TimeoutOptionsHelper.Parse(options);
var uri = new Utility.Uri(url?.Trim());
var prefix = uri.HostAndPath?.TrimPath();
if (!string.IsNullOrEmpty(prefix))
{
_ossOptions.Path = prefix;
}
var auth = AuthOptionsHelper.ParseWithAlias(options, uri, OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY_SECRET)
.RequireCredentials();
if (options.ContainsKey(OSS_ACCESS_KEY_ID))
_ossOptions = new AliyunOSSOptions()
{
_ossOptions.AccessKeyId = options[OSS_ACCESS_KEY_ID];
}
if (options.ContainsKey(OSS_ACCESS_KEY_SECRET))
{
_ossOptions.AccessKeySecret = options[OSS_ACCESS_KEY_SECRET];
}
if (options.ContainsKey(OSS_BUCKET_NAME))
{
_ossOptions.BucketName = options[OSS_BUCKET_NAME];
}
if (options.ContainsKey(OSS_ENDPOINT))
{
_ossOptions.Endpoint = options[OSS_ENDPOINT];
}
BucketName = options.GetValueOrDefault(OSS_BUCKET_NAME),
Endpoint = options.GetValueOrDefault(OSS_ENDPOINT),
AccessKeyId = auth.Username!,
AccessKeySecret = auth.Password!,
Path = prefix ?? string.Empty
};
}
private OssClient GetClient(bool isUseNewServiceClient = true)
@@ -122,11 +110,11 @@ namespace Duplicati.Library.Backend.AliyunOSS
ObjectListing result;
try
{
result = await Task.Run(() => client.ListObjects(listObjectsRequest), cancelToken).ConfigureAwait(ConfigureAwaitOptions.ForceYielding);
result = await Utility.Utility.WithTimeout(_timeouts.ListTimeout, cancelToken, async ct => await Task.Run(() => client.ListObjects(listObjectsRequest), ct).ConfigureAwait(ConfigureAwaitOptions.ForceYielding))
.ConfigureAwait(false);
if (result.HttpStatusCode != HttpStatusCode.OK)
{
throw new Exception(result.HttpStatusCode.ToString());
}
}
catch (OssException ex)
@@ -164,29 +152,27 @@ namespace Duplicati.Library.Backend.AliyunOSS
await GetAsync(remotename, fs, cancelToken).ConfigureAwait(false);
}
public Task DeleteAsync(string remotename, CancellationToken cancelToken)
public async Task DeleteAsync(string remotename, CancellationToken cancelToken)
{
try
{
var client = GetClient();
var bucketName = _ossOptions.BucketName;
var objectName = $"{_ossOptions.Path.TrimPath()}/{remotename}".TrimPath();
var res = client.DeleteObject(bucketName, objectName);
if (res?.HttpStatusCode != HttpStatusCode.OK)
await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancelToken, ct =>
{
Logging.Log.WriteInformationMessage(LOGTAG, "Delete", "Delete object failed. it may have been deleted");
}
var res = client.DeleteObject(bucketName, objectName);
if (res?.HttpStatusCode != HttpStatusCode.OK)
Logging.Log.WriteInformationMessage(LOGTAG, "Delete", "Delete object failed. it may have been deleted");
return Task.CompletedTask;
});
}
catch (Exception ex)
{
Logging.Log.WriteErrorMessage(LOGTAG, "Delete", ex, "Delete object failed. {0}", ex.Message);
throw;
}
return Task.CompletedTask;
}
public Task TestAsync(CancellationToken cancelToken)
@@ -212,8 +198,9 @@ namespace Duplicati.Library.Backend.AliyunOSS
var client = GetClient();
try
{
using var timeoutStream = stream.ObserveReadTimeout(_timeouts.ReadWriteTimeout, false);
var objectResult = await Task.Factory.FromAsync(
(cb, state) => client.BeginPutObject(bucketName, objectName, stream, cb, state),
(cb, state) => client.BeginPutObject(bucketName, objectName, timeoutStream, cb, state),
client.EndPutObject,
null).ConfigureAwait(false);
@@ -230,23 +217,24 @@ namespace Duplicati.Library.Backend.AliyunOSS
public async Task GetAsync(string remotename, Stream stream, CancellationToken cancelToken)
{
var bucketName = _ossOptions.BucketName;
var objectName = $"{_ossOptions.Path.TrimPath()}/{remotename}".TrimPath();
var client = GetClient(false);
try
{
var obj = await Task.Factory.FromAsync(
(cb, state) => client.BeginGetObject(bucketName, objectName, null, null),
client.EndGetObject,
null).ConfigureAwait(false);
var obj = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancelToken, async ct
=> await Task.Factory.FromAsync(
(cb, state) => client.BeginGetObject(bucketName, objectName, null, null),
client.EndGetObject,
null).ConfigureAwait(false)
).ConfigureAwait(false);
if (obj.HttpStatusCode != HttpStatusCode.OK)
throw new Exception("Get failed");
using (var requestStream = obj.Content)
await Library.Utility.Utility.CopyStreamAsync(requestStream, stream, cancelToken).ConfigureAwait(false);
using (var timeoutStream = requestStream.ObserveWriteTimeout(_timeouts.ReadWriteTimeout, false))
await Utility.Utility.CopyStreamAsync(requestStream, timeoutStream, cancelToken).ConfigureAwait(false);
}
catch (Exception ex)
{
@@ -274,15 +262,15 @@ namespace Duplicati.Library.Backend.AliyunOSS
// copy file
var req = new CopyObjectRequest(sourceBucket, sourceObject, targetBucket, targetObject);
var res = await Task.Factory.FromAsync(
(cb, state) => client.BeginCopyObject(req, cb, state),
client.EndCopyResult,
null).ConfigureAwait(false);
var res = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancelToken, async ct
=> await Task.Factory.FromAsync(
(cb, state) => client.BeginCopyObject(req, cb, state),
client.EndCopyResult,
null).ConfigureAwait(false)
).ConfigureAwait(false);
if (res?.HttpStatusCode != HttpStatusCode.OK)
{
throw new Exception("file rename failed");
}
// del old file
await DeleteAsync(oldname, cancelToken).ConfigureAwait(false);
@@ -310,10 +298,11 @@ namespace Duplicati.Library.Backend.AliyunOSS
get
{
return new List<ICommandLineArgument>([
new CommandLineArgument(OSS_ACCESS_KEY_ID, CommandLineArgument.ArgumentType.String, Strings.OSSBackend.OSSAccessKeyIdDescriptionShort, Strings.OSSBackend.OSSAccessKeyIdDescriptionLong),
new CommandLineArgument(OSS_ACCESS_KEY_SECRET, CommandLineArgument.ArgumentType.Password, Strings.OSSBackend.OSSAccessKeySecretDescriptionShort, Strings.OSSBackend.OSSAccessKeySecretDescriptionLong),
new CommandLineArgument(OSS_ACCESS_KEY_ID, CommandLineArgument.ArgumentType.String, Strings.OSSBackend.OSSAccessKeyIdDescriptionShort, Strings.OSSBackend.OSSAccessKeyIdDescriptionLong, null, [AuthOptionsHelper.AuthUsernameOption]),
new CommandLineArgument(OSS_ACCESS_KEY_SECRET, CommandLineArgument.ArgumentType.Password, Strings.OSSBackend.OSSAccessKeySecretDescriptionShort, Strings.OSSBackend.OSSAccessKeySecretDescriptionLong, null, [AuthOptionsHelper.AuthPasswordOption]),
new CommandLineArgument(OSS_BUCKET_NAME, CommandLineArgument.ArgumentType.String, Strings.OSSBackend.OSSBucketNameDescriptionShort, Strings.OSSBackend.OSSBucketNameDescriptionLong),
new CommandLineArgument(OSS_ENDPOINT, CommandLineArgument.ArgumentType.String, Strings.OSSBackend.OSSEndpointDescriptionShort, Strings.OSSBackend.OSSEndpointDescriptionLong)
new CommandLineArgument(OSS_ENDPOINT, CommandLineArgument.ArgumentType.String, Strings.OSSBackend.OSSEndpointDescriptionShort, Strings.OSSBackend.OSSEndpointDescriptionLong),
.. TimeoutOptionsHelper.GetOptions()
]);
}
}
@@ -31,19 +31,19 @@ namespace Duplicati.Library.Backend.AliyunOSS
/// A bucket is a container used for storing objects (Object), and all objects must belong to a bucket.
/// 存储空间是您用于存储对象(Object)的容器,所有的对象都必须隶属于某个存储空间。
/// </summary>
public string BucketName { get; set; }
public required string? BucketName { get; set; }
/// <summary>
/// AccessKeyId is used to identify the user. AccessKeySecret is the key used by the user to encrypt signature strings and by OSS to verify signature strings. The AccessKeySecret must be kept confidential.
/// AccessKeyId用于标识用户,AccessKeySecret是用户用于加密签名字符串和OSS用来验证签名字符串的密钥,其中AccessKeySecret 必须保密。
/// </summary>
public string AccessKeyId { get; set; }
public required string AccessKeyId { get; set; }
/// <summary>
/// AccessKeyId is used to identify the user. AccessKeySecret is the key used by the user to encrypt signature strings and by OSS to verify signature strings. The AccessKeySecret must be kept confidential.
/// AccessKeyId用于标识用户,AccessKeySecret是用户用于加密签名字符串和OSS用来验证签名字符串的密钥,其中AccessKeySecret 必须保密。
/// </summary>
public string AccessKeySecret { get; set; }
public required string AccessKeySecret { get; set; }
/// <summary>
/// Endpoint represents the access domain name for OSS external services.
@@ -54,13 +54,13 @@ namespace Duplicati.Library.Backend.AliyunOSS
/// oss-cn-hongkong.aliyuncs.com
/// oss-us-west-1.aliyuncs.com
/// </summary>
public string Endpoint { get; set; }
public required string? Endpoint { get; set; }
/// <summary>
/// A path or subfolder in a bucket
/// 桶中的路径或子文件夹
/// </summary>
public string Path { get; set; }
public required string Path { get; set; }
}
}
@@ -4,6 +4,9 @@
<OutputType>Library</OutputType>
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
<Copyright>Copyright © 2025 Team Duplicati, MIT license</Copyright>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<RootNamespace>Duplicati.Library.Backend.AliyunOSS</RootNamespace>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\..\Common\Duplicati.Library.Common.csproj" />
@@ -28,7 +28,8 @@ namespace Duplicati.Library.Backend.AliyunOSS
/// </summary>
/// <param name="path">The path to trim.</param>
/// <returns>The trimmed path.</returns>
public static string TrimPath(this string path)
[return: System.Diagnostics.CodeAnalysis.NotNullIfNotNull("path")]
public static string? TrimPath(this string? path)
{
return path?.Trim().Trim('/').Trim('\\').Trim('/').Trim();
}
@@ -22,6 +22,7 @@
using Azure;
using Azure.Storage.Blobs.Models;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility.Options;
namespace Duplicati.Library.Backend.AzureBlob
{
@@ -35,47 +36,29 @@ namespace Duplicati.Library.Backend.AzureBlob
// This constructor is needed by the BackendLoader.
public AzureBlobBackend()
{
_azureBlob = null!;
}
// ReSharper disable once UnusedMember.Global
// This constructor is needed by the BackendLoader.
public AzureBlobBackend(string url, Dictionary<string, string> options)
public AzureBlobBackend(string url, Dictionary<string, string?> options)
{
var uri = new Utility.Uri(url);
uri.RequireHost();
string storageAccountName = null;
string accessKey = null;
string sasToken = null;
string containerName = uri.Host.ToLowerInvariant();
var containerName = uri.Host.ToLowerInvariant();
if (options.ContainsKey("auth-username"))
storageAccountName = options["auth-username"];
if (options.ContainsKey("auth-password"))
accessKey = options["auth-password"];
var auth = AuthOptionsHelper.ParseWithAlias(options, uri, "azure-account-name", "azure-access-key");
var timeouts = TimeoutOptionsHelper.Parse(options);
if (options.ContainsKey("azure-account-name"))
storageAccountName = options["azure-account-name"];
if (options.ContainsKey("azure-access-key"))
accessKey = options["azure-access-key"];
if (options.ContainsKey("azure-access-sas-token"))
sasToken = options["azure-access-sas-token"];
if (!string.IsNullOrEmpty(uri.Username))
storageAccountName = uri.Username;
if (!string.IsNullOrEmpty(uri.Password))
accessKey = uri.Password;
if (string.IsNullOrWhiteSpace(storageAccountName))
{
var sasToken = options.GetValueOrDefault("azure-access-sas-token");
if (!auth.HasUsername)
throw new UserInformationException(Strings.AzureBlobBackend.NoStorageAccountName, "AzureNoAccountName");
}
if (string.IsNullOrWhiteSpace(accessKey) && string.IsNullOrWhiteSpace(sasToken))
{
if (!auth.HasPassword && string.IsNullOrWhiteSpace(sasToken))
throw new UserInformationException(Strings.AzureBlobBackend.NoAccessKeyOrSasToken, "AzureNoAccessKeyOrSasToken");
}
_azureBlob = new AzureBlobWrapper(storageAccountName, accessKey, sasToken, containerName);
_azureBlob = new AzureBlobWrapper(auth.Username!, auth.Password, sasToken, containerName, timeouts);
}
public string DisplayName => Strings.AzureBlobBackend.DisplayName;
@@ -119,29 +102,25 @@ namespace Duplicati.Library.Backend.AzureBlob
{
get
{
return new List<ICommandLineArgument>(new ICommandLineArgument[] {
return [
new CommandLineArgument("azure-account-name",
CommandLineArgument.ArgumentType.String,
Strings.AzureBlobBackend.StorageAccountNameDescriptionShort,
Strings.AzureBlobBackend.StorageAccountNameDescriptionLong),
Strings.AzureBlobBackend.StorageAccountNameDescriptionLong,
null,
[AuthOptionsHelper.AuthUsernameOption]),
new CommandLineArgument("azure-access-key",
CommandLineArgument.ArgumentType.Password,
Strings.AzureBlobBackend.AccessKeyDescriptionShort,
Strings.AzureBlobBackend.AccessKeyDescriptionLong),
Strings.AzureBlobBackend.AccessKeyDescriptionLong,
null,
[AuthOptionsHelper.AuthPasswordOption]),
new CommandLineArgument("azure-access-sas-token",
CommandLineArgument.ArgumentType.Password,
Strings.AzureBlobBackend.SasTokenDescriptionShort,
Strings.AzureBlobBackend.SasTokenDescriptionLong),
new CommandLineArgument("auth-password",
CommandLineArgument.ArgumentType.Password,
Strings.AzureBlobBackend.AuthPasswordDescriptionShort,
Strings.AzureBlobBackend.AuthPasswordDescriptionLong),
new CommandLineArgument("auth-username",
CommandLineArgument.ArgumentType.String,
Strings.AzureBlobBackend.AuthUsernameDescriptionShort,
Strings.AzureBlobBackend.AuthUsernameDescriptionLong)
});
.. TimeoutOptionsHelper.GetOptions()
];
}
}
@@ -156,7 +135,7 @@ namespace Duplicati.Library.Backend.AzureBlob
{
return WrapWithExceptionHandler(_azureBlob.AddContainerAsync(cancellationToken));
}
/// <summary>
/// Wraps the task with exception handling
/// </summary>
@@ -25,6 +25,8 @@ using System.Runtime.CompilerServices;
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Uri = System.Uri;
using Duplicati.Library.Utility.Options;
using Duplicati.Library.Utility;
namespace Duplicati.Library.Backend.AzureBlob
{
@@ -34,6 +36,7 @@ namespace Duplicati.Library.Backend.AzureBlob
public class AzureBlobWrapper
{
private readonly BlobContainerClient _container;
private readonly TimeoutOptionsHelper.Timeouts _timeouts;
/// <summary>
/// Gets an array of DNS names associated with the blob container.
@@ -56,7 +59,8 @@ namespace Duplicati.Library.Backend.AzureBlob
/// <param name="accessKey">The access key for the storage account.</param>
/// <param name="sasToken">The Shared Access Signature (SAS) token for authentication.</param>
/// <param name="containerName">The name of the blob container.</param>
public AzureBlobWrapper(string accountName, string accessKey, string sasToken, string containerName)
/// <param name="timeouts">The timeout options.</param>
public AzureBlobWrapper(string accountName, string? accessKey, string? sasToken, string containerName, TimeoutOptionsHelper.Timeouts timeouts)
{
BlobServiceClient blobServiceClient;
if (sasToken != null)
@@ -71,6 +75,7 @@ namespace Duplicati.Library.Backend.AzureBlob
}
_container = blobServiceClient.GetBlobContainerClient(containerName);
_timeouts = timeouts;
}
/// <summary>
@@ -80,8 +85,10 @@ namespace Duplicati.Library.Backend.AzureBlob
/// <returns>A task that represents the asynchronous operation.</returns>
public async Task AddContainerAsync(CancellationToken cancellationToken)
{
// Even though PublicAccessType.None is by default, we set it explicitly to highlight it.
await _container.CreateAsync(PublicAccessType.None, cancellationToken: cancellationToken).ConfigureAwait(false);
await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct =>
// Even though PublicAccessType.None is by default, we set it explicitly to highlight it.
await _container.CreateAsync(PublicAccessType.None, cancellationToken: ct).ConfigureAwait(false)
).ConfigureAwait(false);
}
/// <summary>
@@ -94,6 +101,7 @@ namespace Duplicati.Library.Backend.AzureBlob
public async Task GetFileStreamAsync(string keyName, Stream target, CancellationToken cancellationToken)
{
var blobClient = _container.GetBlobClient(keyName);
using var timeoutStream = target.ObserveWriteTimeout(_timeouts.ReadWriteTimeout, false);
await blobClient.DownloadToAsync(target, cancellationToken).ConfigureAwait(false);
}
@@ -107,7 +115,8 @@ namespace Duplicati.Library.Backend.AzureBlob
public async Task AddFileStream(string keyName, Stream source, CancellationToken cancelToken)
{
var blobClient = _container.GetBlobClient(keyName);
await blobClient.UploadAsync(source, true, cancelToken).ConfigureAwait(false);
using var timeoutStream = source.ObserveReadTimeout(_timeouts.ReadWriteTimeout, false);
await blobClient.UploadAsync(timeoutStream, true, cancelToken).ConfigureAwait(false);
}
/// <summary>
@@ -118,9 +127,11 @@ namespace Duplicati.Library.Backend.AzureBlob
public async Task DeleteObjectAsync(string keyName, CancellationToken cancelToken)
{
var blobClient = _container.GetBlobClient(keyName);
await blobClient.DeleteIfExistsAsync(cancellationToken: cancelToken).ConfigureAwait(false);
await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancelToken, async ct =>
await blobClient.DeleteIfExistsAsync(cancellationToken: ct).ConfigureAwait(false)
).ConfigureAwait(false);
}
/// <summary>
/// List container files.
/// </summary>
@@ -130,29 +141,31 @@ namespace Duplicati.Library.Backend.AzureBlob
public virtual async IAsyncEnumerable<IFileEntry> ListContainerEntriesAsync([EnumeratorCancellation] CancellationToken cancelToken)
{
await using var blobEnumerator = _container.GetBlobsAsync().GetAsyncEnumerator(cancelToken);
while (true)
{
bool hasNext;
try
{
hasNext = await blobEnumerator.MoveNextAsync().ConfigureAwait(false);
hasNext = await Utility.Utility.WithTimeout(_timeouts.ListTimeout, cancelToken, async ct
=> await blobEnumerator.MoveNextAsync().ConfigureAwait(false)
).ConfigureAwait(false);
}
catch (Azure.RequestFailedException ex) when (ex.Status == 404)
{
throw new FolderMissingException(ex);
}
if (!hasNext) break;
cancelToken.ThrowIfCancellationRequested();
if (blobEnumerator.Current is { } blob)
{
var blobName = Uri.UnescapeDataString(blob.Name.Replace("+", "%2B"));
var lastModified = blob.Properties.LastModified?.UtcDateTime ?? DateTime.UtcNow;
yield return new FileEntry(
blobName,
blob.Properties.ContentLength ?? 0,
@@ -4,7 +4,9 @@
<TargetFramework>net8.0</TargetFramework>
<OutputType>Library</OutputType>
<Copyright>Copyright © 2025 Team Duplicati, MIT license</Copyright>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<RootNamespace>Duplicati.Library.Backend.AzureBlob</RootNamespace>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Azure.Storage.Blobs" Version="12.23.0" />
@@ -23,7 +23,8 @@ using Duplicati.Library.Localization.Short;
namespace Duplicati.Library.Backend.AzureBlob.Strings;
internal static class AzureBlobBackend {
internal static class AzureBlobBackend
{
public static string DescriptionV2 => LC.L(@"This backend can read and write data to Azure blob storage. Allowed format is ""azure://bucketname"".");
public static string DisplayName => LC.L(@"Azure blob");
public static string ContainerNameDescriptionLong => LC.L(@"All files will be written to the container specified.");
@@ -36,8 +37,4 @@ internal static class AzureBlobBackend {
public static string SasTokenDescriptionLong => LC.L(@"The Azure shared access signature (SAS) token which can be obtained by selecting the ""Shared access signature"" blade on the storage account dashboard, or inside a container blade.");
public static string SasTokenDescriptionShort => LC.L(@"The SAS token");
public static string NoAccessKeyOrSasToken => LC.L(@"No Azure access key or SAS token given");
public static string AuthPasswordDescriptionLong => LC.L(@"The password used to connect to the server. This may also be supplied as the environment variable ""AUTH_PASSWORD"".");
public static string AuthPasswordDescriptionShort => LC.L(@"Supply the password used to connect to the server");
public static string AuthUsernameDescriptionLong => LC.L(@"The username used to connect to the server. This may also be supplied as the environment variable ""AUTH_USERNAME"".");
public static string AuthUsernameDescriptionShort => LC.L(@"Supply the username used to connect to the server");
}
+135 -133
View File
@@ -28,6 +28,7 @@ using System.Reflection;
using Duplicati.Library.Backend.Backblaze.Model;
using FileEntry = Duplicati.Library.Common.IO.FileEntry;
using System.Runtime.CompilerServices;
using Duplicati.Library.Utility.Options;
namespace Duplicati.Library.Backend.Backblaze;
@@ -36,11 +37,6 @@ namespace Duplicati.Library.Backend.Backblaze;
/// </summary>
public class B2 : IStreamingBackend, ITimeoutExemptBackend
{
/// <summary>
/// The default timeout in seconds for LIST/CreateFolder operations
/// </summary>
private const int SHORT_OPERATION_TIMEOUT_SECONDS = 30;
/// <summary>
/// The option key for specifying the Backblaze B2 account ID
/// </summary>
@@ -104,7 +100,7 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend
/// <summary>
/// The type of bucket (e.g., allPrivate, allPublic) being accessed or created
/// </summary>
private readonly string _bucketType;
private readonly string? _bucketType;
/// <summary>
/// The number of files to retrieve per API request
@@ -114,27 +110,32 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend
/// <summary>
/// Custom download URL for the B2 service, if specified
/// </summary>
private readonly string _downloadUrl;
private readonly string? _downloadUrl;
/// <summary>
/// Helper class for handling B2 authentication and API requests
/// </summary>
private readonly B2AuthHelper _b2AuthHelper;
/// <summary>
/// The timeout options for the backend
/// </summary>
private readonly TimeoutOptionsHelper.Timeouts _timeouts;
/// <summary>
/// Cached upload URL and authorization token for file uploads
/// </summary>
private UploadUrlResponse _uploadUrl;
private UploadUrlResponse? _uploadUrl;
/// <summary>
/// Cache of file listings, mapping filenames to their versions
/// </summary>
private Dictionary<string, List<FileEntity>> _filecache;
private Dictionary<string, List<FileEntity>>? _filecache;
/// <summary>
/// The current bucket entity containing bucket information and credentials
/// </summary>
private BucketEntity _bucket;
private BucketEntity? _bucket;
/// <summary>
/// A scoped instance of Http client to be used with the backend, this will use a
@@ -153,6 +154,12 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend
/// </summary>
public B2()
{
_bucketName = null!;
_prefix = null!;
_urlencodedPrefix = null!;
_b2AuthHelper = null!;
_timeouts = null!;
_httpClient = null!;
}
/// <summary>
@@ -160,7 +167,7 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend
/// </summary>
/// <param name="url">URL in Duplicati Uri format</param>
/// <param name="options">options to be used in the backend</param>
public B2(string url, Dictionary<string, string> options)
public B2(string url, Dictionary<string, string?> options)
{
var uri = new Utility.Uri(url);
@@ -177,28 +184,12 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend
if (options.TryGetValue(B2_CREATE_BUCKET_TYPE_OPTION, out var option1))
_bucketType = option1;
string accountId = null;
string accountKey = null;
var auth = AuthOptionsHelper.ParseWithAlias(options, uri, B2_ID_OPTION, B2_KEY_OPTION);
// Takes the account ID and key from the options, or from the URL with the cascading precedence
if (options.TryGetValue("auth-username", out var authUsernameOption))
accountId = authUsernameOption;
if (options.TryGetValue("auth-password", out var authPasswordOption))
accountKey = authPasswordOption;
if (options.TryGetValue(B2_ID_OPTION, out var accountIdOption))
accountId = accountIdOption;
if (options.TryGetValue(B2_KEY_OPTION, out var accountKeyOption))
accountKey = accountKeyOption;
if (!string.IsNullOrEmpty(uri.Username))
accountId = uri.Username;
if (!string.IsNullOrEmpty(uri.Password))
accountKey = uri.Password;
if (string.IsNullOrEmpty(accountId))
if (!auth.HasUsername)
throw new UserInformationException(Strings.B2.NoB2UserIDError, "B2MissingUserID");
if (string.IsNullOrEmpty(accountKey))
if (!auth.HasPassword)
throw new UserInformationException(Strings.B2.NoB2KeyError, "B2MissingKey");
_pageSize = DEFAULT_PAGE_SIZE;
@@ -216,7 +207,8 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend
_httpClient = HttpClientHelper.CreateClient();
_httpClient.Timeout = Timeout.InfiniteTimeSpan;
_b2AuthHelper = new B2AuthHelper(accountId, accountKey, _httpClient);
_timeouts = TimeoutOptionsHelper.Parse(options);
_b2AuthHelper = new B2AuthHelper(auth.Username!, auth.Password!, _httpClient, _timeouts);
}
@@ -232,20 +224,20 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend
/// <exception cref="FolderMissingException"></exception>
private async Task<BucketEntity> GetBucketAsync(CancellationToken cancellationToken)
{
if (_bucket != null) return _bucket;
if (_bucket != null)
return _bucket;
var config = await _b2AuthHelper.GetConfigAsync(cancellationToken).ConfigureAwait(false);
await _cacheLock.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
using var timeoutToken = new CancellationTokenSource();
timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS));
using var combinedCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken);
var buckets = await _b2AuthHelper.PostAndGetJsonDataAsync<ListBucketsResponse>(
$"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_list_buckets",
new ListBucketsRequest(accountId: _b2AuthHelper.AccountId),
combinedCancellationToken.Token).ConfigureAwait(false);
var buckets = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct
=> await _b2AuthHelper.PostAndGetJsonDataAsync<ListBucketsResponse>(
$"{config.APIUrl}/b2api/v1/b2_list_buckets",
new ListBucketsRequest(accountId: config.AccountID),
ct).ConfigureAwait(false)
).ConfigureAwait(false);
return _bucket ??= buckets?.Buckets?.FirstOrDefault(x =>
string.Equals(x.BucketName, _bucketName, StringComparison.OrdinalIgnoreCase))
@@ -266,14 +258,18 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend
if (_uploadUrl != null)
return _uploadUrl;
using var timeoutToken = new CancellationTokenSource();
timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS));
using var combinedCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken);
var config = await _b2AuthHelper.GetConfigAsync(cancellationToken).ConfigureAwait(false);
var bucket = await GetBucketAsync(cancellationToken).ConfigureAwait(false);
_uploadUrl = await _b2AuthHelper.PostAndGetJsonDataAsync<UploadUrlResponse>(
$"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_get_upload_url",
new UploadUrlRequest { BucketID = (await GetBucketAsync(cancellationToken)).BucketID },
combinedCancellationToken.Token
_uploadUrl = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct
=> await _b2AuthHelper.PostAndGetJsonDataAsync<UploadUrlResponse>(
$"{config.APIUrl}/b2api/v1/b2_get_upload_url",
new UploadUrlRequest
{
BucketID = bucket.BucketID
?? throw new Exception("BucketID is null")
},
ct).ConfigureAwait(false)
).ConfigureAwait(false);
return _uploadUrl;
@@ -289,38 +285,32 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend
private async Task<string> GetFileId(string filename, CancellationToken cancelToken)
{
if (_filecache != null && _filecache.TryGetValue(filename, out var value))
return value.OrderByDescending(x => x.UploadTimestamp).First().FileID;
return value.OrderByDescending(x => x.UploadTimestamp).First().FileID
?? throw new InvalidDataException("Missing file ID");
await RebuildFileCache(cancelToken).ConfigureAwait(false);
if (_filecache != null && _filecache.TryGetValue(filename, out var value1))
return value1.OrderByDescending(x => x.UploadTimestamp).First().FileID;
return value1.OrderByDescending(x => x.UploadTimestamp).First().FileID
?? throw new InvalidDataException("Missing file ID");
throw new FileMissingException();
}
/// <summary>
/// Returns the DownloadURL, either cached, or making a call to the server's /b2_authorize_account
/// </summary>
private string DownloadUrl => string.IsNullOrEmpty(_downloadUrl) ? _b2AuthHelper.DownloadUrl : _downloadUrl;
/// <inheritdoc/>
public IList<ICommandLineArgument> SupportedCommands =>
new List<ICommandLineArgument>([
new CommandLineArgument(B2_ID_OPTION, CommandLineArgument.ArgumentType.String, Strings.B2.B2accountidDescriptionShort, Strings.B2.B2accountidDescriptionLong, null,
["auth-password"], null),
new CommandLineArgument(B2_KEY_OPTION, CommandLineArgument.ArgumentType.Password, Strings.B2.B2applicationkeyDescriptionShort, Strings.B2.B2applicationkeyDescriptionLong, null,
["auth-username"], null),
new CommandLineArgument("auth-password", CommandLineArgument.ArgumentType.Password, Strings.B2.AuthPasswordDescriptionShort, Strings.B2.AuthPasswordDescriptionLong),
new CommandLineArgument("auth-username", CommandLineArgument.ArgumentType.String, Strings.B2.AuthUsernameDescriptionShort, Strings.B2.AuthUsernameDescriptionLong),
new CommandLineArgument(B2_ID_OPTION, CommandLineArgument.ArgumentType.String, Strings.B2.B2accountidDescriptionShort, Strings.B2.B2accountidDescriptionLong, null, [AuthOptionsHelper.AuthUsernameOption], null),
new CommandLineArgument(B2_KEY_OPTION, CommandLineArgument.ArgumentType.Password, Strings.B2.B2applicationkeyDescriptionShort, Strings.B2.B2applicationkeyDescriptionLong, null, [AuthOptionsHelper.AuthPasswordOption], null),
new CommandLineArgument(B2_CREATE_BUCKET_TYPE_OPTION, CommandLineArgument.ArgumentType.String, Strings.B2.B2createbuckettypeDescriptionShort, Strings.B2.B2createbuckettypeDescriptionLong, DEFAULT_BUCKET_TYPE),
new CommandLineArgument(B2_PAGESIZE_OPTION, CommandLineArgument.ArgumentType.Integer, Strings.B2.B2pagesizeDescriptionShort, Strings.B2.B2pagesizeDescriptionLong, DEFAULT_PAGE_SIZE.ToString()),
new CommandLineArgument(B2_DOWNLOAD_URL_OPTION, CommandLineArgument.ArgumentType.String, Strings.B2.B2downloadurlDescriptionShort, Strings.B2.B2downloadurlDescriptionLong)
new CommandLineArgument(B2_DOWNLOAD_URL_OPTION, CommandLineArgument.ArgumentType.String, Strings.B2.B2downloadurlDescriptionShort, Strings.B2.B2downloadurlDescriptionLong),
..TimeoutOptionsHelper.GetOptions()
]);
public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken)
{
TempFile tmp = null;
TempFile? tmp = null;
var measure = stream;
while (measure is OverrideableStream os &&
@@ -339,7 +329,7 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend
// Compute the hash
using (var hashalg = HashFactory.CreateHasher("SHA1"))
sha1 = Library.Utility.Utility.ByteArrayAsHexString(hashalg.ComputeHash(measure));
sha1 = Utility.Utility.ByteArrayAsHexString(hashalg.ComputeHash(measure));
measure.Position = p;
}
@@ -367,14 +357,15 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend
// For PutAsync, no timeout is specified. The only thing that can stop it is the cancellation token
using var request = new HttpRequestMessage(HttpMethod.Post, uploadUrlData.UploadUrl);
using var timeoutStream = stream.ObserveReadTimeout(_timeouts.ReadWriteTimeout, false);
request.Headers.TryAddWithoutValidation("Authorization", uploadUrlData.AuthorizationToken);
request.Headers.Add("X-Bz-Content-Sha1", sha1);
request.Headers.Add("X-Bz-File-Name", _urlencodedPrefix + Utility.Uri.UrlPathEncode(remotename));
request.Content = new StreamContent(stream, B2_RECOMMENDED_CHUNK_SIZE);
request.Content = new StreamContent(timeoutStream, B2_RECOMMENDED_CHUNK_SIZE);
request.Content.Headers.Add("Content-Type", "application/octet-stream");
request.Content.Headers.Add("Content-Length", stream.Length.ToString());
request.Content.Headers.Add("Content-Length", timeoutStream.Length.ToString());
var response = await _httpClient.UploadStream(request, cancelToken).ConfigureAwait(false);
var rdata = await response.Content.ReadAsStreamAsync(cancelToken).ConfigureAwait(false);
@@ -382,7 +373,8 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend
UploadFileResponse fileinfo;
using (var tr = new StreamReader(rdata))
await using (var jr = new Newtonsoft.Json.JsonTextReader(tr))
fileinfo = new Newtonsoft.Json.JsonSerializer().Deserialize<UploadFileResponse>(jr);
fileinfo = new Newtonsoft.Json.JsonSerializer().Deserialize<UploadFileResponse>(jr)
?? throw new Exception("Failed to parse response");
// Delete old versions
if (_filecache!.ContainsKey(remotename))
@@ -426,17 +418,18 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend
/// <exception cref="Exception">Exceptions arising from either code execution or FileMissingException</exception>
public async Task GetAsync(string remotename, Stream stream, CancellationToken cancellationToken)
{
if (_filecache == null || !_filecache.ContainsKey(remotename))
await RebuildFileCache(cancellationToken).ConfigureAwait(false);
using var request = _filecache != null && _filecache.ContainsKey(remotename)
? _b2AuthHelper.CreateRequest(
$"{DownloadUrl}/b2api/v1/b2_download_file_by_id?fileId={Utility.Uri.UrlEncode(await GetFileId(remotename, cancellationToken))}")
: _b2AuthHelper.CreateRequest(
$"{DownloadUrl}/{_urlencodedPrefix}{Utility.Uri.UrlPathEncode(remotename)}");
var config = await _b2AuthHelper.GetConfigAsync(cancellationToken).ConfigureAwait(false);
HttpResponseMessage response = null;
using var request = _filecache != null && _filecache.ContainsKey(remotename)
? await _b2AuthHelper.CreateRequestAsync(
$"{config.DownloadUrl}/b2api/v1/b2_download_file_by_id?fileId={Utility.Uri.UrlEncode(await GetFileId(remotename, cancellationToken))}", HttpMethod.Get, cancellationToken).ConfigureAwait(false)
: await _b2AuthHelper.CreateRequestAsync(
$"{config.DownloadUrl}/{_urlencodedPrefix}{Utility.Uri.UrlPathEncode(remotename)}", HttpMethod.Get, cancellationToken).ConfigureAwait(false);
HttpResponseMessage? response = null;
try
{
// For GetAsync, no timeout is specified. The only thing that can stop it is the cancellation token
@@ -444,14 +437,15 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend
.ConfigureAwait(false);
response.EnsureSuccessStatusCode();
await using var responseStream = await response.Content.ReadAsStreamAsync(cancellationToken);
await responseStream.CopyToAsync(stream, cancellationToken).ConfigureAwait(false);
using var timeoutStream = responseStream.ObserveReadTimeout(_timeouts.ReadWriteTimeout, false);
await timeoutStream.CopyToAsync(stream, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
if (B2AuthHelper.GetExceptionStatusCode(ex) == HttpStatusCode.NotFound)
throw new FileMissingException();
_b2AuthHelper.AttemptParseAndThrowException(ex, response);
await _b2AuthHelper.AttemptParseAndThrowExceptionAsync(ex, response, cancellationToken).ConfigureAwait(false);
throw;
}
finally
@@ -464,13 +458,14 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend
/// Implementation of interface method for listing remote folder contents
/// </summary>
/// <returns>List of IFileEntry with directory listing result</returns>
private async Task RebuildFileCache(CancellationToken cancellationToken)
private async Task<Dictionary<string, List<FileEntity>>> RebuildFileCache(CancellationToken cancellationToken)
{
_filecache = null;
var cache = new Dictionary<string, List<FileEntity>>();
string nextFileId = null;
string nextFileName = null;
string listVersionUrl = $"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_list_file_versions";
string? nextFileId = null;
string? nextFileName = null;
var config = await _b2AuthHelper.GetConfigAsync(cancellationToken).ConfigureAwait(false);
var listVersionUrl = $"{config.APIUrl}/b2api/v1/b2_list_file_versions";
var listRetryHelper = RetryAfterHelper.CreateOrGetRetryAfterHelper(listVersionUrl);
@@ -478,22 +473,20 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend
{
try
{
using var timeoutToken = new CancellationTokenSource();
timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS));
using var combinedCancellationToken =
CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken);
var resp = await _b2AuthHelper.PostAndGetJsonDataAsync<ListFilesResponse>(
$"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_list_file_versions",
new ListFilesRequest
{
BucketID = (await GetBucketAsync(cancellationToken)).BucketID,
MaxFileCount = _pageSize,
Prefix = _prefix,
StartFileID = nextFileId,
StartFileName = nextFileName
},
combinedCancellationToken.Token
var resp = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct
=> await _b2AuthHelper.PostAndGetJsonDataAsync<ListFilesResponse>(
$"{config.APIUrl}/b2api/v1/b2_list_file_versions",
new ListFilesRequest
{
BucketID = (await GetBucketAsync(cancellationToken)).BucketID
?? throw new Exception("BucketID is null"),
MaxFileCount = _pageSize,
Prefix = _prefix,
StartFileID = nextFileId,
StartFileName = nextFileName
},
ct
).ConfigureAwait(false)
).ConfigureAwait(false);
nextFileId = resp.NextFileID;
@@ -504,6 +497,9 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend
foreach (var file in resp.Files)
{
if (string.IsNullOrWhiteSpace(file.FileName))
continue;
if (!file.FileName.StartsWith(_prefix, StringComparison.Ordinal))
continue;
@@ -530,7 +526,7 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend
}
} while (nextFileId != null);
_filecache = cache;
return _filecache = cache;
}
/// <summary>
@@ -539,8 +535,8 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend
/// <returns>List of IFileEntry with directory listing result</returns>
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancellationToken)
{
await RebuildFileCache(cancellationToken).ConfigureAwait(false);
foreach (var x in _filecache)
var filecache = await RebuildFileCache(cancellationToken).ConfigureAwait(false);
foreach (var x in filecache)
{
var newest = x.Value.OrderByDescending(y => y.UploadTimestamp).First();
var ts = Utility.Utility.EPOCH.AddMilliseconds(newest.UploadTimestamp);
@@ -581,28 +577,29 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend
{
try
{
using var timeoutToken = new CancellationTokenSource();
timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS));
using var combinedCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken);
var filecache = _filecache;
if (filecache == null || !filecache.ContainsKey(remotename))
filecache = await RebuildFileCache(cancellationToken).ConfigureAwait(false);
if (_filecache == null || !_filecache.ContainsKey(remotename))
await RebuildFileCache(cancellationToken).ConfigureAwait(false);
if (!_filecache.TryGetValue(remotename, out var value))
if (!filecache.TryGetValue(remotename, out var value))
throw new FileMissingException();
var config = await _b2AuthHelper.GetConfigAsync(cancellationToken).ConfigureAwait(false);
foreach (var n in value.OrderBy(x => x.UploadTimestamp))
await _b2AuthHelper.PostAndGetJsonDataAsync<DeleteResponse>(
$"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_delete_file_version",
new DeleteRequest
{
FileName = _prefix + remotename,
FileId = n.FileID
},
combinedCancellationToken.Token
await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct
=> await _b2AuthHelper.PostAndGetJsonDataAsync<DeleteResponse>(
$"{config.APIUrl}/b2api/v1/b2_delete_file_version",
new DeleteRequest
{
FileName = _prefix + remotename,
FileId = n.FileID
},
ct
).ConfigureAwait(false)
).ConfigureAwait(false);
_filecache[remotename].Clear();
filecache[remotename].Clear();
}
catch
{
@@ -626,19 +623,18 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend
/// <exception cref="Exception"></exception>
public async Task CreateFolderAsync(CancellationToken cancellationToken)
{
using var timeoutToken = new CancellationTokenSource();
timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS));
using var combinedCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken);
_bucket = await _b2AuthHelper.PostAndGetJsonDataAsync<BucketEntity>(
$"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_create_bucket",
new BucketEntity
{
AccountID = _b2AuthHelper.AccountId,
BucketName = _bucketName,
BucketType = _bucketType
},
combinedCancellationToken.Token
var config = await _b2AuthHelper.GetConfigAsync(cancellationToken);
_bucket = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct
=> await _b2AuthHelper.PostAndGetJsonDataAsync<BucketEntity>(
$"{config.APIUrl}/b2api/v1/b2_create_bucket",
new BucketEntity
{
AccountID = config.AccountID,
BucketName = _bucketName,
BucketType = _bucketType
},
ct
).ConfigureAwait(false)
).ConfigureAwait(false);
}
@@ -652,12 +648,18 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend
public string Description => Strings.B2.Description;
/// <inheritdoc/>
public Task<string[]> GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new string[] {
new System.Uri(B2AuthHelper.AUTH_URL).Host,
_b2AuthHelper?.ApiDnsName,
_b2AuthHelper?.DownloadDnsName
}.Where(x => !string.IsNullOrEmpty(x))
.ToArray());
public async Task<string[]> GetDNSNamesAsync(CancellationToken cancelToken)
{
var config = await _b2AuthHelper.GetConfigAsync(cancelToken);
return new string?[] {
B2AuthHelper.AUTH_URL,
config.APIUrl,
config.DownloadUrl
}.WhereNotNullOrWhiteSpace()
.Select(x => new System.Uri(x).Host)
.Distinct()
.ToArray();
}
/// <summary>
/// Handles disposal of the backend's resources.
@@ -23,9 +23,9 @@ using System.Net;
using System.Text;
using Duplicati.Library.Backend.Backblaze.Model;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility.Options;
using Newtonsoft.Json;
using Exception = System.Exception;
using Uri = System.Uri;
namespace Duplicati.Library.Backend.Backblaze;
@@ -35,12 +35,21 @@ namespace Duplicati.Library.Backend.Backblaze;
/// <param name="userid">Username</param>
/// <param name="password">Password</param>
/// <param name="httpClient">HttpClient instance to use and pass along to JsonWebHelperHttpClient</param>
public class B2AuthHelper(string userid, string password, HttpClient httpClient) : JsonWebHelperHttpClient(httpClient)
public class B2AuthHelper(string userid, string password, HttpClient httpClient, TimeoutOptionsHelper.Timeouts timeouts) : JsonWebHelperHttpClient(httpClient)
{
/// <summary>
/// The configuration details after authorization
/// </summary>
/// <param name="AccountID">The account ID</param>
/// <param name="APIUrl">The API URL</param>
/// <param name="AuthorizationToken">The authorization token</param>
/// <param name="DownloadUrl">The download URL</param>
public record AuthResponse(string AccountID, string APIUrl, string AuthorizationToken, string DownloadUrl);
/// <summary>
/// Cached authorization response
/// </summary>
private AuthResponse _CachedAuthResponse;
private AuthResponse? _CachedAuthResponse;
/// <summary>
/// Cached authorization response expiration time
@@ -57,46 +66,22 @@ public class B2AuthHelper(string userid, string password, HttpClient httpClient)
/// </summary>
internal const string AUTH_URL = "https://api.backblazeb2.com/b2api/v1/b2_authorize_account";
/// <summary>
/// Timeout for authentication requests
/// </summary>
private const int AUTHENTICATION_TIMEOUT_SECONDS = 10;
/// <summary>
/// Maximum number of retries for authorization
/// </summary>
private const int MAX_AUTHORIZATION_RETRIES = 5;
/// <summary>
/// Authorization token (fetches from Config, which will refresh if needed)
/// </summary>
private string AuthorizationToken => Config.AuthorizationToken;
/// <summary>
/// API URL (fetches from Config, which will refresh if needed)
/// </summary>
public string ApiUrl => Config.APIUrl;
/// <summary>
/// Download URL (fetches from Config, which will refresh if needed)
/// </summary>
public string DownloadUrl => Config.DownloadUrl;
/// <summary>
/// Account ID (fetches from Config, which will refresh if needed)
/// </summary>
public string AccountId => Config.AccountID;
/// <summary>
/// Creates an HTTP request message with authorization header
/// </summary>
/// <param name="url">The URL for the request</param>
/// <param name="method">HTTP method (defaults to GET if null)</param>
/// <returns>Configured HttpRequestMessage</returns>
public override HttpRequestMessage CreateRequest(string url, string method = null)
public override async Task<HttpRequestMessage> CreateRequestAsync(string url, HttpMethod method, CancellationToken cancellationToken)
{
HttpRequestMessage request = base.CreateRequest(url, method);
request.Headers.TryAddWithoutValidation("Authorization", AuthorizationToken);
var request = await base.CreateRequestAsync(url, method, cancellationToken).ConfigureAwait(false);
var config = await GetConfigAsync(cancellationToken).ConfigureAwait(false);
request.Headers.TryAddWithoutValidation("Authorization", config.AuthorizationToken);
request.Headers.Add("User-Agent", UserAgent);
return request;
}
@@ -105,88 +90,91 @@ public class B2AuthHelper(string userid, string password, HttpClient httpClient)
/// Cleans the url to remove trailing slashes
/// </summary>
/// <param name="url">URL</param>
private string DropTrailingSlashes(string url)
private static string DropTrailingSlashes(string url)
{
if (string.IsNullOrWhiteSpace(url))
return url;
while (url.EndsWith("/", StringComparison.Ordinal))
url = url.Substring(0, url.Length - 1);
return url;
}
/// <summary>
/// API DNS Name
/// </summary>
public string ApiDnsName =>
_CachedAuthResponse == null || string.IsNullOrWhiteSpace(_CachedAuthResponse.APIUrl)
? null
: new Uri(_CachedAuthResponse.APIUrl).Host;
/// <summary>
/// Download DNS Name
/// </summary>
public string DownloadDnsName =>
_CachedAuthResponse == null || string.IsNullOrWhiteSpace(_CachedAuthResponse.DownloadUrl)
? null
: new Uri(_CachedAuthResponse.DownloadUrl).Host;
private AuthResponse Config
public async Task<AuthResponse> GetConfigAsync(CancellationToken cancelToken)
{
get
if (_CachedAuthResponse != null && _configExpires >= DateTime.UtcNow)
return _CachedAuthResponse;
var retries = 0;
while (true)
{
if (_CachedAuthResponse != null && _configExpires >= DateTime.UtcNow) return _CachedAuthResponse;
var retries = 0;
while (true)
HttpResponseMessage? response = null;
try
{
HttpResponseMessage response = null;
try
using var request = await base.CreateRequestAsync(AUTH_URL, HttpMethod.Get, cancelToken).ConfigureAwait(false);
request.Headers.Add("Authorization", $"Basic {Convert.ToBase64String(Encoding.UTF8.GetBytes(userid + ":" + password))}");
request.Headers.Add("ContentType", "application/json; charset=utf-8");
var authResponse = await Utility.Utility.WithTimeout(timeouts.ShortTimeout, cancelToken, async ct =>
{
using var request = base.CreateRequest(AUTH_URL);
request.Headers.Add("Authorization", $"Basic {Convert.ToBase64String(Encoding.UTF8.GetBytes(userid + ":" + password))}");
request.Headers.Add("ContentType", "application/json; charset=utf-8");
using var timeoutToken = new CancellationTokenSource();
timeoutToken.CancelAfter(TimeSpan.FromSeconds(AUTHENTICATION_TIMEOUT_SECONDS));
response = _httpClient.Send(request, timeoutToken.Token);
response = await _httpClient.SendAsync(request, ct).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
_CachedAuthResponse = ReadJsonResponse<AuthResponse>(response);
_CachedAuthResponse.APIUrl = DropTrailingSlashes(_CachedAuthResponse.APIUrl);
_CachedAuthResponse.DownloadUrl = DropTrailingSlashes(_CachedAuthResponse.DownloadUrl);
_configExpires = DateTime.UtcNow + TimeSpan.FromMinutes(CACHE_EXPIRATION_MINUTES);
return await ReadJsonResponseAsync<ApiAuthResponse>(response, ct).ConfigureAwait(false)
?? throw new Exception("Failed to parse authorization response");
}).ConfigureAwait(false);
response.Dispose();
return _CachedAuthResponse;
}
catch (Exception ex)
if (string.IsNullOrWhiteSpace(authResponse.AccountID))
throw new Exception("Account ID is missing from authorization response");
if (string.IsNullOrWhiteSpace(authResponse.APIUrl))
throw new Exception("API URL is missing from authorization response");
if (string.IsNullOrWhiteSpace(authResponse.AuthorizationToken))
throw new Exception("Authorization token is missing from authorization response");
if (string.IsNullOrWhiteSpace(authResponse.DownloadUrl))
throw new Exception("Download URL is missing from authorization response");
authResponse.APIUrl = DropTrailingSlashes(authResponse.APIUrl);
authResponse.DownloadUrl = DropTrailingSlashes(authResponse.DownloadUrl);
_configExpires = DateTime.UtcNow + TimeSpan.FromMinutes(CACHE_EXPIRATION_MINUTES);
return _CachedAuthResponse = new AuthResponse(authResponse.AccountID, authResponse.APIUrl, authResponse.AuthorizationToken, authResponse.DownloadUrl);
}
catch (Exception ex)
{
var clientError = false;
try
{
var clientError = false;
try
// Only retry once on client errors
if (ex is HttpRequestException { StatusCode: not null } exception)
{
// Only retry once on client errors
if (ex is HttpRequestException { StatusCode: not null } exception)
{
var sc = (int)exception.StatusCode;
clientError = sc is >= 400 and <= 499;
}
var sc = (int)exception.StatusCode;
clientError = sc is >= 400 and <= 499;
}
catch
{
// ignored
}
if (retries >= (clientError ? 1 : MAX_AUTHORIZATION_RETRIES))
{
AttemptParseAndThrowException(ex, response);
throw;
}
Thread.Sleep(TimeSpan.FromSeconds(Math.Pow(2, retries)));
retries++;
}
catch
{
// ignored
}
if (retries >= (clientError ? 1 : MAX_AUTHORIZATION_RETRIES))
{
await AttemptParseAndThrowExceptionAsync(ex, response, cancelToken);
throw;
}
Thread.Sleep(TimeSpan.FromSeconds(Math.Pow(2, retries)));
retries++;
}
finally
{
response?.Dispose();
}
}
}
@@ -197,18 +185,23 @@ public class B2AuthHelper(string userid, string password, HttpClient httpClient)
/// <param name="ex">Exception to be parsed</param>
/// <param name="responseContext">Response context</param>
/// <exception cref="Exception">New detailed exception</exception>
public override void AttemptParseAndThrowException(Exception ex, HttpResponseMessage responseContext = null)
public override async Task AttemptParseAndThrowExceptionAsync(Exception ex, HttpResponseMessage? responseContext, CancellationToken cancelToken)
{
if (ex is not HttpRequestException || responseContext == null)
return;
if (ex is HttpRequestException && responseContext != null && responseContext.StatusCode == HttpStatusCode.TooManyRequests)
if (ex is HttpRequestException && responseContext.StatusCode == HttpStatusCode.TooManyRequests)
throw new TooManyRequestException(responseContext.Headers.RetryAfter);
using var stream = responseContext.Content.ReadAsStream();
using var reader = new StreamReader(stream);
var rawData = reader.ReadToEnd();
var errorResponse = JsonConvert.DeserializeObject<ErrorResponse>(rawData);
var rawData = await Utility.Utility.WithTimeout(timeouts.ShortTimeout, cancelToken, ct =>
{
using var stream = responseContext.Content.ReadAsStream();
using var reader = new StreamReader(stream);
return reader.ReadToEnd();
}).ConfigureAwait(false);
var errorResponse = JsonConvert.DeserializeObject<ErrorResponse>(rawData)
?? new ErrorResponse { Status = (int)responseContext.StatusCode, Message = responseContext.ReasonPhrase };
throw new UserInformationException($"Backblaze ErrorResponse: {errorResponse.Status} - {errorResponse.Code}: {errorResponse.Message}", "BackblazeErrorResponse");
}
@@ -4,7 +4,9 @@
<TargetFramework>net8.0</TargetFramework>
<OutputType>Library</OutputType>
<Copyright>Copyright © 2025 Team Duplicati, MIT license</Copyright>
<ImplicitUsings>true</ImplicitUsings>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<RootNamespace>Duplicati.Library.Backend.Box</RootNamespace>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
@@ -29,5 +29,5 @@ namespace Duplicati.Library.Backend.Backblaze.Model;
internal abstract class AccountIDEntity
{
[JsonProperty("accountId")]
public string AccountID { get; set; }
public string? AccountID { get; set; }
}
@@ -26,14 +26,14 @@ namespace Duplicati.Library.Backend.Backblaze.Model;
/// <summary>
/// Model for JSON parsing of AuthResponse
/// </summary>
internal class AuthResponse
internal class ApiAuthResponse
{
[JsonProperty("accountId")]
public string AccountID { get; set; }
public string? AccountID { get; set; }
[JsonProperty("apiUrl")]
public string APIUrl { get; set; }
public string? APIUrl { get; set; }
[JsonProperty("authorizationToken")]
public string AuthorizationToken { get; set; }
public string? AuthorizationToken { get; set; }
[JsonProperty("downloadUrl")]
public string DownloadUrl { get; set; }
public string? DownloadUrl { get; set; }
}
@@ -34,17 +34,17 @@ internal class BucketEntity : AccountIDEntity
/// This property can be null as specified by the NullValueHandling attribute.
/// </summary>
[JsonProperty("bucketId", NullValueHandling = NullValueHandling.Ignore)]
public string BucketID { get; set; }
public string? BucketID { get; set; }
/// <summary>
/// Gets or sets the name of the bucket.
/// </summary>
[JsonProperty("bucketName")]
public string BucketName { get; set; }
public required string BucketName { get; set; }
/// <summary>
/// Gets or sets the type of the bucket (e.g., "allPrivate", "allPublic").
/// </summary>
[JsonProperty("bucketType")]
public string BucketType { get; set; }
public string? BucketType { get; set; }
}
@@ -32,5 +32,5 @@ internal abstract class BucketIDEntity
/// Gets or sets the unique identifier of the bucket.
/// </summary>
[JsonProperty("bucketId")]
public string BucketID { get; set; }
public string? BucketID { get; set; }
}
@@ -34,12 +34,12 @@ internal class DeleteRequest
/// This is the full path of the file in the bucket.
/// </summary>
[JsonProperty("fileName")]
public string FileName { get; set; }
public string? FileName { get; set; }
/// <summary>
/// Gets or sets the unique identifier of the file to be deleted.
/// This ID is assigned by Backblaze B2 when the file is uploaded.
/// </summary>
[JsonProperty("fileId")]
public string FileId { get; set; }
public string? FileId { get; set; }
}
@@ -29,9 +29,9 @@ namespace Duplicati.Library.Backend.Backblaze.Model;
internal class ErrorResponse
{
[JsonProperty("code")]
public string Code { get; set; }
public string? Code { get; set; }
[JsonProperty("message")]
public string Message { get; set; }
public string? Message { get; set; }
[JsonProperty("status")]
public long Status { get; set; }
}
@@ -34,14 +34,14 @@ internal class FileEntity
/// This ID is used to reference the file in various API operations.
/// </summary>
[JsonProperty("fileId")]
public string FileID { get; set; }
public string? FileID { get; set; }
/// <summary>
/// Gets or sets the name of the file as stored in Backblaze B2.
/// This is the full path of the file within the bucket.
/// </summary>
[JsonProperty("fileName")]
public string FileName { get; set; }
public string? FileName { get; set; }
/// <summary>
/// Gets or sets the action performed on the file.
@@ -49,7 +49,7 @@ internal class FileEntity
/// and "delete" for deleted files.
/// </summary>
[JsonProperty("action")]
public string Action { get; set; }
public string? Action { get; set; }
/// <summary>
/// Gets or sets the size of the file in bytes.
@@ -31,7 +31,7 @@ internal class ListBucketsRequest : AccountIDEntity
/// Initializes a new instance of the ListBucketsRequest class.
/// </summary>
/// <param name="accountId">The account ID for which to list buckets.</param>
public ListBucketsRequest(string accountId)
public ListBucketsRequest(string? accountId)
{
AccountID = accountId;
}
@@ -34,5 +34,5 @@ internal class ListBucketsResponse
/// Each element contains information about a single bucket in the account.
/// </summary>
[JsonProperty("buckets")]
public BucketEntity[] Buckets { get; set; }
public BucketEntity[]? Buckets { get; set; }
}
@@ -34,14 +34,14 @@ internal class ListFilesRequest : BucketIDEntity
/// Used for pagination to continue listing from a specific point.
/// </summary>
[JsonProperty("startFileName", NullValueHandling = NullValueHandling.Ignore)]
public string StartFileName { get; set; }
public string? StartFileName { get; set; }
/// <summary>
/// Gets or sets the file ID to start listing from.
/// Used in conjunction with StartFileName for pagination.
/// </summary>
[JsonProperty("startFileId", NullValueHandling = NullValueHandling.Ignore)]
public string StartFileID { get; set; }
public string? StartFileID { get; set; }
/// <summary>
/// Gets or sets the maximum number of files to return in a single request.
@@ -54,5 +54,5 @@ internal class ListFilesRequest : BucketIDEntity
/// Only files whose names begin with this prefix will be returned.
/// </summary>
[JsonProperty("prefix")]
public string Prefix { get; set; }
public required string Prefix { get; set; }
}
@@ -34,19 +34,19 @@ internal class ListFilesResponse
/// This value should be used in the next request to continue listing files.
/// </summary>
[JsonProperty("nextFileName")]
public string NextFileName { get; set; }
public string? NextFileName { get; set; }
/// <summary>
/// Gets or sets the ID of the next file to use for pagination.
/// This value should be used in conjunction with NextFileName for subsequent requests.
/// </summary>
[JsonProperty("nextFileId")]
public string NextFileID { get; set; }
public string? NextFileID { get; set; }
/// <summary>
/// Gets or sets the array of file entities returned by this request.
/// Each element contains information about a single file in the bucket.
/// </summary>
[JsonProperty("files")]
public FileEntity[] Files { get; set; }
public FileEntity[]? Files { get; set; }
}
@@ -33,19 +33,19 @@ internal class UploadFileResponse : AccountIDEntity
/// Gets or sets the unique identifier of the bucket containing the file.
/// </summary>
[JsonProperty("bucketId")]
public string BucketID { get; set; }
public string? BucketID { get; set; }
/// <summary>
/// Gets or sets the unique identifier assigned to the uploaded file.
/// </summary>
[JsonProperty("fileId")]
public string FileID { get; set; }
public string? FileID { get; set; }
/// <summary>
/// Gets or sets the name of the uploaded file.
/// </summary>
[JsonProperty("fileName")]
public string FileName { get; set; }
public string? FileName { get; set; }
/// <summary>
/// Gets or sets the size of the uploaded file in bytes.
@@ -57,11 +57,11 @@ internal class UploadFileResponse : AccountIDEntity
/// Gets or sets the SHA1 hash of the file content.
/// </summary>
[JsonProperty("contentSha1")]
public string ContentSha1 { get; set; }
public string? ContentSha1 { get; set; }
/// <summary>
/// Gets or sets the MIME type of the uploaded file.
/// </summary>
[JsonProperty("contentType")]
public string ContentType { get; set; }
public string? ContentType { get; set; }
}
@@ -34,12 +34,12 @@ internal class UploadUrlResponse : BucketIDEntity
/// This URL is specific to the bucket and should be used for subsequent upload requests.
/// </summary>
[JsonProperty("uploadUrl")]
public string UploadUrl { get; set; }
public string? UploadUrl { get; set; }
/// <summary>
/// Gets or sets the authorization token required for upload requests.
/// This token must be included in the headers when uploading files using the UploadUrl.
/// </summary>
[JsonProperty("authorizationToken")]
public string AuthorizationToken { get; set; }
public string? AuthorizationToken { get; set; }
}
+27 -29
View File
@@ -1,45 +1,43 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Localization.Short;
namespace Duplicati.Library.Backend.Strings {
internal static class B2 {
namespace Duplicati.Library.Backend.Strings
{
internal static class B2
{
public static string Description { get { return LC.L(@"This backend can read and write data to the Backblaze B2 Cloud Storage. Allowed format is ""b2://bucketname/prefix""."); } }
public static string DisplayName { get { return LC.L(@"B2 Cloud Storage"); } }
public static string B2applicationkeyDescriptionLong { get { return LC.L(@"B2 Cloud Storage Application Key can be obtained after logging into your Backblaze account. This can also be supplied through the option --{0}.", "auth-password"); } }
public static string B2applicationkeyDescriptionShort { get { return LC.L(@"B2 Cloud Storage Application Key"); } }
public static string B2accountidDescriptionLong { get { return LC.L(@"B2 Cloud Storage Account ID can be obtained after logging into your Backblaze account. This can also be supplied through the option --{0}.", "auth-username"); } }
public static string B2accountidDescriptionShort { get { return LC.L(@"B2 Cloud Storage Account ID"); } }
public static string AuthPasswordDescriptionLong { get { return LC.L(@"The password used to connect to the server. This may also be supplied as the environment variable ""AUTH_PASSWORD""."); } }
public static string AuthPasswordDescriptionShort { get { return LC.L(@"Supply the password used to connect to the server"); } }
public static string AuthUsernameDescriptionLong { get { return LC.L(@"The username used to connect to the server. This may also be supplied as the environment variable ""AUTH_USERNAME""."); } }
public static string AuthUsernameDescriptionShort { get { return LC.L(@"Supply the username used to connect to the server"); } }
public static string NoB2KeyError { get { return LC.L(@"No B2 Cloud Storage Application Key given"); } }
public static string NoB2UserIDError { get { return LC.L(@"No B2 Cloud Storage Account ID given"); } }
public static string B2createbuckettypeDescriptionLong { get { return LC.L(@"By default, a private bucket is created. Use this option to set the bucket type. Refer to the B2 documentation for allowed types."); } }
public static string B2createbuckettypeDescriptionShort { get { return LC.L(@"The bucket type used when creating a bucket"); } }
public static string B2pagesizeDescriptionLong { get { return LC.L(@"Use this option to set the page size for listing contents of B2 buckets. A lower number means less data, but can increase the number of Class C transaction on B2. Suggested values are between 100 and 1000."); } }
public static string B2pagesizeDescriptionShort { get { return LC.L(@"The size of file-listing pages"); } }
public static string B2pagesizeDescriptionLong { get { return LC.L(@"Use this option to set the page size for listing contents of B2 buckets. A lower number means less data, but can increase the number of Class C transaction on B2. Suggested values are between 100 and 1000."); } }
public static string B2pagesizeDescriptionShort { get { return LC.L(@"The size of file-listing pages"); } }
public static string B2downloadurlDescriptionLong { get { return LC.L(@"Change this if you want to use your custom domain to download files, and uploading will not be affected. The default download URL depends on your account and looks like ""https://f00X.backblazeb2.com""."); } }
public static string B2downloadurlDescriptionShort { get { return LC.L(@"The base URL to use for downloading files"); } }
public static string InvalidPageSizeError(string argname, string value) { return LC.L(@"The setting ""{0}"" is invalid for ""{1}"". It must be an integer larger than zero.", value, argname); }
}
public static string InvalidPageSizeError(string argname, string? value) { return LC.L(@"The setting ""{0}"" is invalid for ""{1}"". It must be an integer larger than zero.", value, argname); }
}
}
+124 -306
View File
@@ -19,25 +19,21 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System.Net;
using System.Net.Http.Json;
using System.Runtime.CompilerServices;
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
using Duplicati.Library.Utility.Options;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
using Uri = Duplicati.Library.Utility.Uri;
namespace Duplicati.Library.Backend.Box
{
// ReSharper disable once ClassNeverInstantiated.Global
// This class is instantiated dynamically in the BackendLoader.
public class BoxBackend : IBackend, IStreamingBackend
public class BoxBackend : IStreamingBackend
{
private static readonly string LOGTAG = Logging.Log.LogTagFromType<BoxBackend>();
private static readonly string TOKEN_URL = OAuthHelper.OAUTH_LOGIN_URL("box.com");
private const string AUTHID_OPTION = "authid";
private const string REALLY_DELETE_OPTION = "box-delete-from-trash";
@@ -46,123 +42,115 @@ namespace Duplicati.Library.Backend.Box
private const int PAGE_SIZE = 200;
private readonly BoxHelper m_oauth;
private readonly string m_path;
private readonly bool m_deleteFromTrash;
private readonly BoxHelper _oAuthHelper;
private readonly string _path;
private readonly bool _deleteFromTrash;
private string m_currentfolder;
private readonly Dictionary<string, string> m_filecache = new Dictionary<string, string>();
private string? _currentFolder;
private readonly Dictionary<string, string> _fileCache = new();
private readonly TimeoutOptionsHelper.Timeouts _timeouts;
private class BoxHelper : OAuthHelper
private class BoxHelper : OAuthHelperHttpClient
{
public BoxHelper(string authid)
private readonly TimeoutOptionsHelper.Timeouts _timeouts;
public BoxHelper(string authid, TimeoutOptionsHelper.Timeouts timeouts)
: base(authid, "box.com")
{
AutoAuthHeader = true;
_timeouts = timeouts;
}
protected override void ParseException(Exception ex)
public override async Task AttemptParseAndThrowExceptionAsync(Exception ex, HttpResponseMessage responseContext, CancellationToken cancellationToken)
{
Exception newex = null;
try
{
if (ex is WebException exception && exception.Response is HttpWebResponse hs)
{
string rawdata = null;
using (var rs = Library.Utility.AsyncHttpRequest.TrySetTimeout(hs.GetResponseStream()))
using (var sr = new System.IO.StreamReader(rs))
rawdata = sr.ReadToEnd();
if (ex is not HttpRequestException || responseContext == null)
return;
if (string.IsNullOrWhiteSpace(rawdata))
return;
if (responseContext is { StatusCode: HttpStatusCode.TooManyRequests })
throw new TooManyRequestException(responseContext.Headers.RetryAfter);
newex = new Exception("Raw message: " + rawdata);
var msg = JsonConvert.DeserializeObject<ErrorResponse>(rawdata);
newex = new Exception(string.Format("{0} - {1}: {2}", msg.Status, msg.Code, msg.Message));
/*if (msg.ContextInfo != null && msg.ContextInfo.Length > 0)
newex = new Exception(string.Format("{0} - {1}: {2}{3}{4}", msg.Status, msg.Code, msg.Message, Environment.NewLine, string.Join("; ", from n in msg.ContextInfo select n.Message)));
*/
}
}
catch (Exception ex2)
{
Library.Logging.Log.WriteWarningMessage(LOGTAG, "BoxErrorParser", ex2, "Failed to parse error from Box");
}
if (newex != null)
throw newex;
await using var stream = await responseContext.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
using var reader = new StreamReader(stream);
var rawData = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, ct => reader.ReadToEndAsync(ct)).ConfigureAwait(false);
var errorResponse = JsonConvert.DeserializeObject<ErrorResponse>(rawData)
?? new ErrorResponse { Status = (int)responseContext.StatusCode, Code = "Unknown", Message = rawData };
throw new UserInformationException($"Box.com ErrorResponse: {errorResponse.Status} - {errorResponse.Code}: {errorResponse.Message}", "box.com");
}
}
// ReSharper disable once UnusedMember.Global
// This constructor is needed by the BackendLoader.
public BoxBackend()
{
_oAuthHelper = null!;
_path = null!;
_timeouts = null!;
}
// ReSharper disable once UnusedMember.Global
// This constructor is needed by the BackendLoader.
public BoxBackend(string url, Dictionary<string, string> options)
public BoxBackend(string url, Dictionary<string, string?> options)
{
var uri = new Utility.Uri(url);
var uri = new Uri(url);
m_path = Util.AppendDirSeparator(uri.HostAndPath, "/");
_path = Util.AppendDirSeparator(uri.HostAndPath, "/");
string authid = null;
if (options.ContainsKey(AUTHID_OPTION))
authid = options[AUTHID_OPTION];
var authid = AuthIdOptionsHelper.Parse(options)
.RequireCredentials(TOKEN_URL)
.AuthId!;
m_deleteFromTrash = Library.Utility.Utility.ParseBoolOption(options, REALLY_DELETE_OPTION);
_deleteFromTrash = Utility.Utility.ParseBoolOption(options, REALLY_DELETE_OPTION);
_timeouts = TimeoutOptionsHelper.Parse(options);
_oAuthHelper = new BoxHelper(authid, _timeouts);
m_oauth = new BoxHelper(authid);
}
private async Task<string> GetCurrentFolderWithCacheAsync(CancellationToken cancelToken)
{
if (m_currentfolder == null)
await GetCurrentFolderAsync(false, cancelToken).ConfigureAwait(false);
if (_currentFolder == null)
return await GetCurrentFolderAsync(false, cancelToken).ConfigureAwait(false);
return m_currentfolder;
return _currentFolder;
}
private async Task GetCurrentFolderAsync(bool create, CancellationToken cancelToken)
private async Task<string> GetCurrentFolderAsync(bool create, CancellationToken cancelToken)
{
var parentid = "0";
foreach (var p in m_path.Split(new string[] { "/" }, StringSplitOptions.RemoveEmptyEntries))
foreach (var p in _path.Split(["/"], StringSplitOptions.RemoveEmptyEntries))
{
var el = (MiniFolder)await PagedFileListResponse(parentid, true, cancelToken).FirstOrDefaultAsync(x => x.Name == p).ConfigureAwait(false);
var el = (MiniFolder?)await PagedFileListResponse(parentid, true, cancelToken).FirstOrDefaultAsync(x => x.Name == p, cancellationToken: cancelToken).ConfigureAwait(false);
if (el == null)
{
if (!create)
throw new FolderMissingException();
el = await m_oauth.PostAndGetJSONDataAsync<ListFolderResponse>(
string.Format("{0}/folders", BOX_API_URL),
cancelToken,
new CreateItemRequest() { Name = p, Parent = new IDReference() { ID = parentid } }
).ConfigureAwait(false);
el = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancelToken, ct => _oAuthHelper.PostAndGetJsonDataAsync<ListFolderResponse>(
$"{BOX_API_URL}/folders",
new CreateItemRequest
{
Name = p,
Parent = new IDReference { ID = parentid }
},
ct
)).ConfigureAwait(false);
}
parentid = el.ID;
if (string.IsNullOrWhiteSpace(parentid))
throw new InvalidDataException($"Invalid folder ID for {p} in {_path}");
}
m_currentfolder = parentid;
return _currentFolder = parentid;
}
private async Task<string> GetFileIDAsync(string name, CancellationToken cancelToken)
private async Task<string> GetFileIdAsync(string name, CancellationToken cancelToken)
{
if (m_filecache.ContainsKey(name))
return m_filecache[name];
if (_fileCache.TryGetValue(name, out var async))
return async;
// Make sure we enumerate this, otherwise the m_filecache is empty.
var currentFolder = await GetCurrentFolderWithCacheAsync(cancelToken).ConfigureAwait(false);
await PagedFileListResponse(currentFolder, false, cancelToken).LastOrDefaultAsync().ConfigureAwait(false);
await PagedFileListResponse(currentFolder, false, cancelToken).LastOrDefaultAsync(cancellationToken: cancelToken).ConfigureAwait(false);
if (m_filecache.ContainsKey(name))
return m_filecache[name];
if (_fileCache.TryGetValue(name, out var idAsync))
return idAsync;
throw new FileMissingException();
}
@@ -173,29 +161,30 @@ namespace Duplicati.Library.Backend.Box
var done = false;
if (!onlyfolders)
m_filecache.Clear();
_fileCache.Clear();
do
{
var resp = await m_oauth.GetJSONDataAsync<ShortListResponse>($"{BOX_API_URL}/folders/{parentid}/items?limit={PAGE_SIZE}&offset={offset}&fields=name,size,modified_at", cancelToken).ConfigureAwait(false);
var resp = await Utility.Utility.WithTimeout(_timeouts.ListTimeout, cancelToken, ct => _oAuthHelper.GetJsonDataAsync<ShortListResponse>($"{BOX_API_URL}/folders/{parentid}/items?limit={PAGE_SIZE}&offset={offset}&fields=name,size,modified_at", ct)).ConfigureAwait(false);
if (resp.Entries == null || resp.Entries.Length == 0)
break;
foreach (var f in resp.Entries)
{
if (string.IsNullOrWhiteSpace(f.Name) || string.IsNullOrWhiteSpace(f.ID))
continue;
if (onlyfolders && f.Type != "folder")
{
done = true;
break;
}
else
{
if (!onlyfolders && f.Type == "file")
m_filecache[f.Name] = f.ID;
yield return f;
}
if (!onlyfolders && f.Type == "file")
_fileCache[f.Name] = f.ID;
yield return f;
}
offset = offset + PAGE_SIZE;
@@ -206,62 +195,63 @@ namespace Duplicati.Library.Backend.Box
} while (!done);
}
#region IStreamingBackend implementation
public async Task PutAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken)
public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken)
{
var currentFolder = await GetCurrentFolderWithCacheAsync(cancelToken).ConfigureAwait(false);
var createreq = new CreateItemRequest()
var createreq = new CreateItemRequest
{
Name = remotename,
Parent = new IDReference()
Parent = new IDReference
{
ID = currentFolder
}
};
if (m_filecache.Count == 0)
await PagedFileListResponse(currentFolder, false, cancelToken).LastOrDefaultAsync().ConfigureAwait(false);
if (_fileCache.Count == 0)
await PagedFileListResponse(currentFolder, false, cancelToken).LastOrDefaultAsync(cancelToken).ConfigureAwait(false);
var existing = m_filecache.ContainsKey(remotename);
var existing = _fileCache.ContainsKey(remotename);
var multipartForm = new MultipartFormDataContent();
try
{
string url;
var items = new List<MultipartItem>(2);
if (existing)
url = $"{BOX_UPLOAD_URL}/{m_filecache[remotename]}/content";
url = $"{BOX_UPLOAD_URL}/{_fileCache[remotename]}/content";
else
{
url = $"{BOX_UPLOAD_URL}/content";
items.Add(new MultipartItem(createreq, "attributes"));
multipartForm.Add(JsonContent.Create(createreq), "attributes");
}
items.Add(new MultipartItem(stream, "file", remotename));
using var timeoutStream = stream.ObserveReadTimeout(_timeouts.ReadWriteTimeout, false);
multipartForm.Add(new StreamContent(timeoutStream), "file", remotename);
var res = (await m_oauth.PostMultipartAndGetJSONDataAsync<FileList>(url, null, cancelToken, items.ToArray())).Entries.First();
m_filecache[remotename] = res.ID;
var res = (await _oAuthHelper.PostMultipartAndGetJsonDataAsync<FileList>(url, cancelToken, multipartForm)).Entries?.FirstOrDefault();
if (res == null || string.IsNullOrWhiteSpace(res.ID))
throw new InvalidDataException("No file ID returned after upload");
_fileCache[remotename] = res.ID;
}
catch
{
m_filecache.Clear();
_fileCache.Clear();
throw;
}
}
public async Task GetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken)
public async Task GetAsync(string remotename, Stream stream, CancellationToken cancelToken)
{
var fileId = await GetFileIDAsync(remotename, cancelToken).ConfigureAwait(false);
using (var resp = await m_oauth.GetResponseAsync(string.Format("{0}/files/{1}/content", BOX_API_URL, fileId), cancelToken).ConfigureAwait(false))
using (var rs = Duplicati.Library.Utility.AsyncHttpRequest.TrySetTimeout(resp.GetResponseStream()))
await Library.Utility.Utility.CopyStreamAsync(rs, stream, cancelToken).ConfigureAwait(false);
var fileId = await GetFileIdAsync(remotename, cancelToken).ConfigureAwait(false);
using var request = await _oAuthHelper.CreateRequestAsync($"{BOX_API_URL}/files/{fileId}/content", HttpMethod.Get, cancelToken).ConfigureAwait(false);
using var resp = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancelToken, ct => _oAuthHelper.GetResponseAsync(request, HttpCompletionOption.ResponseHeadersRead, ct)).ConfigureAwait(false);
resp.EnsureSuccessStatusCode();
await using var responseStream = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancelToken, ct => resp.Content.ReadAsStreamAsync(ct)).ConfigureAwait(false);
using var ts = responseStream.ObserveReadTimeout(_timeouts.ReadWriteTimeout);
await Utility.Utility.CopyStreamAsync(ts, stream, cancelToken).ConfigureAwait(false);
}
#endregion
#region IBackend implementation
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancelToken)
{
var currentFolder = await GetCurrentFolderWithCacheAsync(cancelToken).ConfigureAwait(false);
@@ -271,33 +261,37 @@ namespace Duplicati.Library.Backend.Box
public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken)
{
using (System.IO.FileStream fs = System.IO.File.OpenRead(filename))
await PutAsync(remotename, fs, cancelToken);
await using FileStream fs = File.OpenRead(filename);
await PutAsync(remotename, fs, cancelToken).ConfigureAwait(false);
}
public async Task GetAsync(string remotename, string filename, CancellationToken cancelToken)
{
using (System.IO.FileStream fs = System.IO.File.Create(filename))
await GetAsync(remotename, fs, cancelToken).ConfigureAwait(false);
await using FileStream fs = File.Create(filename);
await GetAsync(remotename, fs, cancelToken).ConfigureAwait(false);
}
public async Task DeleteAsync(string remotename, CancellationToken cancelToken)
{
var fileid = await GetFileIDAsync(remotename, cancelToken).ConfigureAwait(false);
var fileId = await GetFileIdAsync(remotename, cancelToken).ConfigureAwait(false);
try
{
using (var r = await m_oauth.GetResponseAsync(string.Format("{0}/files/{1}", BOX_API_URL, fileid), cancelToken, null, "DELETE").ConfigureAwait(false))
using (var request = await _oAuthHelper.CreateRequestAsync($"{BOX_API_URL}/files/{fileId}", HttpMethod.Delete, cancelToken).ConfigureAwait(false))
using (var r = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancelToken, ct => _oAuthHelper.GetResponseAsync(request, HttpCompletionOption.ResponseContentRead, ct)).ConfigureAwait(false))
{
}
if (m_deleteFromTrash)
using (var r = await m_oauth.GetResponseAsync(string.Format("{0}/files/{1}/trash", BOX_API_URL, fileid), cancelToken, null, "DELETE").ConfigureAwait(false))
if (_deleteFromTrash)
{
using (var request = await _oAuthHelper.CreateRequestAsync($"{BOX_API_URL}/files/{fileId}/trash", HttpMethod.Delete, cancelToken).ConfigureAwait(false))
using (var r = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancelToken, ct => _oAuthHelper.GetResponseAsync(request, HttpCompletionOption.ResponseContentRead, ct)).ConfigureAwait(false))
{
}
}
}
catch
{
m_filecache.Clear();
_fileCache.Clear();
throw;
}
}
@@ -310,202 +304,26 @@ namespace Duplicati.Library.Backend.Box
return GetCurrentFolderAsync(true, cancellationToken);
}
public string DisplayName
{
get
{
return Strings.Box.DisplayName;
}
}
public string DisplayName => Strings.Box.DisplayName;
public string ProtocolKey
{
get
{
return "box";
}
}
public string ProtocolKey => "box";
public IList<ICommandLineArgument> SupportedCommands
{
get
{
return new List<ICommandLineArgument>(new ICommandLineArgument[] {
new CommandLineArgument(AUTHID_OPTION, CommandLineArgument.ArgumentType.Password, Strings.Box.AuthidShort, Strings.Box.AuthidLong(OAuthHelper.OAUTH_LOGIN_URL("box.com"))),
new CommandLineArgument(REALLY_DELETE_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.Box.ReallydeleteShort, Strings.Box.ReallydeleteLong),
});
}
}
public IList<ICommandLineArgument> SupportedCommands =>
[
.. AuthIdOptionsHelper.GetOptions(TOKEN_URL),
new CommandLineArgument(REALLY_DELETE_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.Box.ReallydeleteShort, Strings.Box.ReallydeleteLong),
.. TimeoutOptionsHelper.GetOptions()
];
public string Description
{
get
{
return Strings.Box.Description;
}
}
public string Description => Strings.Box.Description;
public Task<string[]> GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new string[] {
public Task<string[]> GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new[] {
new System.Uri(BOX_API_URL).Host,
new System.Uri(BOX_UPLOAD_URL).Host
});
#endregion
#region IDisposable implementation
}.Distinct().WhereNotNullOrWhiteSpace().ToArray());
public void Dispose()
{
}
#endregion
private class MiniUser : IDReference
{
[JsonProperty("type")]
public string Type { get; set; }
[JsonProperty("name")]
public string Name { get; set; }
[JsonProperty("login")]
public string Login { get; set; }
}
private class MiniFolder : IDReference
{
[JsonProperty("type")]
public string Type { get; set; }
[JsonProperty("name")]
public string Name { get; set; }
[JsonProperty("etag")]
public string ETag { get; set; }
[JsonProperty("sequence_id")]
public string SequenceID { get; set; }
}
private class FileEntity : MiniFolder
{
public FileEntity() { Size = -1; }
[JsonProperty("sha1")]
public string SHA1 { get; set; }
[JsonProperty("size", NullValueHandling = NullValueHandling.Ignore)]
public long Size { get; set; }
[JsonProperty("modified_at", NullValueHandling = NullValueHandling.Ignore)]
public DateTime ModifiedAt { get; set; }
}
private class FolderList
{
[JsonProperty("total_count")]
public long TotalCount { get; set; }
[JsonProperty("entries")]
public MiniFolder[] Entries { get; set; }
}
private class FileList
{
[JsonProperty("total_count")]
public long TotalCount { get; set; }
[JsonProperty("entries")]
public FileEntity[] Entries { get; set; }
[JsonProperty("offset")]
public long Offset { get; set; }
[JsonProperty("limit")]
public long Limit { get; set; }
}
private class UploadEmail
{
[JsonProperty("access")]
public string Access { get; set; }
[JsonProperty("email")]
public string Email { get; set; }
}
private class ListFolderResponse : MiniFolder
{
[JsonProperty("created_at")]
public DateTime CreatedAt { get; set; }
[JsonProperty("modified_at")]
public DateTime ModifiedAt { get; set; }
[JsonProperty("description")]
public string Description { get; set; }
[JsonProperty("size")]
public long Size { get; set; }
[JsonProperty("path_collection")]
public FolderList PathCollection { get; set; }
[JsonProperty("created_by")]
public MiniUser CreatedBy { get; set; }
[JsonProperty("modified_by")]
public MiniUser ModifiedBy { get; set; }
[JsonProperty("owned_by")]
public MiniUser OwnedBy { get; set; }
[JsonProperty("shared_link")]
public MiniUser SharedLink { get; set; }
[JsonProperty("folder_upload_email")]
public UploadEmail FolderUploadEmail { get; set; }
[JsonProperty("parent")]
public MiniFolder Parent { get; set; }
[JsonProperty("item_status")]
public string ItemStatus { get; set; }
[JsonProperty("item_collection")]
public FileList ItemCollection { get; set; }
}
private class OrderEntry
{
[JsonProperty("by")]
public string By { get; set; }
[JsonProperty("direction")]
public string Direction { get; set; }
}
private class ShortListResponse : FileList
{
[JsonProperty("order")]
public OrderEntry[] Order { get; set; }
}
private class IDReference
{
[JsonProperty("id")]
public string ID { get; set; }
}
private class CreateItemRequest
{
[JsonProperty("name")]
public string Name { get; set; }
[JsonProperty("parent")]
public IDReference Parent { get; set; }
}
private class ErrorResponse
{
[JsonProperty("type")]
public string Type { get; set; }
[JsonProperty("status")]
public int Status { get; set; }
[JsonProperty("code")]
public string Code { get; set; }
[JsonProperty("help_url")]
public string HelpUrl { get; set; }
[JsonProperty("message")]
public string Message { get; set; }
[JsonProperty("request_id")]
public string RequestId { get; set; }
}
}
}
@@ -4,6 +4,9 @@
<TargetFramework>net8.0</TargetFramework>
<OutputType>Library</OutputType>
<Copyright>Copyright © 2025 Team Duplicati, MIT license</Copyright>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<RootNamespace>Duplicati.Library.Backend.Box</RootNamespace>
</PropertyGroup>
<ItemGroup>
@@ -0,0 +1,9 @@
using Newtonsoft.Json;
namespace Duplicati.Library.Backend.Box;
public class CreateItemRequest
{
[JsonProperty("name")] public string? Name { get; set; }
[JsonProperty("parent")] public IDReference? Parent { get; set; }
}
@@ -0,0 +1,13 @@
using Newtonsoft.Json;
namespace Duplicati.Library.Backend.Box;
public class ErrorResponse
{
[JsonProperty("type")] public string? Type { get; set; }
[JsonProperty("status")] public int Status { get; set; }
[JsonProperty("code")] public string? Code { get; set; }
[JsonProperty("help_url")] public string? HelpUrl { get; set; }
[JsonProperty("message")] public string? Message { get; set; }
[JsonProperty("request_id")] public string? RequestId { get; set; }
}
@@ -0,0 +1,19 @@
using Newtonsoft.Json;
namespace Duplicati.Library.Backend.Box;
public class FileEntity : MiniFolder
{
public FileEntity()
{
Size = -1;
}
[JsonProperty("sha1")] public string? SHA1 { get; set; }
[JsonProperty("size", NullValueHandling = NullValueHandling.Ignore)]
public long Size { get; set; }
[JsonProperty("modified_at", NullValueHandling = NullValueHandling.Ignore)]
public DateTime ModifiedAt { get; set; }
}
@@ -0,0 +1,11 @@
using Newtonsoft.Json;
namespace Duplicati.Library.Backend.Box;
public class FileList
{
[JsonProperty("total_count")] public long TotalCount { get; set; }
[JsonProperty("entries")] public FileEntity[]? Entries { get; set; }
[JsonProperty("offset")] public long Offset { get; set; }
[JsonProperty("limit")] public long Limit { get; set; }
}
@@ -0,0 +1,9 @@
using Newtonsoft.Json;
namespace Duplicati.Library.Backend.Box;
public class FolderList
{
[JsonProperty("total_count")] public long TotalCount { get; set; }
[JsonProperty("entries")] public MiniFolder[]? Entries { get; set; }
}
@@ -0,0 +1,8 @@
using Newtonsoft.Json;
namespace Duplicati.Library.Backend.Box;
public class IDReference
{
[JsonProperty("id")] public string? ID { get; set; }
}
@@ -0,0 +1,21 @@
using System;
using Newtonsoft.Json;
namespace Duplicati.Library.Backend.Box;
public class ListFolderResponse : MiniFolder
{
[JsonProperty("created_at")] public DateTime CreatedAt { get; set; }
[JsonProperty("modified_at")] public DateTime ModifiedAt { get; set; }
[JsonProperty("description")] public string? Description { get; set; }
[JsonProperty("size")] public long Size { get; set; }
[JsonProperty("path_collection")] public FolderList? PathCollection { get; set; }
[JsonProperty("created_by")] public MiniUser? CreatedBy { get; set; }
[JsonProperty("modified_by")] public MiniUser? ModifiedBy { get; set; }
[JsonProperty("owned_by")] public MiniUser? OwnedBy { get; set; }
[JsonProperty("shared_link")] public MiniUser? SharedLink { get; set; }
[JsonProperty("folder_upload_email")] public UploadEmail? FolderUploadEmail { get; set; }
[JsonProperty("parent")] public MiniFolder? Parent { get; set; }
[JsonProperty("item_status")] public string? ItemStatus { get; set; }
[JsonProperty("item_collection")] public FileList? ItemCollection { get; set; }
}
@@ -0,0 +1,11 @@
using Newtonsoft.Json;
namespace Duplicati.Library.Backend.Box;
public class MiniFolder : IDReference
{
[JsonProperty("type")] public string? Type { get; set; }
[JsonProperty("name")] public string? Name { get; set; }
[JsonProperty("etag")] public string? ETag { get; set; }
[JsonProperty("sequence_id")] public string? SequenceID { get; set; }
}
@@ -0,0 +1,10 @@
using Newtonsoft.Json;
namespace Duplicati.Library.Backend.Box;
public class MiniUser : IDReference
{
[JsonProperty("type")] public string? Type { get; set; }
[JsonProperty("name")] public string? Name { get; set; }
[JsonProperty("login")] public string? Login { get; set; }
}
@@ -0,0 +1,9 @@
using Newtonsoft.Json;
namespace Duplicati.Library.Backend.Box;
public class OrderEntry
{
[JsonProperty("by")] public string? By { get; set; }
[JsonProperty("direction")] public string? Direction { get; set; }
}
@@ -0,0 +1,8 @@
using Newtonsoft.Json;
namespace Duplicati.Library.Backend.Box;
public class ShortListResponse : FileList
{
[JsonProperty("order")] public OrderEntry[]? Order { get; set; }
}
@@ -0,0 +1,9 @@
using Newtonsoft.Json;
namespace Duplicati.Library.Backend.Box;
public class UploadEmail
{
[JsonProperty("access")] public string? Access { get; set; }
[JsonProperty("email")] public string? Email { get; set; }
}
+12 -13
View File
@@ -17,16 +17,15 @@
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Localization.Short;
namespace Duplicati.Library.Backend.Strings
{
internal static class Box {
public static string Description { get { return LC.L(@"This backend can read and write data to Box.com. Allowed format is ""box://folder/subfolder""."); } }
public static string DisplayName { get { return LC.L(@"Box.com"); } }
public static string AuthidLong(string url) { return LC.L(@"The authorization token retrieved from {0}", url); }
public static string AuthidShort { get { return LC.L(@"The authorization code"); } }
public static string ReallydeleteLong { get { return LC.L(@"After deleting a file, it may end up in the trash folder where it will be deleted after a grace period. Use this command to force immediate removal of delete files."); } }
public static string ReallydeleteShort { get { return LC.L(@"Force delete files"); } }
}
}
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Localization.Short;
namespace Duplicati.Library.Backend.Strings
{
internal static class Box
{
public static string Description { get { return LC.L(@"This backend can read and write data to Box.com. Allowed format is ""box://folder/subfolder""."); } }
public static string DisplayName { get { return LC.L(@"Box.com"); } }
public static string ReallydeleteLong { get { return LC.L(@"After deleting a file, it may end up in the trash folder where it will be deleted after a grace period. Use this command to force immediate removal of delete files."); } }
public static string ReallydeleteShort { get { return LC.L(@"Force delete files"); } }
}
}
@@ -1,109 +0,0 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using SMBLibrary;
namespace Duplicati.Library.Backend.CIFS.Model;
/// <summary>
/// Connection parameters for establishing an SMB connection.
/// </summary>
public record SMBConnectionParameters
{
/// <summary>
/// The name or IP address of the SMB server
/// </summary>
public string ServerName { get; init; }
/// <summary>
/// The transport protocol type used for SMB communication
/// </summary>
public SMBTransportType TransportType { get; init; }
/// <summary>
/// The name of the network share to connect to
/// </summary>
public string ShareName { get; init; }
/// <summary>
/// The path within the share to access
/// </summary>
public string Path { get; init; }
/// <summary>
/// The authentication domain name
/// </summary>
public string AuthDomain { get; init; }
/// <summary>
/// The username for authentication
/// </summary>
public string AuthUser { get; init; }
/// <summary>
/// The password for authentication
/// </summary>
public string AuthPassword { get; init; }
/// <summary>
/// Write buffer size for SMB operations (will be capped automatically by SMB negotiated values)
/// </summary>
public int? WriteBufferSize { get; init; }
/// <summary>
/// Read buffer size for SMB operations (will be capped automatically by SMB negotiated values)
/// </summary>
public int? ReadBufferSize { get; init; }
/// <summary>
/// Creates a new instance of SMB connection parameters
/// </summary>
/// <param name="serverName">The name or IP address of the SMB server</param>
/// <param name="transportType">The transport protocol type used for SMB communication</param>
/// <param name="shareName">The name of the network share to connect to</param>
/// <param name="path">The path within the share to access</param>
/// <param name="authDomain">The authentication domain name</param>
/// <param name="authUser">The username for authentication</param>
/// <param name="authPassword">The password for authentication</param>
/// <param name="readBufferSize">Read buffer size for SMB operations (will be capped automatically by SMB negotiated values)</param>
/// <param name="writeBufferSize">Write buffer size for SMB operations (will be capped automatically by SMB negotiated values)</param>
public SMBConnectionParameters(
string serverName,
SMBTransportType transportType,
string shareName,
string path,
string authDomain,
string authUser,
string authPassword,
int? readBufferSize,
int? writeBufferSize)
{
ServerName = serverName;
TransportType = transportType;
ShareName = shareName;
Path = path;
AuthDomain = authDomain;
AuthUser = authUser;
AuthPassword = authPassword;
ReadBufferSize = readBufferSize;
WriteBufferSize = writeBufferSize;
}
}
+122 -115
View File
@@ -21,14 +21,11 @@
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using System;
using System.Collections.Generic;
using System.Linq;
using Duplicati.Library.Utility;
using Duplicati.Library.Utility.Options;
using System.Net;
using System.Runtime.CompilerServices;
using System.Security.Cryptography;
using System.Threading;
using System.Threading.Tasks;
namespace Duplicati.Library.Backend
{
@@ -36,54 +33,57 @@ namespace Duplicati.Library.Backend
// This class is instantiated dynamically in the BackendLoader.
public class CloudFiles : IBackend, IStreamingBackend
{
/// <summary>
/// The log tag for this class
/// </summary>
private static readonly string LOGTAG = Logging.Log.LogTagFromType<CloudFiles>();
public const string AUTH_URL_US = "https://identity.api.rackspacecloud.com/auth";
public const string AUTH_URL_UK = "https://lon.auth.api.rackspacecloud.com/v1.0";
private const string DUMMY_HOSTNAME = "api.mosso.com";
private const string AUTH_USERNAME_OPTION = "cloudfiles-username";
private const string AUTH_PASSWORD_OPTION = "cloudfiles-accesskey";
private const int ITEM_LIST_LIMIT = 1000;
private readonly string m_username;
private readonly string m_password;
private readonly string m_path;
private string m_storageUrl = null;
private string m_authToken = null;
private string? m_storageUrl = null;
private string? m_authToken = null;
private readonly string m_authUrl;
private readonly TimeoutOptionsHelper.Timeouts m_timeouts;
// ReSharper disable once UnusedMember.Global
// This constructor is needed by the BackendLoader.
public CloudFiles()
{
m_username = null!;
m_password = null!;
m_path = null!;
m_authUrl = null!;
m_timeouts = null!;
}
// ReSharper disable once UnusedMember.Global
// This constructor is needed by the BackendLoader.
public CloudFiles(string url, Dictionary<string, string> options)
public CloudFiles(string url, Dictionary<string, string?> options)
{
var uri = new Utility.Uri(url);
var auth = AuthOptionsHelper.ParseWithAlias(options, uri, AUTH_USERNAME_OPTION, AUTH_PASSWORD_OPTION);
if (options.ContainsKey("auth-username"))
m_username = options["auth-username"];
if (options.ContainsKey("auth-password"))
m_password = options["auth-password"];
if (options.ContainsKey("cloudfiles-username"))
m_username = options["cloudfiles-username"];
if (options.ContainsKey("cloudfiles-accesskey"))
m_password = options["cloudfiles-accesskey"];
if (!string.IsNullOrEmpty(uri.Username))
m_username = uri.Username;
if (!string.IsNullOrEmpty(uri.Password))
m_password = uri.Password;
if (string.IsNullOrEmpty(m_username))
if (!auth.HasUsername)
throw new UserInformationException(Strings.CloudFiles.NoUserIDError, "CloudFilesNoUserID");
if (string.IsNullOrEmpty(m_password))
if (auth.HasPassword)
throw new UserInformationException(Strings.CloudFiles.NoAPIKeyError, "CloudFilesNoApiKey");
(m_username, m_password) = auth.GetCredentials();
//Fallback to the previous format
if (url.Contains(DUMMY_HOSTNAME))
{
Logging.Log.WriteWarningMessage(LOGTAG, "CloudFilesDeprecatedFormat", null, Strings.CloudFiles.DeprecatedFormatWarning(DUMMY_HOSTNAME));
var u = new System.Uri(url);
if (!string.IsNullOrEmpty(u.UserInfo))
@@ -117,8 +117,11 @@ namespace Duplicati.Library.Backend
if (!m_path.StartsWith("/", StringComparison.Ordinal))
m_path = "/" + m_path;
if (!options.TryGetValue("cloudfiles-authentication-url", out m_authUrl))
m_authUrl = Utility.Utility.ParseBoolOption(options, "cloudfiles-uk-account") ? AUTH_URL_UK : AUTH_URL_US;
var authUrl = options.GetValueOrDefault("cloudfiles-authentication-url");
if (string.IsNullOrEmpty(authUrl))
authUrl = Utility.Utility.ParseBoolOption(options, "cloudfiles-uk-account") ? AUTH_URL_UK : AUTH_URL_US;
m_authUrl = authUrl;
m_timeouts = TimeoutOptionsHelper.Parse(options);
}
#region IBackend Members
@@ -139,19 +142,20 @@ namespace Duplicati.Library.Backend
var markerUrl = "";
bool repeat;
do
{
var doc = new System.Xml.XmlDocument();
var req = CreateRequest("", extraUrl + markerUrl);
var req = await CreateRequest("", extraUrl + markerUrl, cancelToken).ConfigureAwait(false);
try
{
var areq = new Utility.AsyncHttpRequest(req);
using (var resp = (HttpWebResponse)areq.GetResponse())
using (var s = areq.GetResponseStream())
doc.Load(s);
await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, _ =>
{
var areq = new Utility.AsyncHttpRequest(req);
using (var resp = (HttpWebResponse)areq.GetResponse())
using (var s = areq.GetResponseStream())
doc.Load(s);
}).ConfigureAwait(false);
}
catch (WebException wex)
{
@@ -163,26 +167,26 @@ namespace Duplicati.Library.Backend
throw;
}
System.Xml.XmlNodeList lst = doc.SelectNodes("container/object");
var lst = doc.SelectNodes("container/object");
//Perhaps the folder does not exist?
//The response should be 404 from the server, but it is not :(
if (lst.Count == 0 && markerUrl == "") //Only on first iteration
if (lst == null || lst.Count == 0 && markerUrl == "") //Only on first iteration
{
try { await CreateFolderAsync(cancelToken).ConfigureAwait(false); }
catch { } //Ignore
}
if (lst == null)
yield break;
var lastItemName = "";
foreach (System.Xml.XmlNode n in lst)
{
var name = n["name"].InnerText;
long size;
DateTime mod;
if (!long.TryParse(n["bytes"].InnerText, out size))
var name = n["name"]?.InnerText;
if (!long.TryParse(n["bytes"]?.InnerText, out var size))
size = -1;
if (!DateTime.TryParse(n["last_modified"].InnerText, out mod))
if (!DateTime.TryParse(n["last_modified"]?.InnerText, out var mod))
mod = new DateTime();
lastItemName = name;
@@ -199,61 +203,55 @@ namespace Duplicati.Library.Backend
public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken)
{
using (System.IO.FileStream fs = System.IO.File.OpenRead(filename))
using (var fs = File.OpenRead(filename))
await PutAsync(remotename, fs, cancelToken);
}
public async Task GetAsync(string remotename, string filename, CancellationToken cancelToken)
{
using (System.IO.FileStream fs = System.IO.File.Create(filename))
using (var fs = File.Create(filename))
await GetAsync(remotename, fs, cancelToken).ConfigureAwait(false);
}
public Task DeleteAsync(string remotename, CancellationToken cancelToken)
public async Task DeleteAsync(string remotename, CancellationToken cancelToken)
{
try
{
var req = CreateRequest("/" + remotename, "");
req.Method = "DELETE";
var areq = new Utility.AsyncHttpRequest(req);
using (var resp = (HttpWebResponse)areq.GetResponse())
var req = await CreateRequest("/" + remotename, "", cancelToken).ConfigureAwait(false);
await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ =>
{
if (resp.StatusCode == System.Net.HttpStatusCode.NotFound)
throw new FileMissingException();
req.Method = "DELETE";
var areq = new AsyncHttpRequest(req);
using (var resp = (HttpWebResponse)areq.GetResponse())
{
if (resp.StatusCode == HttpStatusCode.NotFound)
throw new FileMissingException();
if ((int)resp.StatusCode >= 300)
throw new WebException(Strings.CloudFiles.FileDeleteError, null, WebExceptionStatus.ProtocolError, resp);
else
using (areq.GetResponseStream())
{ }
}
if ((int)resp.StatusCode >= 300)
throw new WebException(Strings.CloudFiles.FileDeleteError, null, WebExceptionStatus.ProtocolError, resp);
else
using (areq.GetResponseStream())
{ }
}
}).ConfigureAwait(false);
}
catch (System.Net.WebException wex)
catch (WebException wex)
{
if (wex.Response is HttpWebResponse response && response.StatusCode == System.Net.HttpStatusCode.NotFound)
if (wex.Response is HttpWebResponse response && response.StatusCode == HttpStatusCode.NotFound)
throw new FileMissingException(wex);
else
throw;
}
return Task.CompletedTask;
}
public IList<ICommandLineArgument> SupportedCommands
{
get
{
return new List<ICommandLineArgument>([
new CommandLineArgument("auth-password", CommandLineArgument.ArgumentType.Password, Strings.CloudFiles.DescriptionAuthPasswordShort, Strings.CloudFiles.DescriptionAuthPasswordLong),
new CommandLineArgument("auth-username", CommandLineArgument.ArgumentType.String, Strings.CloudFiles.DescriptionAuthUsernameShort, Strings.CloudFiles.DescriptionAuthUsernameLong),
new CommandLineArgument("cloudfiles-username", CommandLineArgument.ArgumentType.String, Strings.CloudFiles.DescriptionUsernameShort, Strings.CloudFiles.DescriptionUsernameLong, null, new string[] {"auth-username"} ),
new CommandLineArgument("cloudfiles-accesskey", CommandLineArgument.ArgumentType.Password, Strings.CloudFiles.DescriptionPasswordShort, Strings.CloudFiles.DescriptionPasswordLong, null, new string[] {"auth-password"}),
new CommandLineArgument("cloudfiles-uk-account", CommandLineArgument.ArgumentType.Boolean, Strings.CloudFiles.DescriptionUKAccountShort, Strings.CloudFiles.DescriptionUKAccountLong("cloudfiles-authentication-url", AUTH_URL_UK)),
new CommandLineArgument("cloudfiles-authentication-url", CommandLineArgument.ArgumentType.String, Strings.CloudFiles.DescriptionAuthenticationURLShort, Strings.CloudFiles.DescriptionAuthenticationURLLong_v2("cloudfiles-uk-account"), AUTH_URL_US),
]);
}
}
public IList<ICommandLineArgument> SupportedCommands =>
[
new CommandLineArgument(AUTH_USERNAME_OPTION, CommandLineArgument.ArgumentType.String, Strings.CloudFiles.DescriptionUsernameShort, Strings.CloudFiles.DescriptionUsernameLong, null, [AuthOptionsHelper.AuthUsernameOption] ),
new CommandLineArgument(AUTH_PASSWORD_OPTION, CommandLineArgument.ArgumentType.Password, Strings.CloudFiles.DescriptionPasswordShort, Strings.CloudFiles.DescriptionPasswordLong, null, [AuthOptionsHelper.AuthPasswordOption]),
new CommandLineArgument("cloudfiles-uk-account", CommandLineArgument.ArgumentType.Boolean, Strings.CloudFiles.DescriptionUKAccountShort, Strings.CloudFiles.DescriptionUKAccountLong("cloudfiles-authentication-url", AUTH_URL_UK)),
new CommandLineArgument("cloudfiles-authentication-url", CommandLineArgument.ArgumentType.String, Strings.CloudFiles.DescriptionAuthenticationURLShort, Strings.CloudFiles.DescriptionAuthenticationURLLong_v2("cloudfiles-uk-account"), AUTH_URL_US),
.. TimeoutOptionsHelper.GetOptions(),
];
public string Description
{
@@ -268,15 +266,16 @@ namespace Duplicati.Library.Backend
//The "Folder not found" is not detectable :(
this.TestListAsync(cancelToken);
public Task CreateFolderAsync(CancellationToken cancelToken)
public async Task CreateFolderAsync(CancellationToken cancelToken)
{
var createReq = CreateRequest("", "");
createReq.Method = "PUT";
var areq = new Utility.AsyncHttpRequest(createReq);
using (var resp = (HttpWebResponse)areq.GetResponse())
{ }
return Task.CompletedTask;
var createReq = await CreateRequest("", "", cancelToken).ConfigureAwait(false);
await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ =>
{
createReq.Method = "PUT";
var areq = new AsyncHttpRequest(createReq);
using (var resp = (HttpWebResponse)areq.GetResponse())
{ }
}).ConfigureAwait(false);
}
#endregion
@@ -292,42 +291,46 @@ namespace Duplicati.Library.Backend
#region IStreamingBackend Members
public Task<string[]> GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(
new string[] {
new string?[] {
new System.Uri(m_authUrl).Host,
string.IsNullOrWhiteSpace(m_storageUrl) ? null : new System.Uri(m_storageUrl).Host
}
.Where(x => !string.IsNullOrWhiteSpace(x))
.WhereNotNullOrWhiteSpace()
.ToArray()
);
public async Task GetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken)
{
var req = CreateRequest("/" + remotename, "");
var req = await CreateRequest("/" + remotename, "", cancelToken).ConfigureAwait(false);
req.Method = "GET";
var areq = new Utility.AsyncHttpRequest(req);
var areq = new AsyncHttpRequest(req);
using (var resp = areq.GetResponse())
using (var s = areq.GetResponseStream())
using (var s = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => areq.GetResponseStream()).ConfigureAwait(false))
using (var timeoutStream = s.ObserveReadTimeout(m_timeouts.ReadWriteTimeout))
using (var hasher = MD5.Create())
using (var mds = new Utility.HashCalculatingStream(s, hasher))
using (var mds = new HashCalculatingStream(timeoutStream, hasher))
{
string md5Hash = resp.Headers["ETag"];
var md5Hash = resp.Headers["ETag"];
await Utility.Utility.CopyStreamAsync(mds, stream, true, cancelToken).ConfigureAwait(false);
if (!String.Equals(mds.GetFinalHashString(), md5Hash, StringComparison.OrdinalIgnoreCase))
if (!string.Equals(mds.GetFinalHashString(), md5Hash, StringComparison.OrdinalIgnoreCase))
throw new Exception(Strings.CloudFiles.ETagVerificationError);
}
}
public async Task PutAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken)
{
HttpWebRequest req = CreateRequest("/" + remotename, "");
var req = await CreateRequest("/" + remotename, "", cancelToken).ConfigureAwait(false);
req.Method = "PUT";
req.ContentType = "application/octet-stream";
try { req.ContentLength = stream.Length; }
catch { }
// TODO: When reviewing this, lets build a common method to unwrap the stream passed from
// BackendManager.PutOperation, so we can use the same logic to compute hashes in all backends
//If we can pre-calculate the MD5 hash before transmission, do so
/*if (stream.CanSeek)
{
@@ -353,22 +356,23 @@ namespace Duplicati.Library.Backend
//TODO: We cannot use the local MD5 calculation, because that could involve a throttled read,
// and may invoke various events
{
string fileHash = null;
string? fileHash = null;
long streamLen = -1;
try { streamLen = stream.Length; }
catch { }
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (System.IO.Stream s = areq.GetRequestStream(streamLen))
var areq = new AsyncHttpRequest(req);
using (var s = areq.GetRequestStream(streamLen))
using (var timeoutStream = s.ObserveWriteTimeout(m_timeouts.ReadWriteTimeout))
using (var hasher = MD5.Create())
using (var mds = new Utility.HashCalculatingStream(s, hasher))
using (var mds = new HashCalculatingStream(timeoutStream, hasher))
{
await Utility.Utility.CopyStreamAsync(stream, mds, tryRewindSource: true, cancelToken: cancelToken);
fileHash = mds.GetFinalHashString();
}
string md5Hash = null;
string? md5Hash = null;
//We need to verify the eTag locally
try
@@ -390,7 +394,7 @@ namespace Duplicati.Library.Backend
}
if (md5Hash == null || !String.Equals(md5Hash, fileHash, StringComparison.OrdinalIgnoreCase))
if (md5Hash == null || !string.Equals(md5Hash, fileHash, StringComparison.OrdinalIgnoreCase))
{
//Remove the broken file
try { await DeleteAsync(remotename, cancelToken); }
@@ -403,29 +407,32 @@ namespace Duplicati.Library.Backend
#endregion
private HttpWebRequest CreateRequest(string remotename, string query)
private async Task<HttpWebRequest> CreateRequest(string remotename, string query, CancellationToken cancelToken)
{
//If this is the first call, get an authentication token
if (string.IsNullOrEmpty(m_authToken) || string.IsNullOrEmpty(m_storageUrl))
{
var authReq = (HttpWebRequest)HttpWebRequest.Create(m_authUrl);
authReq.Headers.Add("X-Auth-User", m_username);
authReq.Headers.Add("X-Auth-Key", m_password);
authReq.Method = "GET";
var areq = new Utility.AsyncHttpRequest(authReq);
using (var resp = areq.GetResponse())
await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ =>
{
m_storageUrl = resp.Headers["X-Storage-Url"];
m_authToken = resp.Headers["X-Auth-Token"];
}
var authReq = (HttpWebRequest)HttpWebRequest.Create(m_authUrl);
authReq.Headers.Add("X-Auth-User", m_username);
authReq.Headers.Add("X-Auth-Key", m_password);
authReq.Method = "GET";
if (string.IsNullOrEmpty(m_authToken) || string.IsNullOrEmpty(m_storageUrl))
throw new Exception(Strings.CloudFiles.UnexpectedResponseError);
var areq = new AsyncHttpRequest(authReq);
using (var resp = areq.GetResponse())
{
m_storageUrl = resp.Headers["X-Storage-Url"];
m_authToken = resp.Headers["X-Auth-Token"];
}
if (string.IsNullOrEmpty(m_authToken) || string.IsNullOrEmpty(m_storageUrl))
throw new Exception(Strings.CloudFiles.UnexpectedResponseError);
}).ConfigureAwait(false);
}
var req = (HttpWebRequest)HttpWebRequest.Create(m_storageUrl + UrlEncode(m_path + remotename) + query);
req.Headers.Add("X-Auth-Token", UrlEncode(m_authToken));
req.Headers.Add("X-Auth-Token", UrlEncode(m_authToken!));
req.UserAgent = "Duplicati CloudFiles Backend v" + System.Reflection.Assembly.GetExecutingAssembly().GetName().Version;
req.KeepAlive = false;
@@ -435,9 +442,9 @@ namespace Duplicati.Library.Backend
return req;
}
private string UrlEncode(string value)
private static string UrlEncode(string value)
{
return Library.Utility.Uri.UrlEncode(value).Replace("+", "%20").Replace("%2f", "/");
return Utility.Uri.UrlEncode(value).Replace("+", "%20").Replace("%2f", "/");
}
}
}
@@ -5,6 +5,8 @@
<OutputType>Library</OutputType>
<RootNamespace>Duplicati.Library.Backend</RootNamespace>
<Copyright>Copyright © 2025 Team Duplicati, MIT license</Copyright>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>
<ItemGroup>
+25 -26
View File
@@ -1,34 +1,32 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Localization.Short;
namespace Duplicati.Library.Backend.Strings {
internal static class CloudFiles {
namespace Duplicati.Library.Backend.Strings
{
internal static class CloudFiles
{
public static string Description_v2 { get { return LC.L(@"This backend can read and write data to CloudFiles. Allowed format is ""cloudfiles://container/folder""."); } }
public static string DisplayName { get { return LC.L(@"Rackspace CloudFiles"); } }
public static string DescriptionAuthenticationURLLong_v2(string optionname) { return LC.L(@"CloudFiles use different servers for authentication based on where the account resides. Use this option to set an alternate authentication URL. This option overrides --{0}.", optionname); }
public static string DescriptionAuthenticationURLShort { get { return LC.L(@"Provide another authentication URL"); } }
public static string DescriptionAuthPasswordLong { get { return LC.L(@"The password used to connect to the server. This may also be supplied as the environment variable ""AUTH_PASSWORD""."); } }
public static string DescriptionAuthPasswordShort { get { return LC.L(@"Supply the password used to connect to the server"); } }
public static string DescriptionAuthUsernameLong { get { return LC.L(@"The username used to connect to the server. This may also be supplied as the environment variable ""AUTH_USERNAME""."); } }
public static string DescriptionAuthUsernameShort { get { return LC.L(@"Supply the username used to connect to the server"); } }
public static string DescriptionPasswordLong { get { return LC.L(@"The API Access Key used to authenticate with CloudFiles."); } }
public static string DescriptionPasswordShort { get { return LC.L(@"Supply the access key used to connect to the server"); } }
public static string DescriptionUKAccountLong(string optionname, string optionvalue) { return LC.L(@"Duplicati will assume that the credentials given are for a US account. Use this option if the account is a UK based account. Note that this is equivalent to setting --{0}={1}.", optionname, optionvalue); }
@@ -41,5 +39,6 @@ namespace Duplicati.Library.Backend.Strings {
public static string NoAPIKeyError { get { return LC.L(@"No CloudFiles API Access Key given"); } }
public static string NoUserIDError { get { return LC.L(@"No CloudFiles userID given"); } }
public static string UnexpectedResponseError { get { return LC.L(@"Unexpected CloudFiles response. Perhaps the API has changed?"); } }
public static string DeprecatedFormatWarning(string hostname) { return LC.L(@$"The URL is using the marker hostname {hostname} which is no longer supported. Please use the new format ""cloudfiles://container/folder"" instead."); }
}
}
+46 -71
View File
@@ -1,31 +1,27 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using System;
using System.Collections.Generic;
using System.IO;
using Duplicati.Library.Utility.Options;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
namespace Duplicati.Library.Backend
{
@@ -33,9 +29,7 @@ namespace Duplicati.Library.Backend
// This class is instantiated dynamically in the BackendLoader.
public class Dropbox : IBackend, IStreamingBackend
{
private const string AUTHID_OPTION = "authid";
private readonly string m_accesToken;
private static readonly string TOKEN_URL = OAuthHelper.OAUTH_LOGIN_URL("dropbox");
private readonly string m_path;
private readonly DropboxHelper dbx;
@@ -43,25 +37,27 @@ namespace Duplicati.Library.Backend
// This constructor is needed by the BackendLoader.
public Dropbox()
{
m_path = null!;
dbx = null!;
}
// ReSharper disable once UnusedMember.Global
// This constructor is needed by the BackendLoader.
public Dropbox(string url, Dictionary<string, string> options)
public Dropbox(string url, Dictionary<string, string?> options)
{
var uri = new Utility.Uri(url);
m_path = Library.Utility.Uri.UrlDecode(uri.HostAndPath);
m_path = Utility.Uri.UrlDecode(uri.HostAndPath);
if (m_path.Length != 0 && !m_path.StartsWith("/", StringComparison.Ordinal))
m_path = "/" + m_path;
if (m_path.EndsWith("/", StringComparison.Ordinal))
m_path = m_path.Substring(0, m_path.Length - 1);
if (options.ContainsKey(AUTHID_OPTION))
m_accesToken = options[AUTHID_OPTION];
var authId = AuthIdOptionsHelper.Parse(options);
authId.RequireCredentials(TOKEN_URL);
dbx = new DropboxHelper(m_accesToken);
dbx = new DropboxHelper(authId.AuthId!, TimeoutOptionsHelper.Parse(options));
}
public void Dispose()
@@ -69,15 +65,9 @@ namespace Duplicati.Library.Backend
// do nothing
}
public string DisplayName
{
get { return Strings.Dropbox.DisplayName; }
}
public string DisplayName => Strings.Dropbox.DisplayName;
public string ProtocolKey
{
get { return "dropbox"; }
}
public string ProtocolKey => "dropbox";
private IFileEntry ParseEntry(MetaData md)
{
@@ -92,27 +82,16 @@ namespace Duplicati.Library.Backend
ife.IsFolder = true;
}
try { ife.LastModification = ife.LastAccess = DateTime.Parse(md.server_modified).ToUniversalTime(); }
try
{
if (!string.IsNullOrWhiteSpace(md.server_modified))
ife.LastModification = ife.LastAccess = DateTime.Parse(md.server_modified).ToUniversalTime();
}
catch { }
return ife;
}
private T HandleListExceptions<T>(Func<T> func)
{
try
{
return func();
}
catch (DropboxException de)
{
if (de.errorJSON["error"][".tag"].ToString() == "path" && de.errorJSON["error"]["path"][".tag"].ToString() == "not_found")
throw new FolderMissingException();
throw;
}
}
private async Task<T> HandleListExceptions<T>(Func<Task<T>> func)
{
try
@@ -121,7 +100,7 @@ namespace Duplicati.Library.Backend
}
catch (DropboxException de)
{
if (de.errorJSON["error"][".tag"].ToString() == "path" && de.errorJSON["error"]["path"][".tag"].ToString() == "not_found")
if (de.errorJSON?["error"]?[".tag"]?.ToString() == "path" && de.errorJSON?["error"]?["path"]?[".tag"]?.ToString() == "not_found")
throw new FolderMissingException();
throw;
@@ -132,13 +111,13 @@ namespace Duplicati.Library.Backend
{
var lfr = await HandleListExceptions(() => dbx.ListFiles(m_path, cancelToken)).ConfigureAwait(false);
foreach (var md in lfr.entries)
foreach (var md in lfr.entries ?? [])
yield return ParseEntry(md);
while (lfr.has_more)
{
lfr = await HandleListExceptions(() => dbx.ListFilesContinue(lfr.cursor, cancelToken)).ConfigureAwait(false);
foreach (var md in lfr.entries)
lfr = await HandleListExceptions(() => dbx.ListFilesContinue(lfr.cursor!, cancelToken)).ConfigureAwait(false);
foreach (var md in lfr.entries ?? [])
yield return ParseEntry(md);
}
}
@@ -159,7 +138,7 @@ namespace Duplicati.Library.Backend
{
try
{
string path = String.Format("{0}/{1}", m_path, remotename);
var path = $"{m_path}/{remotename}";
await dbx.DeleteAsync(path, cancelToken).ConfigureAwait(false);
}
catch (DropboxException)
@@ -169,15 +148,11 @@ namespace Duplicati.Library.Backend
}
}
public IList<ICommandLineArgument> SupportedCommands
{
get
{
return new List<ICommandLineArgument>([
new CommandLineArgument(AUTHID_OPTION, CommandLineArgument.ArgumentType.Password, Strings.Dropbox.AuthidShort, Strings.Dropbox.AuthidLong(OAuthHelper.OAUTH_LOGIN_URL("dropbox"))),
]);
}
}
public IList<ICommandLineArgument> SupportedCommands =>
[
.. AuthIdOptionsHelper.GetOptions(TOKEN_URL),
.. TimeoutOptionsHelper.GetOptions(),
];
public string Description { get { return Strings.Dropbox.Description; } }
@@ -195,7 +170,7 @@ namespace Duplicati.Library.Backend
catch (DropboxException de)
{
if (de.errorJSON["error"][".tag"].ToString() == "path" && de.errorJSON["error"]["path"][".tag"].ToString() == "conflict")
if (de.errorJSON?["error"]?[".tag"]?.ToString() == "path" && de.errorJSON["error"]?["path"]?[".tag"]?.ToString() == "conflict")
throw new FolderAreadyExistedException();
throw;
}
@@ -1,31 +1,29 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Utility;
using Duplicati.Library.Utility.Options;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System;
using System.IO;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
namespace Duplicati.Library.Backend
{
@@ -34,9 +32,12 @@ namespace Duplicati.Library.Backend
private const int DROPBOX_MAX_CHUNK_UPLOAD = 10 * 1024 * 1024; // 10 MB max upload
private const string API_ARG_HEADER = "DROPBOX-API-arg";
public DropboxHelper(string accessToken)
private readonly TimeoutOptionsHelper.Timeouts m_timeouts;
public DropboxHelper(string accessToken, TimeoutOptionsHelper.Timeouts timeouts)
: base(accessToken, "dropbox")
{
m_timeouts = timeouts;
base.AutoAuthHeader = true;
// Pre 2022 tokens are direct Dropbox tokens (no ':')
// Post 2022-02-21 tokens are regular authid tokens (with a ':')
@@ -52,7 +53,9 @@ namespace Duplicati.Library.Backend
try
{
return await PostAndGetJSONDataAsync<ListFolderResult>(WebApi.Dropbox.ListFilesUrl(), cancelToken, pa).ConfigureAwait(false);
return await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, ct =>
PostAndGetJSONDataAsync<ListFolderResult>(WebApi.Dropbox.ListFilesUrl(), ct, pa)
).ConfigureAwait(false);
}
catch (Exception ex)
{
@@ -67,7 +70,9 @@ namespace Duplicati.Library.Backend
try
{
return await PostAndGetJSONDataAsync<ListFolderResult>(WebApi.Dropbox.ListFilesContinueUrl(), cancelToken, lfca).ConfigureAwait(false);
return await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, ct =>
PostAndGetJSONDataAsync<ListFolderResult>(WebApi.Dropbox.ListFilesContinueUrl(), ct, lfca)
).ConfigureAwait(false);
}
catch (Exception ex)
{
@@ -82,7 +87,9 @@ namespace Duplicati.Library.Backend
try
{
return await PostAndGetJSONDataAsync<FolderMetadata>(WebApi.Dropbox.CreateFolderUrl(), cancellationToken, pa).ConfigureAwait(false);
return await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancellationToken, ct =>
PostAndGetJSONDataAsync<FolderMetadata>(WebApi.Dropbox.CreateFolderUrl(), ct, pa)
).ConfigureAwait(false);
}
catch (Exception ex)
{
@@ -111,13 +118,14 @@ namespace Duplicati.Library.Backend
ulong globalBytesRead = 0;
using (var rs = areq.GetRequestStream())
using (var ts = rs.ObserveWriteTimeout(m_timeouts.ReadWriteTimeout))
{
int bytesRead = 0;
do
{
bytesRead = await stream.ReadAsync(buffer, 0, sizeToRead, cancelToken).ConfigureAwait(false);
globalBytesRead += (ulong)bytesRead;
await rs.WriteAsync(buffer, 0, bytesRead, cancelToken).ConfigureAwait(false);
await ts.WriteAsync(buffer, 0, bytesRead, cancelToken).ConfigureAwait(false);
}
while (bytesRead > 0 && globalBytesRead < (ulong)chunksize);
}
@@ -149,6 +157,7 @@ namespace Duplicati.Library.Backend
int bytesReadInRequest = 0;
sizeToRead = Math.Min(chunksize, (int)Utility.Utility.DEFAULT_BUFFER_SIZE);
using (var rs = areq.GetRequestStream())
using (var ts = rs.ObserveWriteTimeout(m_timeouts.ReadWriteTimeout))
{
int bytesRead = 0;
do
@@ -156,15 +165,18 @@ namespace Duplicati.Library.Backend
bytesRead = await stream.ReadAsync(buffer, 0, sizeToRead, cancelToken).ConfigureAwait(false);
bytesReadInRequest += bytesRead;
globalBytesRead += (ulong)bytesRead;
await rs.WriteAsync(buffer, 0, bytesRead, cancelToken).ConfigureAwait(false);
await ts.WriteAsync(buffer, 0, bytesRead, cancelToken).ConfigureAwait(false);
}
while (bytesRead > 0 && bytesReadInRequest < chunksize);
}
using (var response = GetResponse(areq))
using (var sr = new StreamReader(response.GetResponseStream()))
await sr.ReadToEndAsync().ConfigureAwait(false);
await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, async ct =>
{
using (var response = GetResponse(areq))
using (var sr = new StreamReader(response.GetResponseStream()))
await sr.ReadToEndAsync().ConfigureAwait(false);
}).ConfigureAwait(false);
}
// finish session and commit
@@ -180,7 +192,9 @@ namespace Duplicati.Library.Backend
req.ContentType = "application/octet-stream";
req.Timeout = 200000;
return ReadJSONResponse<FileMetaData>(req);
return await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct =>
ReadJSONResponseAsync<FileMetaData>(req, ct)
).ConfigureAwait(false);
}
catch (Exception ex)
{
@@ -198,8 +212,10 @@ namespace Duplicati.Library.Backend
var req = CreateRequest(WebApi.Dropbox.DownloadFilesUrl(), "POST");
req.Headers[API_ARG_HEADER] = JsonConvert.SerializeObject(pa);
using (var response = GetResponse(req))
await Utility.Utility.CopyStreamAsync(response.GetResponseStream(), fs, cancelToken).ConfigureAwait(false);
using (var response = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => GetResponse(req)))
using (var rs = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => response.GetResponseStream()))
using (var ts = rs.ObserveReadTimeout(m_timeouts.ReadWriteTimeout))
await Utility.Utility.CopyStreamAsync(ts, fs, cancelToken).ConfigureAwait(false);
}
catch (Exception ex)
{
@@ -212,10 +228,13 @@ namespace Duplicati.Library.Backend
{
try
{
var pa = new PathArg() { path = path };
using (var response = await GetResponseAsync(WebApi.Dropbox.DeleteUrl(), cancelToken, pa).ConfigureAwait(false))
using (var sr = new StreamReader(response.GetResponseStream()))
sr.ReadToEnd();
await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, async ct =>
{
var pa = new PathArg() { path = path };
using (var response = await GetResponseAsync(WebApi.Dropbox.DeleteUrl(), ct, pa).ConfigureAwait(false))
using (var sr = new StreamReader(response.GetResponseStream()))
await sr.ReadToEndAsync(ct).ConfigureAwait(false);
}).ConfigureAwait(false);
}
catch (Exception ex)
{
@@ -232,8 +251,9 @@ namespace Duplicati.Library.Backend
try
{
using (var sr = new StreamReader(exception.Response.GetResponseStream()))
json = sr.ReadToEnd();
if (exception.Response != null)
using (var sr = new StreamReader(exception.Response.GetResponseStream()))
json = sr.ReadToEnd();
}
catch { }
@@ -245,27 +265,27 @@ namespace Duplicati.Library.Backend
if (httpResp.StatusCode == HttpStatusCode.NotFound)
{
if (filerequest)
throw new Duplicati.Library.Interface.FileMissingException(json);
throw new Interface.FileMissingException(json);
else
throw new Duplicati.Library.Interface.FolderMissingException(json);
throw new Interface.FolderMissingException(json);
}
if (httpResp.StatusCode == HttpStatusCode.Conflict)
{
//TODO: Should actually parse and see if something else happens
if (filerequest)
throw new Duplicati.Library.Interface.FileMissingException(json);
throw new Interface.FileMissingException(json);
else
throw new Duplicati.Library.Interface.FolderMissingException(json);
throw new Interface.FolderMissingException(json);
}
if (httpResp.StatusCode == HttpStatusCode.Unauthorized)
ThrowAuthException(json, exception);
if ((int)httpResp.StatusCode == 429 || (int)httpResp.StatusCode == 507)
throw new Duplicati.Library.Interface.UserInformationException(Strings.Dropbox.OverQuotaError(string.IsNullOrWhiteSpace(json) ? exception.Message : json), "DropboxOverQuotaError", ex);
throw new Interface.UserInformationException(Strings.Dropbox.OverQuotaError(string.IsNullOrWhiteSpace(json) ? exception.Message : json), "DropboxOverQuotaError", ex);
}
JObject errJson = null;
JObject? errJson = null;
try
{
errJson = JObject.Parse(json);
@@ -276,20 +296,22 @@ namespace Duplicati.Library.Backend
if (errJson != null)
throw new DropboxException() { errorJSON = errJson };
else if (exception.Response is HttpWebResponse wresp)
throw new InvalidDataException($"Non-json response (code: {wresp.StatusCode}, message: {wresp.StatusDescription}): {json}", ex);
else
throw new InvalidDataException($"Non-json response: {json}");
throw new InvalidDataException($"Non-json response: {json}", ex);
}
}
}
public class DropboxException : Exception
{
public JObject errorJSON { get; set; }
public JObject? errorJSON { get; set; }
}
public class PathArg
{
public string path { get; set; }
public string? path { get; set; }
}
public class FolderMetadata : MetaData
@@ -330,7 +352,7 @@ namespace Duplicati.Library.Backend
public class UploadSessionCursor
{
public string session_id { get; set; }
public string? session_id { get; set; }
public ulong offset { get; set; }
}
@@ -342,7 +364,7 @@ namespace Duplicati.Library.Backend
autorename = false;
mute = true;
}
public string path { get; set; }
public string? path { get; set; }
public string mode { get; set; }
public bool autorename { get; set; }
public bool mute { get; set; }
@@ -351,41 +373,40 @@ namespace Duplicati.Library.Backend
public class UploadSessionStartResult
{
public string session_id { get; set; }
public string? session_id { get; set; }
}
public class ListFolderResult
{
public MetaData[]? entries { get; set; }
public MetaData[] entries { get; set; }
public string cursor { get; set; }
public string? cursor { get; set; }
public bool has_more { get; set; }
}
public class ListFolderContinueArg
{
public string cursor { get; set; }
public string? cursor { get; set; }
}
public class MetaData
{
[JsonProperty(".tag")]
public string tag { get; set; }
public string name { get; set; }
public string server_modified { get; set; }
public string? tag { get; set; }
public string? name { get; set; }
public string? server_modified { get; set; }
public ulong size { get; set; }
public bool IsFile { get { return tag == "file"; } }
// While this is unused, the Dropbox API v2 documentation does not
// declare this to be optional.
// ReSharper disable once UnusedMember.Global
public string id { get; set; }
public string? id { get; set; }
// While this is unused, the Dropbox API v2 documentation does not
// declare this to be optional.
// ReSharper disable once UnusedMember.Global
public string rev { get; set; }
public string? rev { get; set; }
}
public class FileMetaData : MetaData
@@ -4,6 +4,8 @@
<TargetFramework>net8.0</TargetFramework>
<Description>Dropbox backend for Duplicati</Description>
<Copyright>Copyright © 2025 Team Duplicati, MIT license</Copyright>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+29 -50
View File
@@ -1,80 +1,59 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Utility;
namespace Duplicati.Library.Backend.WebApi
{
public static class Dropbox
{
public static string CreateFolderUrl()
{
return Uri.UriBuilder(Url.API, Path.CreateFolder);
}
=> Utility.Uri.UriBuilder(Url.API, Path.CreateFolder);
public static string ListFilesUrl()
{
return Uri.UriBuilder(Url.API, Path.ListFolder);
}
=> Utility.Uri.UriBuilder(Url.API, Path.ListFolder);
public static string ListFilesContinueUrl()
{
return Uri.UriBuilder(Url.API, Path.ListFolderContinue);
}
=> Utility.Uri.UriBuilder(Url.API, Path.ListFolderContinue);
public static string DeleteUrl()
{
return Uri.UriBuilder(Url.API, Path.DeleteFolder);
}
=> Utility.Uri.UriBuilder(Url.API, Path.DeleteFolder);
public static string UploadSessionStartUrl()
{
return Uri.UriBuilder(Url.CONTENT_API_URL, Path.UploadSessionStart);
}
=> Utility.Uri.UriBuilder(Url.CONTENT_API_URL, Path.UploadSessionStart);
public static string UploadSessionAppendUrl()
{
return Uri.UriBuilder(Url.CONTENT_API_URL, Path.UploadSessionAppend);
}
=> Utility.Uri.UriBuilder(Url.CONTENT_API_URL, Path.UploadSessionAppend);
public static string UploadSessionFinishUrl()
{
return Uri.UriBuilder(Url.CONTENT_API_URL, Path.UploadSessionFinish);
}
=> Utility.Uri.UriBuilder(Url.CONTENT_API_URL, Path.UploadSessionFinish);
public static string DownloadFilesUrl()
{
return Uri.UriBuilder(Url.CONTENT_API_URL, Path.DownloadFiles);
}
=> Utility.Uri.UriBuilder(Url.CONTENT_API_URL, Path.DownloadFiles);
public static string[] Hosts()
{
return new[] { new System.Uri(Url.API).Host, new System.Uri(Url.CONTENT_API_URL).Host };
}
=> [new Uri(Url.API).Host, new Uri(Url.CONTENT_API_URL).Host];
private static class Url
{
public const string API = "https://api.dropboxapi.com/2";
public const string CONTENT_API_URL = "https://content.dropboxapi.com/2";
public const string CONTENT_API_URL = "https://content.dropboxapi.com/2";
}
private static class Path
@@ -18,8 +18,6 @@
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Collections.Generic;
using System.Security.Authentication;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility.Options;
@@ -61,7 +59,7 @@ public class AlternateFTPBackend : FTP
{
}
public AlternateFTPBackend(string url, Dictionary<string, string> options) : base(url, options)
public AlternateFTPBackend(string url, Dictionary<string, string?> options) : base(url, options)
{
}
@@ -4,6 +4,8 @@
<TargetFramework>net8.0</TargetFramework>
<OutputType>Library</OutputType>
<Copyright>Copyright © 2025 Team Duplicati, MIT license</Copyright>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>
<ItemGroup>
+11 -32
View File
@@ -19,20 +19,12 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Security;
using System.Runtime.CompilerServices;
using System.Security.Authentication;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
@@ -268,36 +260,23 @@ namespace Duplicati.Library.Backend
[SuppressMessage("ReSharper", "VirtualMemberCallInConstructor")] // The behavior of accessing the virtual properties is as expected
public FTP(string url, Dictionary<string, string?> options)
{
_sslOptions = SslOptionsHelper.SslCertificateOptions.Parse(options);
_sslOptions = SslOptionsHelper.Parse(options);
var u = new Utility.Uri(url);
u.RequireHost();
if (!string.IsNullOrEmpty(u.Username))
var auth = AuthOptionsHelper.Parse(options, u);
if (auth.HasUsername)
{
_userInfo = new NetworkCredential
_userInfo = new NetworkCredential()
{
UserName = u.Username
UserName = auth.Username,
Domain = ""
};
if (!string.IsNullOrEmpty(u.Password))
_userInfo.Password = u.Password;
else if (options.ContainsKey("auth-password"))
_userInfo.Password = options["auth-password"];
}
else
{
if (options.ContainsKey("auth-username"))
{
_userInfo = new NetworkCredential();
_userInfo.UserName = options["auth-username"];
if (options.ContainsKey("auth-password"))
_userInfo.Password = options["auth-password"];
}
}
//Bugfix, see http://connect.microsoft.com/VisualStudio/feedback/details/695227/networkcredential-default-constructor-leaves-domain-null-leading-to-null-object-reference-exceptions-in-framework-code
if (_userInfo != null)
_userInfo.Domain = "";
if (auth.HasPassword)
_userInfo.Password = auth.Password;
}
var parsedurl = u.SetScheme("ftp").SetQuery(null).SetCredentials(null, null).ToString();
parsedurl = Util.AppendDirSeparator(parsedurl, "/");
@@ -341,7 +320,7 @@ namespace Duplicati.Library.Backend
_logToConsole = CoreUtility.ParseBoolOption(options, CONFIG_KEY_FTP_LOGTOCONSOLE);
_logPrivateInfoToConsole = CoreUtility.ParseBoolOption(options, CONFIG_KEY_FTP_LOGPRIVATEINFOTOCONSOLE);
_diagnosticsLog = CoreUtility.ParseBoolOption(options, CONFIG_KEY_FTP_LOGDIAGNOSTICS);
_timeouts = TimeoutOptionsHelper.Timeouts.Parse(options);
_timeouts = TimeoutOptionsHelper.Parse(options);
_ftpConfig = new FtpConfig
{
@@ -478,7 +457,7 @@ namespace Duplicati.Library.Backend
{
var client = await CreateClient(cancelToken).ConfigureAwait(false);
var clientRemoteName = PreparePathForClient(remotename);
await using var inputStream = await client.OpenRead(clientRemoteName, token: cancelToken);
await using var inputStream = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancelToken, ct => client.OpenRead(clientRemoteName, token: ct)).ConfigureAwait(false);
await using var timeoutStream = inputStream.ObserveReadTimeout(_timeouts.ReadWriteTimeout);
await CoreUtility.CopyStreamAsync(inputStream, output, false, cancelToken).ConfigureAwait(false);
}
+1 -1
View File
@@ -69,6 +69,6 @@ namespace Duplicati.Library.Backend
public static string DescriptionRelativePathLong { get { return LC.L(@"Use this option to interpret the url path as a path that is relative to the initial login folder. This option only has an effect if the initial starting folder in the FTP server is not the (virtual) root folder. If not set, the path in the url is treated as absolute, ignoring the initial login folder."); } }
public static string DescriptionUseCwdNamesShort { get { return LC.L(@"Use CWD instead of absolute paths"); } }
public static string DescriptionUseCwdNamesLong { get { return LC.L(@"Use this option to start the connection with a CWD command instead of an absolute path. This can be useful if the FTP server does not support absolute paths."); } }
public static string ErrorCreateFolder(string targetFolderName, string resultingFolder) { return LC.L(@"Error creating folder {0}, gave folder: {1}", targetFolderName, resultingFolder); }
public static string ErrorCreateFolder(string targetFolderName, string? resultingFolder) { return LC.L(@"Error creating folder {0}, gave folder: {1}", targetFolderName, resultingFolder); }
}
}
@@ -5,6 +5,8 @@
<OutputType>Library</OutputType>
<RootNamespace>Duplicati.Library.Backend</RootNamespace>
<Copyright>Copyright © 2025 Team Duplicati, MIT license</Copyright>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>
<ItemGroup>
+62 -61
View File
@@ -21,14 +21,11 @@
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Duplicati.Library.Utility;
using Duplicati.Library.Utility.Options;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Versioning;
using System.Threading;
using System.Threading.Tasks;
namespace Duplicati.Library.Backend
{
@@ -43,71 +40,65 @@ namespace Duplicati.Library.Backend
private const string OPTION_DISABLE_LENGTH_VERIFICATION = "disable-length-verification";
private readonly string m_path;
private string m_username;
private string m_password;
private string? m_username;
private string? m_password;
private readonly bool m_moveFile;
private bool m_hasAutenticated;
private readonly bool m_forceReauth;
private readonly bool m_verifyDestinationLength;
private readonly TimeoutOptionsHelper.Timeouts m_timeouts;
private static readonly ISystemIO systemIO = SystemIO.IO_OS;
public File()
{
m_path = null!;
m_timeouts = null!;
}
public File(string url, Dictionary<string, string> options)
public File(string url, Dictionary<string, string?> options)
{
var uri = new Utility.Uri(url);
m_path = uri.HostAndPath;
var path = uri.HostAndPath;
var auth = AuthOptionsHelper.Parse(options, uri);
m_timeouts = TimeoutOptionsHelper.Parse(options);
m_username = auth.Username;
m_password = auth.Password;
if (options.ContainsKey("auth-username"))
m_username = options["auth-username"];
if (options.ContainsKey("auth-password"))
m_password = options["auth-password"];
if (!string.IsNullOrEmpty(uri.Username))
m_username = uri.Username;
if (!string.IsNullOrEmpty(uri.Password))
m_password = uri.Password;
if (!Path.IsPathRooted(path))
path = systemIO.PathGetFullPath(path);
if (!System.IO.Path.IsPathRooted(m_path))
m_path = systemIO.PathGetFullPath(m_path);
if (options.ContainsKey(OPTION_ALTERNATE_PATHS))
var altPaths = options.GetValueOrDefault(OPTION_ALTERNATE_PATHS);
if (!string.IsNullOrWhiteSpace(altPaths))
{
List<string> paths =
[
m_path,
.. options[OPTION_ALTERNATE_PATHS].Split(new string[] { System.IO.Path.PathSeparator.ToString() }, StringSplitOptions.RemoveEmptyEntries),
path,
.. altPaths.Split(new string[] {Path.PathSeparator.ToString() }, StringSplitOptions.RemoveEmptyEntries),
];
//On windows we expand the drive letter * to all drives
if (OperatingSystem.IsWindows())
{
System.IO.DriveInfo[] drives = System.IO.DriveInfo.GetDrives();
for (int i = 0; i < paths.Count; i++)
var drives = DriveInfo.GetDrives();
for (var i = 0; i < paths.Count; i++)
{
if (paths[i].StartsWith("*:", StringComparison.Ordinal))
{
string rpl_path = paths[i].Substring(1);
var rpl_path = paths[i].Substring(1);
paths.RemoveAt(i);
i--;
foreach (System.IO.DriveInfo di in drives)
foreach (var di in drives)
paths.Insert(++i, di.Name[0] + rpl_path);
}
}
}
string markerfile = null;
//If there is a marker file, we do not allow the primary target path
// to be accepted, unless it contains the marker file
if (options.ContainsKey(OPTION_DESTINATION_MARKER))
{
markerfile = options[OPTION_DESTINATION_MARKER];
m_path = null;
}
var markerfile = options.GetValueOrDefault(OPTION_DESTINATION_MARKER);
if (!string.IsNullOrWhiteSpace(markerfile))
path = null;
foreach (string p in paths)
{
@@ -115,7 +106,7 @@ namespace Duplicati.Library.Backend
{
if (systemIO.DirectoryExists(p) && (markerfile == null || systemIO.FileExists(systemIO.PathCombine(p, markerfile))))
{
m_path = p;
path = p;
break;
}
}
@@ -124,10 +115,11 @@ namespace Duplicati.Library.Backend
}
}
if (m_path == null)
if (string.IsNullOrWhiteSpace(path))
throw new UserInformationException(Strings.FileBackend.NoDestinationWithMarkerFileError(markerfile, paths.ToArray()), "NoDestinationWithMarker");
}
m_path = path;
m_moveFile = Utility.Utility.ParseBoolOption(options, OPTION_MOVE_FILE);
m_forceReauth = Utility.Utility.ParseBoolOption(options, OPTION_FORCE_REAUTH);
m_verifyDestinationLength = !Utility.Utility.ParseBoolOption(options, OPTION_DISABLE_LENGTH_VERIFICATION);
@@ -174,14 +166,19 @@ namespace Duplicati.Library.Backend
}
/// <inheritdoc />
public IAsyncEnumerable<IFileEntry> ListAsync(CancellationToken cancelToken)
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancelToken)
{
PreAuthenticate();
if (!systemIO.DirectoryExists(m_path))
throw new FolderMissingException(Strings.FileBackend.FolderMissingError(m_path));
return systemIO.EnumerateFileEntries(m_path).ToAsyncEnumerable();
var res = await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, _ =>
systemIO.EnumerateFileEntries(m_path)
).ConfigureAwait(false);
foreach (var entry in res)
yield return entry;
}
#if DEBUG_RETRY
@@ -201,17 +198,19 @@ namespace Duplicati.Library.Backend
string targetFilePath = GetRemoteName(targetFilename);
long copiedBytes = 0;
using (var targetStream = systemIO.FileCreate(targetFilePath))
copiedBytes = await Utility.Utility.CopyStreamAsync(sourceStream, targetStream, true, cancelToken).ConfigureAwait(false);
using (var timeoutStream = targetStream.ObserveWriteTimeout(m_timeouts.ReadWriteTimeout))
copiedBytes = await Utility.Utility.CopyStreamAsync(sourceStream, timeoutStream, true, cancelToken).ConfigureAwait(false);
VerifyMatchingSize(targetFilePath, sourceStream, copiedBytes);
}
#endif
public async Task GetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken)
public async Task GetAsync(string remotename, Stream stream, CancellationToken cancelToken)
{
// FileOpenRead has flags System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read
using (var readstream = systemIO.FileOpenRead(GetRemoteName(remotename)))
await Utility.Utility.CopyStreamAsync(readstream, stream, true, cancelToken).ConfigureAwait(false);
using (var timeoutStream = readstream.ObserveReadTimeout(m_timeouts.ReadWriteTimeout))
await Utility.Utility.CopyStreamAsync(timeoutStream, stream, true, cancelToken).ConfigureAwait(false);
}
public Task PutAsync(string targetFilename, string sourceFilePath, CancellationToken cancelToken)
@@ -245,10 +244,11 @@ namespace Duplicati.Library.Backend
return Task.CompletedTask;
}
public Task DeleteAsync(string remotename, CancellationToken cancelToken)
public async Task DeleteAsync(string remotename, CancellationToken cancelToken)
{
systemIO.FileDelete(GetRemoteName(remotename));
return Task.CompletedTask;
await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _
=> systemIO.FileDelete(GetRemoteName(remotename))
).ConfigureAwait(false);
}
public IList<ICommandLineArgument> SupportedCommands
@@ -258,8 +258,7 @@ namespace Duplicati.Library.Backend
var lst = new List<ICommandLineArgument>();
if (OperatingSystem.IsWindows())
lst.AddRange([
new CommandLineArgument("auth-username", CommandLineArgument.ArgumentType.String, Strings.FileBackend.DescriptionAuthUsernameShort, Strings.FileBackend.DescriptionAuthUsernameLong),
new CommandLineArgument("auth-password", CommandLineArgument.ArgumentType.Password, Strings.FileBackend.DescriptionAuthPasswordShort, Strings.FileBackend.DescriptionAuthPasswordLong),
.. AuthOptionsHelper.GetOptions(),
new CommandLineArgument(OPTION_FORCE_REAUTH, CommandLineArgument.ArgumentType.Boolean, Strings.FileBackend.ForceReauthShort, Strings.FileBackend.ForceReauthLong)
]);
@@ -268,6 +267,7 @@ namespace Duplicati.Library.Backend
new CommandLineArgument(OPTION_ALTERNATE_PATHS, CommandLineArgument.ArgumentType.Path, Strings.FileBackend.AlternateTargetPathsShort, Strings.FileBackend.AlternateTargetPathsLong(OPTION_DESTINATION_MARKER, System.IO.Path.PathSeparator)),
new CommandLineArgument(OPTION_MOVE_FILE, CommandLineArgument.ArgumentType.Boolean, Strings.FileBackend.UseMoveForPutShort, Strings.FileBackend.UseMoveForPutLong),
new CommandLineArgument(OPTION_DISABLE_LENGTH_VERIFICATION, CommandLineArgument.ArgumentType.Boolean, Strings.FileBackend.DisableLengthVerificationShort, Strings.FileBackend.DisableLengthVerificationLong),
.. TimeoutOptionsHelper.GetOptions()
]);
return lst;
@@ -285,13 +285,14 @@ namespace Duplicati.Library.Backend
public Task TestAsync(CancellationToken cancelToken)
=> this.TestListAsync(cancelToken);
public Task CreateFolderAsync(CancellationToken cancelToken)
public async Task CreateFolderAsync(CancellationToken cancelToken)
{
if (systemIO.DirectoryExists(m_path))
throw new FolderAreadyExistedException();
systemIO.DirectoryCreate(m_path);
return Task.CompletedTask;
await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _
=> systemIO.DirectoryCreate(m_path)
).ConfigureAwait(false);
}
#endregion
@@ -306,7 +307,7 @@ namespace Duplicati.Library.Backend
#endregion
private System.IO.DriveInfo GetDrive()
private DriveInfo? GetDrive()
{
string root;
if (!OperatingSystem.IsWindows())
@@ -314,7 +315,7 @@ namespace Duplicati.Library.Backend
string path = Util.AppendDirSeparator(systemIO.PathGetFullPath(m_path));
// If the built-in .NET DriveInfo works, use it
try { return new System.IO.DriveInfo(path); }
try { return new DriveInfo(path); }
catch { }
root = "/";
@@ -322,7 +323,7 @@ namespace Duplicati.Library.Backend
//Find longest common prefix from mounted devices
//TODO: Can trick this with symlinks, where the symlink is on one mounted volume,
// and the actual storage is on another
foreach (System.IO.DriveInfo di in System.IO.DriveInfo.GetDrives())
foreach (var di in DriveInfo.GetDrives())
if (path.StartsWith(Util.AppendDirSeparator(di.Name), StringComparison.Ordinal) && di.Name.Length > root.Length)
root = di.Name;
}
@@ -337,7 +338,7 @@ namespace Duplicati.Library.Backend
{
try
{
return new System.IO.DriveInfo(root);
return new DriveInfo(root);
}
catch (ArgumentException)
{
@@ -348,7 +349,7 @@ namespace Duplicati.Library.Backend
return null;
}
public Task<IQuotaInfo> GetQuotaInfoAsync(CancellationToken cancelToken)
public Task<IQuotaInfo?> GetQuotaInfoAsync(CancellationToken cancelToken)
{
var driveInfo = this.GetDrive();
if (driveInfo != null)
@@ -357,7 +358,7 @@ namespace Duplicati.Library.Backend
// If the drive actually has a total size of 0, this should be obvious immediately due to write errors
if (driveInfo.TotalSize > 0)
{
return Task.FromResult<IQuotaInfo>(new QuotaInfo(driveInfo.TotalSize, driveInfo.AvailableFreeSpace));
return Task.FromResult<IQuotaInfo?>(new QuotaInfo(driveInfo.TotalSize, driveInfo.AvailableFreeSpace));
}
}
@@ -365,10 +366,10 @@ namespace Duplicati.Library.Backend
{
// If we can't get the DriveInfo on Windows, fallback to GetFreeDiskSpaceEx
// https://stackoverflow.com/questions/2050343/programmatically-determining-space-available-from-unc-path
return Task.FromResult<IQuotaInfo>(GetDiskFreeSpace(m_path));
return Task.FromResult<IQuotaInfo?>(GetDiskFreeSpace(m_path));
}
return null;
return Task.FromResult<IQuotaInfo?>(null);
}
public Task<string[]> GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(Array.Empty<string>());
@@ -391,7 +392,7 @@ namespace Duplicati.Library.Backend
/// <returns>Quota info</returns>
[System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)]
[SupportedOSPlatform("windows")]
public static QuotaInfo GetDiskFreeSpace(string directory)
public static QuotaInfo? GetDiskFreeSpace(string directory)
{
ulong available;
ulong total;
@@ -439,7 +440,7 @@ namespace Duplicati.Library.Backend
}
}
private static void VerifyMatchingSize(string targetFilePath, Stream sourceStream, long? expectedLength)
private static void VerifyMatchingSize(string targetFilePath, Stream? sourceStream, long? expectedLength)
{
try
{
+20 -24
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Localization.Short;
namespace Duplicati.Library.Backend.Strings
@@ -29,12 +29,8 @@ namespace Duplicati.Library.Backend.Strings
public static string AlternateDestinationMarkerShort { get { return LC.L(@"Look for a file in the destination folder"); } }
public static string AlternateTargetPathsLong(string optionname, char pathseparator) { return LC.L(@"This option allows multiple targets to be specified. The primary target path is placed before the list of paths supplied with this option. Before starting the backup, each folder in the list is checked for existence and optionally the presence of the marker file supplied by --{0}. The first existing path that optionally contains the marker file is then used as the destination. Multiple destinations are separated with a ""{1}"". On Windows, the path may be a UNC path, and the drive letter may be substituted with an asterisk (*), e.g: ""*:\backup"", which will examine all drive letters. If a username and password is supplied, the same credentials are used for all destinations.", optionname, pathseparator); }
public static string AlternateTargetPathsShort { get { return LC.L(@"A list of secondary target paths"); } }
public static string DescriptionAuthPasswordLong { get { return LC.L(@"The password used to connect to the server. This may also be supplied as the environment variable ""AUTH_PASSWORD""."); } }
public static string DescriptionAuthPasswordShort { get { return LC.L(@"Supply the password used to connect to the server"); } }
public static string DescriptionAuthUsernameLong { get { return LC.L(@"The username used to connect to the server. This may also be supplied as the environment variable ""AUTH_USERNAME""."); } }
public static string DescriptionAuthUsernameShort { get { return LC.L(@"Supply the username used to connect to the server"); } }
public static string FolderMissingError(string foldername) { return LC.L(@"The folder {0} does not exist", foldername); }
public static string NoDestinationWithMarkerFileError(string markername, string[] folders) { return LC.L(@"The marker file ""{0}"" was not found in any of the examined destinations: {1}", markername, string.Join(", ", folders)); }
public static string NoDestinationWithMarkerFileError(string? markername, string[] folders) { return LC.L(@"The marker file ""{0}"" was not found in any of the examined destinations: {1}", markername, string.Join(", ", folders)); }
public static string UseMoveForPutLong { get { return LC.L(@"When storing the file, the standard operation is to copy the file and delete the original. This sequence ensures that the operation can be retried if something goes wrong. Activating this option may cause the retry operation to fail. This option has no effect unless the option --{0} is activated.", "disable-streaming-transfers"); } }
public static string UseMoveForPutShort { get { return LC.L(@"Move the file instead of copying it"); } }
public static string ForceReauthLong { get { return LC.L(@"If this option is set, any existing authentication against the remote share is dropped before attempting to authenticate."); } }
+21 -22
View File
@@ -1,25 +1,24 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Runtime.InteropServices;
using System.Runtime.Versioning;
@@ -41,7 +40,7 @@ namespace Duplicati.Library.Backend
public ResourceType dwType;
public ResourceDisplayType dwDisplayType;
public ResourceUsage dwUsage;
public string LocalName;
public string? LocalName;
public string RemoteName;
public string Comment;
public string Provider;
@@ -111,7 +110,7 @@ namespace Duplicati.Library.Backend
try
{
NETRESOURCE rsc = new NETRESOURCE();
var rsc = new NETRESOURCE();
rsc.dwScope = ResourceScope.RESOURCE_GLOBALNET;
rsc.dwType = ResourceType.RESOURCETYPE_DISK;
@@ -4,7 +4,10 @@
<TargetFramework>net8.0</TargetFramework>
<OutputType>Library</OutputType>
<Copyright>Copyright © 2025 Team Duplicati, MIT license</Copyright>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
@@ -1,29 +1,25 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using Duplicati.Library.Interface;
using System.Collections.Generic;
using System.Linq;
using Duplicati.Library.Localization.Short;
namespace Duplicati.Library.Backend.GoogleServices
{
@@ -32,7 +28,7 @@ namespace Duplicati.Library.Backend.GoogleServices
public class GCSConfig : IWebModule
{
private const ConfigType DEFAULT_CONFIG_TYPE = ConfigType.Locations;
private static readonly string DEFAULT_CONFIG_TYPE_STR = Enum.GetName(typeof(ConfigType), DEFAULT_CONFIG_TYPE);
private static readonly string DEFAULT_CONFIG_TYPE_STR = DEFAULT_CONFIG_TYPE.ToString();
private const string KEY_CONFIGTYPE = "gcs-config";
public enum ConfigType
@@ -46,15 +42,13 @@ namespace Duplicati.Library.Backend.GoogleServices
#region IWebModule implementation
public System.Collections.Generic.IDictionary<string, string> Execute(System.Collections.Generic.IDictionary<string, string> options)
public IDictionary<string, string?> Execute(IDictionary<string, string?> options)
{
string k;
options.TryGetValue(KEY_CONFIGTYPE, out k);
options.TryGetValue(KEY_CONFIGTYPE, out var k);
if (string.IsNullOrWhiteSpace(k))
k = DEFAULT_CONFIG_TYPE_STR;
ConfigType ct;
if (!Enum.TryParse<ConfigType>(k, true, out ct))
if (!Enum.TryParse<ConfigType>(k, true, out var ct))
ct = DEFAULT_CONFIG_TYPE;
switch (ct)
@@ -66,23 +60,17 @@ namespace Duplicati.Library.Backend.GoogleServices
}
}
public string Key { get { return "gcs-getconfig"; } }
public string Key => "gcs-getconfig";
public string DisplayName { get { return LC.L("Google Cloud Storage configuration module"); } }
public string DisplayName => Strings.GCSConfig.DisplayName;
public string Description { get { return LC.L("Expose Google Cloud Storage configuration as a web module"); } }
public string Description => Strings.GCSConfig.Description;
public System.Collections.Generic.IList<ICommandLineArgument> SupportedCommands
{
get
{
return new List<ICommandLineArgument>([
new CommandLineArgument(KEY_CONFIGTYPE, CommandLineArgument.ArgumentType.Enumeration, LC.L("The config to get"), LC.L("Provide different config values"), DEFAULT_CONFIG_TYPE_STR, Enum.GetNames(typeof(ConfigType)))
]);
}
}
public IList<ICommandLineArgument> SupportedCommands =>
[
new CommandLineArgument(KEY_CONFIGTYPE, CommandLineArgument.ArgumentType.Enumeration, Strings.GCSConfig.ConfigTypeShort, Strings.GCSConfig.ConfigTypeLong, DEFAULT_CONFIG_TYPE_STR, Enum.GetNames(typeof(ConfigType)))
];
#endregion
}
@@ -23,14 +23,11 @@ using Duplicati.Library.Backend.GoogleServices;
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
using Duplicati.Library.Utility.Options;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Net;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace Duplicati.Library.Backend.GoogleCloudStorage
{
@@ -38,7 +35,7 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
// This class is instantiated dynamically in the BackendLoader.
public class GoogleCloudStorage : IBackend, IStreamingBackend, IRenameEnabledBackend, ITimeoutExemptBackend
{
private const string AUTHID_OPTION = "authid";
private static readonly string TOKEN_URL = OAuthHelper.OAUTH_LOGIN_URL("gcs");
private const string PROJECT_OPTION = "gcs-project";
private const string LOCATION_OPTION = "gcs-location";
@@ -46,17 +43,22 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
private readonly string m_bucket;
private readonly string m_prefix;
private readonly string m_project;
private readonly string? m_project;
private readonly OAuthHelper m_oauth;
private readonly string m_location;
private readonly string m_storage_class;
private readonly string? m_location;
private readonly string? m_storage_class;
private readonly TimeoutOptionsHelper.Timeouts m_timeouts;
public GoogleCloudStorage()
{
m_bucket = null!;
m_prefix = null!;
m_oauth = null!;
m_timeouts = null!;
}
public GoogleCloudStorage(string url, Dictionary<string, string> options)
public GoogleCloudStorage(string url, Dictionary<string, string?> options)
{
var uri = new Utility.Uri(url);
@@ -67,38 +69,37 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
if (m_prefix.StartsWith("/", StringComparison.Ordinal))
m_prefix = m_prefix.Substring(1);
string authid;
options.TryGetValue(AUTHID_OPTION, out authid);
options.TryGetValue(PROJECT_OPTION, out m_project);
options.TryGetValue(LOCATION_OPTION, out m_location);
options.TryGetValue(STORAGECLASS_OPTION, out m_storage_class);
var authId = AuthIdOptionsHelper.Parse(options)
.RequireCredentials(TOKEN_URL);
if (string.IsNullOrEmpty(authid))
throw new UserInformationException(Strings.GoogleCloudStorage.MissingAuthID(AUTHID_OPTION), "GoogleCloudStorageMissingAuthID");
m_timeouts = TimeoutOptionsHelper.Parse(options);
m_project = options.GetValueOrDefault(PROJECT_OPTION);
m_location = options.GetValueOrDefault(LOCATION_OPTION);
m_storage_class = options.GetValueOrDefault(STORAGECLASS_OPTION);
m_oauth = new OAuthHelper(authid, this.ProtocolKey);
m_oauth = new OAuthHelper(authId.AuthId!, this.ProtocolKey);
m_oauth.AutoAuthHeader = true;
}
private class ListBucketResponse
{
public string nextPageToken { get; set; }
public BucketResourceItem[] items { get; set; }
public string? nextPageToken { get; set; }
public BucketResourceItem[]? items { get; set; }
}
private class BucketResourceItem
{
public string name { get; set; }
public string? name { get; set; }
public DateTime? updated { get; set; }
public long? size { get; set; }
}
private class CreateBucketRequest
{
public string name { get; set; }
public string location { get; set; }
public string storageClass { get; set; }
public string? name { get; set; }
public string? location { get; set; }
public string? storageClass { get; set; }
}
private async Task<T> HandleListExceptions<T>(Func<Task<T>> func)
@@ -123,12 +124,15 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
var url = WebApi.GoogleCloudStorage.ListUrl(m_bucket, Utility.Uri.UrlEncode(m_prefix));
while (true)
{
var resp = await HandleListExceptions(() => m_oauth.ReadJSONResponseAsync<ListBucketResponse>(url, cancelToken)).ConfigureAwait(false);
var resp = await HandleListExceptions(() =>
Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, ct =>
m_oauth.ReadJSONResponseAsync<ListBucketResponse>(url, ct))
).ConfigureAwait(false);
if (resp.items != null)
foreach (var f in resp.items)
{
var name = f.name;
var name = f.name ?? "";
if (name.StartsWith(m_prefix, StringComparison.OrdinalIgnoreCase))
name = name.Substring(m_prefix.Length);
if (f.size == null)
@@ -148,33 +152,33 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken)
{
using (System.IO.FileStream fs = System.IO.File.OpenRead(filename))
using (var fs = File.OpenRead(filename))
await PutAsync(remotename, fs, cancelToken).ConfigureAwait(false);
}
public async Task GetAsync(string remotename, string filename, CancellationToken cancelToken)
{
using (System.IO.FileStream fs = System.IO.File.Create(filename))
using (var fs = File.Create(filename))
await GetAsync(remotename, fs, cancelToken).ConfigureAwait(false);
}
public Task DeleteAsync(string remotename, CancellationToken cancelToken)
{
var req = m_oauth.CreateRequest(WebApi.GoogleCloudStorage.DeleteUrl(m_bucket, Library.Utility.Uri.UrlPathEncode(m_prefix + remotename)));
req.Method = "DELETE";
return m_oauth.ReadJSONResponseAsync<object>(req, cancelToken);
return Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct
=> m_oauth.ReadJSONResponseAsync<object>(req, ct));
}
public Task TestAsync(CancellationToken cancelToken)
=> this.TestListAsync(cancelToken);
public async Task CreateFolderAsync(CancellationToken cancelToken)
public Task CreateFolderAsync(CancellationToken cancelToken)
{
if (string.IsNullOrEmpty(m_project))
throw new UserInformationException(Strings.GoogleCloudStorage.ProjectIDMissingError(PROJECT_OPTION), "GoogleCloudStorageMissingProjectID");
var data = System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new CreateBucketRequest
var data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new CreateBucketRequest
{
name = m_bucket,
location = m_location,
@@ -186,23 +190,19 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
req.ContentLength = data.Length;
req.ContentType = "application/json; charset=UTF-8";
var areq = new AsyncHttpRequest(req);
return Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, async ct =>
{
var areq = new AsyncHttpRequest(req);
using (var rs = areq.GetRequestStream())
await rs.WriteAsync(data, 0, data.Length, ct);
using (var rs = areq.GetRequestStream())
await rs.WriteAsync(data, 0, data.Length, cancelToken);
await m_oauth.ReadJSONResponseAsync<BucketResourceItem>(areq, cancelToken).ConfigureAwait(false);
await m_oauth.ReadJSONResponseAsync<BucketResourceItem>(areq, ct).ConfigureAwait(false);
});
}
public string DisplayName
{
get { return Strings.GoogleCloudStorage.DisplayName; }
}
public string DisplayName => Strings.GoogleCloudStorage.DisplayName;
public string ProtocolKey
{
get { return "gcs"; }
}
public string ProtocolKey => "gcs";
public IList<ICommandLineArgument> SupportedCommands
{
@@ -211,50 +211,49 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
StringBuilder locations = new StringBuilder();
StringBuilder storageClasses = new StringBuilder();
foreach (KeyValuePair<string, string> s in WebApi.GoogleCloudStorage.KNOWN_GCS_LOCATIONS)
foreach (var s in WebApi.GoogleCloudStorage.KNOWN_GCS_LOCATIONS)
locations.AppendLine(string.Format("{0}: {1}", s.Key, s.Value));
foreach (KeyValuePair<string, string> s in WebApi.GoogleCloudStorage.KNOWN_GCS_STORAGE_CLASSES)
foreach (var s in WebApi.GoogleCloudStorage.KNOWN_GCS_STORAGE_CLASSES)
storageClasses.AppendLine(string.Format("{0}: {1}", s.Key, s.Value));
return new List<ICommandLineArgument>(new ICommandLineArgument[] {
return [
new CommandLineArgument(LOCATION_OPTION, CommandLineArgument.ArgumentType.String, Strings.GoogleCloudStorage.LocationDescriptionShort, Strings.GoogleCloudStorage.LocationDescriptionLong(locations.ToString())),
new CommandLineArgument(STORAGECLASS_OPTION, CommandLineArgument.ArgumentType.String, Strings.GoogleCloudStorage.StorageclassDescriptionShort, Strings.GoogleCloudStorage.StorageclassDescriptionLong(storageClasses.ToString())),
new CommandLineArgument(AUTHID_OPTION, CommandLineArgument.ArgumentType.Password, Strings.GoogleCloudStorage.AuthidShort, Strings.GoogleCloudStorage.AuthidLong(OAuthHelper.OAUTH_LOGIN_URL("gcs"))),
.. AuthIdOptionsHelper.GetOptions(TOKEN_URL),
new CommandLineArgument(PROJECT_OPTION, CommandLineArgument.ArgumentType.String, Strings.GoogleCloudStorage.ProjectDescriptionShort, Strings.GoogleCloudStorage.ProjectDescriptionLong),
});
.. TimeoutOptionsHelper.GetOptions(),
];
}
}
public string Description
{
get { return Strings.GoogleCloudStorage.Description; }
}
public string Description => Strings.GoogleCloudStorage.Description;
public Task<string[]> GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(WebApi.GoogleCloudStorage.Hosts());
#endregion
public async Task PutAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken)
public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken)
{
var item = new BucketResourceItem { name = m_prefix + remotename };
var url = WebApi.GoogleCloudStorage.PutUrl(m_bucket);
var res = await GoogleCommon.ChunkedUploadWithResumeAsync<BucketResourceItem, BucketResourceItem>(m_oauth, item, url, stream, cancelToken).ConfigureAwait(false);
var res = await GoogleCommon.ChunkedUploadWithResumeAsync<BucketResourceItem, BucketResourceItem>(m_oauth, item, url, stream, m_timeouts.ShortTimeout, m_timeouts.ReadWriteTimeout, cancelToken).ConfigureAwait(false);
if (res == null)
throw new Exception("Upload succeeded, but no data was returned");
}
public async Task GetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken)
public async Task GetAsync(string remotename, Stream stream, CancellationToken cancelToken)
{
try
{
var url = WebApi.GoogleCloudStorage.GetUrl(m_bucket, Library.Utility.Uri.UrlPathEncode(m_prefix + remotename));
var url = WebApi.GoogleCloudStorage.GetUrl(m_bucket, Utility.Uri.UrlPathEncode(m_prefix + remotename));
var req = m_oauth.CreateRequest(url);
var areq = new AsyncHttpRequest(req);
using (var resp = areq.GetResponse())
using (var rs = areq.GetResponseStream())
await Library.Utility.Utility.CopyStreamAsync(rs, stream, cancelToken).ConfigureAwait(false);
using (var resp = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => areq.GetResponse()).ConfigureAwait(false))
using (var rs = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => areq.GetResponseStream()).ConfigureAwait(false))
using (var ts = rs.ObserveReadTimeout(m_timeouts.ReadWriteTimeout))
await Utility.Utility.CopyStreamAsync(ts, stream, cancelToken).ConfigureAwait(false);
}
catch (WebException wex)
{
@@ -265,9 +264,9 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
}
}
public async Task RenameAsync(string oldname, string newname, CancellationToken cancelToken)
public Task RenameAsync(string oldname, string newname, CancellationToken cancelToken)
{
var data = System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new BucketResourceItem
var data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new BucketResourceItem
{
name = m_prefix + newname,
}));
@@ -277,11 +276,14 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
req.ContentLength = data.Length;
req.ContentType = "application/json; charset=UTF-8";
var areq = new AsyncHttpRequest(req);
using (var rs = areq.GetRequestStream())
await rs.WriteAsync(data, 0, data.Length, cancelToken).ConfigureAwait(false);
return Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, async ct =>
{
var areq = new AsyncHttpRequest(req);
using (var rs = areq.GetRequestStream())
await rs.WriteAsync(data, 0, data.Length, ct).ConfigureAwait(false);
await m_oauth.ReadJSONResponseAsync<BucketResourceItem>(req, cancelToken).ConfigureAwait(false);
await m_oauth.ReadJSONResponseAsync<BucketResourceItem>(req, ct).ConfigureAwait(false);
});
}
#region IDisposable implementation
@@ -1,30 +1,27 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Utility;
using Newtonsoft.Json;
using System;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
namespace Duplicati.Library.Backend.GoogleServices
{
@@ -42,7 +39,7 @@ namespace Duplicati.Library.Backend.GoogleServices
/// <param name="oauth">The Oauth instance</param>
/// <param name="uploaduri">The resumeable uploaduri</param>
/// <param name="streamlength">The length of the entire stream</param>
private static long QueryUploadRange<T>(OAuthHelper oauth, string uploaduri, long streamlength, out T response)
private static long QueryUploadRange<T>(OAuthHelper oauth, string uploaduri, long streamlength, out T? response)
where T : class
{
response = null;
@@ -51,8 +48,9 @@ namespace Duplicati.Library.Backend.GoogleServices
req.ContentLength = 0;
req.Headers["Content-Range"] = string.Format("bytes */{0}", streamlength);
// TODO: Apply timeout when this is upgraded to HttpClient
var areq = new AsyncHttpRequest(req);
using(var resp = oauth.GetResponseWithoutException(areq))
using (var resp = oauth.GetResponseWithoutException(areq))
{
var code = (int)resp.StatusCode;
@@ -62,7 +60,7 @@ namespace Duplicati.Library.Backend.GoogleServices
response = oauth.ReadJSONResponse<T>(resp);
if (response == null)
throw new Exception(string.Format("Upload succeeded, but no data was returned, status code: {0}", code));
return streamlength;
}
@@ -73,9 +71,9 @@ namespace Duplicati.Library.Backend.GoogleServices
// https://code.google.com/a/google.com/p/apps-api-issues/issues/detail?id=3884
if (resp.Headers["Range"] == null)
return 0;
return 0;
else
return long.Parse(resp.Headers["Range"].Split(new char[] { '-' })[1]) + 1;
return long.Parse(resp.Headers["Range"]!.Split(new char[] { '-' })[1]) + 1;
}
else
throw new WebException(string.Format("Unexpected status code: {0}", code), null, WebExceptionStatus.ServerProtocolViolation, resp);
@@ -90,10 +88,12 @@ namespace Duplicati.Library.Backend.GoogleServices
/// <param name="requestdata">The data to submit as JSON metadata.</param>
/// <param name="url">The URL to register the upload session against.</param>
/// <param name="stream">The stream with content data to upload.</param>
/// <param name="shortTimeout">The short request timeout.</param>
/// <param name="readWriteTimeout">The read write timeout.</param>
/// <param name="method">The HTTP Method.</param>
/// <typeparam name="TRequest">The type of data to upload as metadata.</typeparam>
/// <typeparam name="TResponse">The type of data returned from the upload.</typeparam>
public static async Task<TResponse> ChunkedUploadWithResumeAsync<TRequest, TResponse>(OAuthHelper oauth, TRequest requestdata, string url, System.IO.Stream stream, CancellationToken cancelToken, string method = "POST")
public static async Task<TResponse> ChunkedUploadWithResumeAsync<TRequest, TResponse>(OAuthHelper oauth, TRequest requestdata, string url, Stream stream, TimeSpan shortTimeout, TimeSpan readWriteTimeout, CancellationToken cancelToken, string method = "POST")
where TRequest : class
where TResponse : class
{
@@ -105,25 +105,30 @@ namespace Duplicati.Library.Backend.GoogleServices
if (data != null)
req.ContentType = "application/json; charset=UTF-8";
req.Headers["X-Upload-Content-Type"] = "application/octet-stream";
req.Headers["X-Upload-Content-Length"] = stream.Length.ToString();
var areq = new AsyncHttpRequest(req);
if (data != null)
using(var rs = areq.GetRequestStream())
await rs.WriteAsync(data, 0, data.Length, cancelToken).ConfigureAwait(false);
string uploaduri;
using(var resp = (HttpWebResponse)areq.GetResponse())
var uploaduri = await Utility.Utility.WithTimeout(shortTimeout, cancelToken, async ct =>
{
if (resp.StatusCode != HttpStatusCode.OK || string.IsNullOrWhiteSpace(resp.Headers["Location"]))
throw new WebException("Failed to start upload session", null, WebExceptionStatus.UnknownError, resp);
var areq = new AsyncHttpRequest(req);
if (data != null)
using (var rs = areq.GetRequestStream())
await rs.WriteAsync(data, 0, data.Length, cancelToken).ConfigureAwait(false);
uploaduri = resp.Headers["Location"];
}
using (var resp = (HttpWebResponse)areq.GetResponse())
{
if (resp.StatusCode != HttpStatusCode.OK || string.IsNullOrWhiteSpace(resp.Headers["Location"]))
throw new WebException("Failed to start upload session", null, WebExceptionStatus.UnknownError, resp);
return await ChunkedUploadAsync<TResponse>(oauth, uploaduri, stream, cancelToken).ConfigureAwait(false);
return resp.Headers["Location"];
}
}).ConfigureAwait(false);
if (string.IsNullOrWhiteSpace(uploaduri))
throw new Exception("Failed to start upload session");
return await ChunkedUploadAsync<TResponse>(oauth, uploaduri, stream, shortTimeout, readWriteTimeout, cancelToken).ConfigureAwait(false);
}
/// <summary>
@@ -133,8 +138,10 @@ namespace Duplicati.Library.Backend.GoogleServices
/// <param name="oauth">The Oauth instance</param>
/// <param name="uploaduri">The resumeable uploaduri</param>
/// <param name="stream">The stream with data to upload.</param>
/// <param name="shortTimeout">The short request timeout.</param>
/// <param name="readWriteTimeout">The read write timeout.</param>
/// <typeparam name="T">The type of data in the response.</typeparam>
private static async Task<T> ChunkedUploadAsync<T>(OAuthHelper oauth, string uploaduri, System.IO.Stream stream, CancellationToken cancelToken)
private static async Task<T> ChunkedUploadAsync<T>(OAuthHelper oauth, string uploaduri, Stream stream, TimeSpan shortTimeout, TimeSpan readWriteTimeout, CancellationToken cancelToken)
where T : class
{
var queryRange = false;
@@ -143,14 +150,13 @@ namespace Duplicati.Library.Backend.GoogleServices
var buffer = new byte[Library.Utility.Utility.DEFAULT_BUFFER_SIZE];
// Repeatedly try uploading until all retries are done
while(true)
while (true)
{
try
{
if (queryRange)
{
T re;
offset = GoogleCommon.QueryUploadRange(oauth, uploaduri, stream.Length, out re);
offset = QueryUploadRange<T>(oauth, uploaduri, stream.Length, out var re);
queryRange = false;
if (re != null)
@@ -172,25 +178,26 @@ namespace Duplicati.Library.Backend.GoogleServices
// Upload the remaining data
var areq = new AsyncHttpRequest(req);
using(var rs = areq.GetRequestStream())
using (var rs = areq.GetRequestStream())
using (var ts = rs.ObserveWriteTimeout(readWriteTimeout))
{
var remaining = chunkSize;
while(remaining > 0)
while (remaining > 0)
{
var n = stream.Read(buffer, 0, (int)Math.Min(remaining, Library.Utility.Utility.DEFAULT_BUFFER_SIZE));
await rs.WriteAsync(buffer, 0, n, cancelToken);
await ts.WriteAsync(buffer, 0, n, cancelToken);
remaining -= n;
}
}
// Check the response
using(var resp = oauth.GetResponseWithoutException(areq))
using (var resp = await Utility.Utility.WithTimeout(shortTimeout, cancelToken, _ => oauth.GetResponseWithoutException(areq)).ConfigureAwait(false))
{
var code = (int)resp.StatusCode;
if (code == 308 && resp.Headers["Range"] != null)
{
offset = long.Parse(resp.Headers["Range"].Split(new char[] {'-'})[1]) + 1;
offset = long.Parse(resp.Headers["Range"]!.Split(new char[] { '-' })[1]) + 1;
retries = 0;
}
else if (code >= 200 && code <= 299)
@@ -23,16 +23,11 @@ using Duplicati.Library.Backend.GoogleServices;
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
using Duplicati.Library.Utility.Options;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace Duplicati.Library.Backend.GoogleDrive
{
@@ -40,41 +35,51 @@ namespace Duplicati.Library.Backend.GoogleDrive
// This class is instantiated dynamically in the BackendLoader.
public class GoogleDrive : IBackend, IStreamingBackend, IQuotaEnabledBackend, IRenameEnabledBackend, ITimeoutExemptBackend
{
private const string AUTHID_OPTION = "authid";
private const string TEAMDRIVE_ID = "googledrive-teamdrive-id";
private const string FOLDER_MIMETYPE = "application/vnd.google-apps.folder";
private static readonly string TOKEN_URL = OAuthHelper.OAUTH_LOGIN_URL("googledrive");
private readonly string m_path;
private readonly string m_teamDriveID;
private readonly string? m_teamDriveID;
private readonly OAuthHelper m_oauth;
private readonly Dictionary<string, GoogleDriveFolderItem[]> m_filecache;
private readonly TimeoutOptionsHelper.Timeouts m_timeouts;
private string m_currentFolderId;
private string? m_currentFolderId;
public GoogleDrive()
{
m_path = null!;
m_oauth = null!;
m_filecache = null!;
m_timeouts = null!;
}
public GoogleDrive(string url, Dictionary<string, string> options)
public GoogleDrive(string url, Dictionary<string, string?> options)
{
var uri = new Utility.Uri(url);
m_path = Util.AppendDirSeparator(uri.HostAndPath, "/");
string authid = null;
if (options.ContainsKey(AUTHID_OPTION))
authid = options[AUTHID_OPTION];
var auth = AuthIdOptionsHelper.Parse(options)
.RequireCredentials(TOKEN_URL);
if (options.ContainsKey(TEAMDRIVE_ID))
m_teamDriveID = options[TEAMDRIVE_ID];
m_oauth = new OAuthHelper(authid, this.ProtocolKey) { AutoAuthHeader = true };
m_timeouts = TimeoutOptionsHelper.Parse(options);
m_teamDriveID = options.GetValueOrDefault(TEAMDRIVE_ID);
m_oauth = new OAuthHelper(auth.AuthId!, this.ProtocolKey) { AutoAuthHeader = true };
m_filecache = new Dictionary<string, GoogleDriveFolderItem[]>();
}
private async Task<string> GetFolderIdAsync(string path, bool autocreate, CancellationToken cancelToken)
{
var curparent = m_teamDriveID ?? (await GetAboutInfoAsync(cancelToken).ConfigureAwait(false)).rootFolderId;
var curparent = string.IsNullOrWhiteSpace(m_teamDriveID)
? (await GetAboutInfoAsync(cancelToken).ConfigureAwait(false)).rootFolderId
: m_teamDriveID;
if (string.IsNullOrEmpty(curparent))
throw new UserInformationException("Unable to get root folder", "GoogleDriveNoRootFolder");
var curdisplay = new StringBuilder("/");
foreach (var p in path.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries))
@@ -97,6 +102,10 @@ namespace Duplicati.Library.Backend.GoogleDrive
}
curdisplay.Append(p).Append("/");
if (string.IsNullOrWhiteSpace(curparent))
throw new UserInformationException($"Unable to get folder id for {curdisplay}", "GoogleDriveNoFolderId");
}
return curparent;
@@ -110,11 +119,9 @@ namespace Duplicati.Library.Backend.GoogleDrive
return m_currentFolderId;
}
private async Task<GoogleDriveFolderItem[]> GetFileEntriesAsync(string remotename, bool throwMissingException, CancellationToken cancelToken)
private async Task<GoogleDriveFolderItem[]?> GetFileEntriesAsync(string remotename, bool throwMissingException, CancellationToken cancelToken)
{
GoogleDriveFolderItem[] entries;
m_filecache.TryGetValue(remotename, out entries);
m_filecache.TryGetValue(remotename, out var entries);
if (entries != null)
return entries;
@@ -124,7 +131,7 @@ namespace Duplicati.Library.Backend.GoogleDrive
if (entries == null || entries.Length == 0)
{
if (throwMissingException)
throw new FileMissingException();
throw new FileMissingException(Strings.GoogleDrive.FilenotFound(remotename));
else
return null;
}
@@ -139,7 +146,7 @@ namespace Duplicati.Library.Backend.GoogleDrive
#region IStreamingBackend implementation
public async Task PutAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken)
public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken)
{
try
{
@@ -147,10 +154,9 @@ namespace Duplicati.Library.Backend.GoogleDrive
if (m_filecache.Count == 0)
await foreach (var file in ListAsync(cancelToken).ConfigureAwait(false)) { /* Enumerate the full listing */ }
GoogleDriveFolderItem[] files;
m_filecache.TryGetValue(remotename, out files);
m_filecache.TryGetValue(remotename, out var files);
string fileId = null;
string? fileId = null;
if (files != null)
{
if (files.Length == 1)
@@ -174,7 +180,7 @@ namespace Duplicati.Library.Backend.GoogleDrive
teamDriveId = m_teamDriveID
};
var res = await GoogleCommon.ChunkedUploadWithResumeAsync<GoogleDriveFolderItem, GoogleDriveFolderItem>(m_oauth, item, url, stream, cancelToken, isUpdate ? "PUT" : "POST").ConfigureAwait(false);
var res = await GoogleCommon.ChunkedUploadWithResumeAsync<GoogleDriveFolderItem, GoogleDriveFolderItem>(m_oauth, item, url, stream, m_timeouts.ShortTimeout, m_timeouts.ReadWriteTimeout, cancelToken, isUpdate ? "PUT" : "POST").ConfigureAwait(false);
m_filecache[remotename] = [res];
}
catch
@@ -184,19 +190,22 @@ namespace Duplicati.Library.Backend.GoogleDrive
}
}
public async Task GetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken)
public async Task GetAsync(string remotename, Stream stream, CancellationToken cancelToken)
{
// Prevent repeated download url lookups
if (m_filecache.Count == 0)
await foreach (var file in ListAsync(cancelToken).ConfigureAwait(false)) { /* Enumerate the full listing */ }
var fileId = (await GetFileEntriesAsync(remotename, true, cancelToken).ConfigureAwait(false)).OrderByDescending(x => x.createdDate).First().id;
var fileId = (await GetFileEntriesAsync(remotename, true, cancelToken).ConfigureAwait(false))?.OrderByDescending(x => x.createdDate).FirstOrDefault()?.id;
if (string.IsNullOrEmpty(fileId))
throw new FileMissingException(Strings.GoogleDrive.FilenotFound(remotename));
var req = m_oauth.CreateRequest(WebApi.GoogleDrive.GetUrl(fileId));
var areq = new AsyncHttpRequest(req);
using (var resp = (HttpWebResponse)areq.GetResponse())
using (var rs = areq.GetResponseStream())
await Duplicati.Library.Utility.Utility.CopyStreamAsync(rs, stream, cancelToken);
using (var resp = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => (HttpWebResponse)areq.GetResponse()).ConfigureAwait(false))
using (var rs = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => areq.GetResponseStream()).ConfigureAwait(false))
using (var ts = rs.ObserveReadTimeout(m_timeouts.ReadWriteTimeout))
await Utility.Utility.CopyStreamAsync(ts, stream, cancelToken);
}
#endregion
@@ -216,7 +225,7 @@ namespace Duplicati.Library.Backend.GoogleDrive
var currentFolderId = await GetCurrentFolderIdAsync(cancelToken).ConfigureAwait(false);
await foreach (var n in ListFolder(currentFolderId, null, null, cancelToken).ConfigureAwait(false))
{
FileEntry fe = null;
FileEntry? fe = null;
if (n.fileSize == null)
fe = new FileEntry(n.title);
@@ -231,10 +240,9 @@ namespace Duplicati.Library.Backend.GoogleDrive
if (!fe.IsFolder)
{
GoogleDriveFolderItem[] lst;
if (!m_filecache.TryGetValue(fe.Name, out lst))
if (!m_filecache.TryGetValue(fe.Name, out var lst))
{
m_filecache[fe.Name] = new GoogleDriveFolderItem[] { n };
m_filecache[fe.Name] = [n];
}
else
{
@@ -262,13 +270,13 @@ namespace Duplicati.Library.Backend.GoogleDrive
public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken)
{
using (var fs = System.IO.File.OpenRead(filename))
using (var fs = File.OpenRead(filename))
await PutAsync(remotename, fs, cancelToken).ConfigureAwait(false);
}
public async Task GetAsync(string remotename, string filename, CancellationToken cancelToken)
{
using (var fs = System.IO.File.Create(filename))
using (var fs = File.Create(filename))
await GetAsync(remotename, fs, cancelToken);
}
@@ -276,13 +284,19 @@ namespace Duplicati.Library.Backend.GoogleDrive
{
try
{
foreach (var fileid in from n in await GetFileEntriesAsync(remotename, true, cancelToken).ConfigureAwait(false) select n.id)
var entries = await GetFileEntriesAsync(remotename, true, cancelToken).ConfigureAwait(false);
if (entries == null)
throw new FileMissingException(Strings.GoogleDrive.FilenotFound(remotename));
foreach (var fileid in entries.Select(x => x.id).WhereNotNullOrWhiteSpace())
{
var url = WebApi.GoogleDrive.DeleteUrl(Library.Utility.Uri.UrlPathEncode(fileid), m_teamDriveID);
await m_oauth.GetJSONDataAsync<object>(url, cancelToken, x =>
{
x.Method = "DELETE";
});
await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct
=> m_oauth.GetJSONDataAsync<object>(url, cancelToken, x =>
{
x.Method = "DELETE";
})
).ConfigureAwait(false);
}
m_filecache.Remove(remotename);
@@ -304,32 +318,19 @@ namespace Duplicati.Library.Backend.GoogleDrive
m_currentFolderId = await GetFolderIdAsync(m_path, true, cancelToken).ConfigureAwait(false);
}
public string DisplayName
{
get
{
return Strings.GoogleDrive.DisplayName;
}
}
public string DisplayName => Strings.GoogleDrive.DisplayName;
public string ProtocolKey
{
get
{
return "googledrive";
}
}
public string ProtocolKey => "googledrive";
public System.Collections.Generic.IList<ICommandLineArgument> SupportedCommands => new List<ICommandLineArgument>(new ICommandLineArgument[] {
new CommandLineArgument(AUTHID_OPTION,
CommandLineArgument.ArgumentType.Password,
Strings.GoogleDrive.AuthidShort,
Strings.GoogleDrive.AuthidLong(OAuthHelper.OAUTH_LOGIN_URL("googledrive"))),
new CommandLineArgument(TEAMDRIVE_ID,
CommandLineArgument.ArgumentType.String,
Strings.GoogleDrive.TeamDriveIdShort,
Strings.GoogleDrive.TeamDriveIdLong),
});
public IList<ICommandLineArgument> SupportedCommands =>
[
.. AuthIdOptionsHelper.GetOptions(TOKEN_URL),
new CommandLineArgument(TEAMDRIVE_ID,
CommandLineArgument.ArgumentType.String,
Strings.GoogleDrive.TeamDriveIdShort,
Strings.GoogleDrive.TeamDriveIdLong),
.. TimeoutOptionsHelper.GetOptions(),
];
public string Description
{
@@ -342,7 +343,7 @@ namespace Duplicati.Library.Backend.GoogleDrive
#endregion
#region IQuotaEnabledBackend implementation
public async Task<IQuotaInfo> GetQuotaInfoAsync(CancellationToken cancelToken)
public async Task<IQuotaInfo?> GetQuotaInfoAsync(CancellationToken cancelToken)
{
try
{
@@ -365,9 +366,10 @@ namespace Duplicati.Library.Backend.GoogleDrive
try
{
var files = await GetFileEntriesAsync(oldname, true, cancellationToken).ConfigureAwait(false);
if (files == null)
throw new FileMissingException(Strings.GoogleDrive.FilenotFound(oldname));
if (files.Length > 1)
throw new UserInformationException(string.Format(Strings.GoogleDrive.MultipleEntries(oldname, m_path)),
"GoogleDriveMultipleEntries");
throw new UserInformationException(Strings.GoogleDrive.MultipleEntries(oldname, m_path), "GoogleDriveMultipleEntries");
using var stream = new MemoryStream();
await GetAsync(oldname, stream, cancellationToken).ConfigureAwait(false);
@@ -396,13 +398,13 @@ namespace Duplicati.Library.Backend.GoogleDrive
private class GoogleDriveParentReference
{
public string id { get; set; }
public string? id { get; set; }
}
private class GoogleDriveListResponse
{
public string nextPageToken { get; set; }
public GoogleDriveFolderItem[] items { get; set; }
public string? nextPageToken { get; set; }
public GoogleDriveFolderItem[]? items { get; set; }
}
private class GoogleDriveFolderItemLabels
@@ -412,41 +414,44 @@ namespace Duplicati.Library.Backend.GoogleDrive
private class GoogleDriveFolderItem
{
public string id { get; set; }
public string title { get; set; }
public string description { get; set; }
public string mimeType { get; set; }
public GoogleDriveFolderItemLabels labels { get; set; }
public string? id { get; set; }
public string? title { get; set; }
public string? description { get; set; }
public string? mimeType { get; set; }
public GoogleDriveFolderItemLabels? labels { get; set; }
public DateTime? createdDate { get; set; }
public DateTime? modifiedDate { get; set; }
public long? fileSize { get; set; }
public string teamDriveId { get; set; }
public GoogleDriveParentReference[] parents { get; set; }
public string? teamDriveId { get; set; }
public GoogleDriveParentReference[]? parents { get; set; }
}
private class GoogleDriveAboutResponse
{
public long? quotaBytesTotal { get; set; }
public long? quotaBytesUsed { get; set; }
public string rootFolderId { get; set; }
public string? rootFolderId { get; set; }
}
private async IAsyncEnumerable<GoogleDriveFolderItem> ListFolder(string parentfolder, bool? onlyFolders, string name, [EnumeratorCancellation] CancellationToken cancelToken)
private async IAsyncEnumerable<GoogleDriveFolderItem> ListFolder(string parentfolder, bool? onlyFolders, string? name, [EnumeratorCancellation] CancellationToken cancelToken)
{
var fileQuery = new string[] {
var fileQuery = new string?[] {
string.IsNullOrEmpty(name) ? null : string.Format("title = '{0}'", EscapeTitleEntries(name)),
onlyFolders == null ? null : string.Format("mimeType {0}= '{1}'", onlyFolders.Value ? "" : "!", FOLDER_MIMETYPE),
string.Format("'{0}' in parents", EscapeTitleEntries(parentfolder)),
"trashed=false"
};
var encodedFileQuery = Library.Utility.Uri.UrlEncode(string.Join(" and ", fileQuery.Where(x => x != null)));
var encodedFileQuery = Utility.Uri.UrlEncode(string.Join(" and ", fileQuery.Where(x => x != null)));
var url = WebApi.GoogleDrive.ListUrl(encodedFileQuery, m_teamDriveID);
while (true)
{
var res = await m_oauth.GetJSONDataAsync<GoogleDriveListResponse>(url, cancelToken).ConfigureAwait(false);
foreach (var n in res.items)
var res = await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, ct
=> m_oauth.GetJSONDataAsync<GoogleDriveListResponse>(url, ct)
).ConfigureAwait(false);
foreach (var n in res.items ?? [])
yield return n;
var token = res.nextPageToken;
@@ -458,9 +463,8 @@ namespace Duplicati.Library.Backend.GoogleDrive
}
private Task<GoogleDriveAboutResponse> GetAboutInfoAsync(CancellationToken cancelToken)
{
return m_oauth.GetJSONDataAsync<GoogleDriveAboutResponse>(WebApi.GoogleDrive.AboutInfoUrl(), cancelToken);
}
=> Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct
=> m_oauth.GetJSONDataAsync<GoogleDriveAboutResponse>(WebApi.GoogleDrive.AboutInfoUrl(), ct));
private Task<GoogleDriveFolderItem> CreateFolderAsync(string name, string parent, CancellationToken cancelToken)
{
@@ -470,25 +474,27 @@ namespace Duplicati.Library.Backend.GoogleDrive
description = name,
mimeType = FOLDER_MIMETYPE,
labels = new GoogleDriveFolderItemLabels { hidden = true },
parents = new GoogleDriveParentReference[] { new GoogleDriveParentReference { id = parent } }
parents = [new GoogleDriveParentReference { id = parent }]
};
var data = System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(folder));
var data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(folder));
return m_oauth.GetJSONDataAsync<GoogleDriveFolderItem>(
WebApi.GoogleDrive.CreateFolderUrl(m_teamDriveID),
cancelToken,
x =>
{
x.Method = "POST";
x.ContentType = "application/json; charset=UTF-8";
x.ContentLength = data.Length;
return Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct
=> m_oauth.GetJSONDataAsync<GoogleDriveFolderItem>(
WebApi.GoogleDrive.CreateFolderUrl(m_teamDriveID),
ct,
x =>
{
x.Method = "POST";
x.ContentType = "application/json; charset=UTF-8";
x.ContentLength = data.Length;
}, async (req, ct) =>
{
using (var rs = req.GetRequestStream())
await rs.WriteAsync(data, 0, data.Length, ct).ConfigureAwait(false);
});
}, async (req, ct) =>
{
using (var rs = req.GetRequestStream())
await rs.WriteAsync(data, 0, data.Length, ct).ConfigureAwait(false);
})
);
}
}
}
@@ -1,35 +1,33 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Localization.Short;
namespace Duplicati.Library.Backend.Strings
{
internal static class GoogleCloudStorage {
internal static class GoogleCloudStorage
{
public static string Description { get { return LC.L(@"This backend can read and write data to Google Cloud Storage. Allowed format is ""gcs://bucket/folder""."); } }
public static string DisplayName { get { return LC.L(@"Google Cloud Storage"); } }
public static string MissingAuthID(string url) { return LC.L(@"You need an AuthID. You can get it from: {0}", url); }
public static string ProjectIDMissingError(string projectoption) { return LC.L(@"You must supply a project ID with --{0} for creating a bucket.", projectoption); }
public static string AuthidLong(string url) { return LC.L(@"The authorization token retrieved from {0}", url); }
public static string AuthidShort { get { return LC.L(@"The authorization code"); } }
public static string LocationDescriptionLong(string regions) { return LC.L(@"This option is only used when creating new buckets. Use this option to change what region the data is stored in. Charges vary with bucket location. Known bucket locations:
{0}", regions); }
public static string LocationDescriptionShort { get { return LC.L(@"Specify location option for creating a bucket"); } }
@@ -40,13 +38,21 @@ namespace Duplicati.Library.Backend.Strings
public static string ProjectDescriptionShort { get { return LC.L(@"Specify project for creating a bucket"); } }
}
internal static class GoogleDrive {
internal static class GoogleDrive
{
public static string Description { get { return LC.L(@"This backend can read and write data to Google Drive. Allowed format is ""googledrive://folder/subfolder""."); } }
public static string DisplayName { get { return LC.L(@"Google Drive"); } }
public static string MultipleEntries(string folder, string parent) { return LC.L(@"There is more than one item named ""{0}"" in the folder ""{1}"".", folder, parent); }
public static string AuthidLong(string url) { return LC.L(@"The authorization token retrieved from {0}", url); }
public static string AuthidShort { get { return LC.L(@"The authorization code"); } }
public static string FilenotFound(string name) { return LC.L(@"File not found: {0}", name); }
public static string TeamDriveIdLong { get { return LC.L("This option sets the team drive to use. Leaving it empty uses the personal drive."); } }
public static string TeamDriveIdShort { get { return LC.L("Team drive ID"); } }
}
internal static class GCSConfig
{
public static string DisplayName => LC.L("Google Cloud Storage configuration module");
public static string Description => LC.L("Expose Google Cloud Storage configuration as a web module");
public static string ConfigTypeShort => LC.L("The config to get");
public static string ConfigTypeLong => LC.L("Provide different config values");
}
}
@@ -1,25 +1,24 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System.Collections.Generic;
using System.Collections.Specialized;
using Duplicati.Library.Utility;
@@ -28,41 +27,41 @@ namespace Duplicati.Library.Backend.WebApi
public static class GoogleCloudStorage
{
// From: https://cloud.google.com/storage/docs/bucket-locations
public static readonly KeyValuePair<string, string>[] KNOWN_GCS_LOCATIONS = {
new KeyValuePair<string, string>("(default)", null),
new KeyValuePair<string, string>("Europe", "EU"),
new KeyValuePair<string, string>("United States", "US"),
new KeyValuePair<string, string>("Asia", "ASIA"),
public static readonly KeyValuePair<string, string?>[] KNOWN_GCS_LOCATIONS = {
new KeyValuePair<string, string?>("(default)", null),
new KeyValuePair<string, string?>("Europe", "EU"),
new KeyValuePair<string, string?>("United States", "US"),
new KeyValuePair<string, string?>("Asia", "ASIA"),
//Regional buckets: https://cloud.google.com/storage/docs/regional-buckets
new KeyValuePair<string, string>("Eastern Asia-Pacific", "ASIA-EAST1"),
new KeyValuePair<string, string>("Central United States 1", "US-CENTRAL1"),
new KeyValuePair<string, string>("Central United States 2", "US-CENTRAL2"),
new KeyValuePair<string, string>("Eastern United States 1", "US-EAST1"),
new KeyValuePair<string, string>("Eastern United States 2", "US-EAST2"),
new KeyValuePair<string, string>("Eastern United States 3", "US-EAST3"),
new KeyValuePair<string, string>("Western United States", "US-WEST1"),
new KeyValuePair<string, string?>("Eastern Asia-Pacific", "ASIA-EAST1"),
new KeyValuePair<string, string?>("Central United States 1", "US-CENTRAL1"),
new KeyValuePair<string, string?>("Central United States 2", "US-CENTRAL2"),
new KeyValuePair<string, string?>("Eastern United States 1", "US-EAST1"),
new KeyValuePair<string, string?>("Eastern United States 2", "US-EAST2"),
new KeyValuePair<string, string?>("Eastern United States 3", "US-EAST3"),
new KeyValuePair<string, string?>("Western United States", "US-WEST1"),
};
public static readonly KeyValuePair<string, string>[] KNOWN_GCS_STORAGE_CLASSES = {
new KeyValuePair<string, string>("(default)", null),
new KeyValuePair<string, string>("Standard", "STANDARD"),
new KeyValuePair<string, string>("Durable Reduced Availability (DRA)", "DURABLE_REDUCED_AVAILABILITY"),
new KeyValuePair<string, string>("Nearline", "NEARLINE"),
public static readonly KeyValuePair<string, string?>[] KNOWN_GCS_STORAGE_CLASSES = {
new KeyValuePair<string, string?>("(default)", null),
new KeyValuePair<string, string?>("Standard", "STANDARD"),
new KeyValuePair<string, string?>("Durable Reduced Availability (DRA)", "DURABLE_REDUCED_AVAILABILITY"),
new KeyValuePair<string, string?>("Nearline", "NEARLINE"),
};
public static string[] Hosts()
{
return new [] { new System.Uri(Url.UPLOAD).Host,
new System.Uri(Url.API).Host };
}
public static string[] Hosts() =>
[
new System.Uri(Url.UPLOAD).Host,
new System.Uri(Url.API).Host
];
public static string DeleteUrl(string bucketId, string objectId)
{
var path = BucketObjectPath(bucketId, objectId);
return Uri.UriBuilder(Url.API, path);
return Utility.Uri.UriBuilder(Url.API, path);
}
public static string CreateFolderUrl(string projectId)
@@ -72,20 +71,16 @@ namespace Duplicati.Library.Backend.WebApi
{ QueryParam.Project, projectId }
};
return Uri.UriBuilder(Url.API, Path.Bucket, queryParams);
return Utility.Uri.UriBuilder(Url.API, Path.Bucket, queryParams);
}
public static string RenameUrl(string bucketId, string objectId)
{
return Uri.UriBuilder(Url.API, BucketObjectPath(bucketId, objectId));
}
=> Utility.Uri.UriBuilder(Url.API, BucketObjectPath(bucketId, objectId));
public static string ListUrl(string bucketId, string prefix)
{
return ListUrl(bucketId, prefix, null);
}
=> ListUrl(bucketId, prefix, null);
public static string ListUrl(string bucketId, string prefix, string token)
public static string ListUrl(string bucketId, string prefix, string? token)
{
var queryParams = new NameValueCollection {
{ QueryParam.Prefix, prefix }
@@ -96,7 +91,7 @@ namespace Duplicati.Library.Backend.WebApi
queryParams.Set(QueryParam.PageToken, token);
}
return Uri.UriBuilder(Url.API, BucketObjectPath(bucketId), queryParams);
return Utility.Uri.UriBuilder(Url.API, BucketObjectPath(bucketId), queryParams);
}
public static string PutUrl(string bucketId)
@@ -106,7 +101,7 @@ namespace Duplicati.Library.Backend.WebApi
{ QueryParam.UploadType, QueryValue.Resumable }
};
var path = UrlPath.Create(Path.Bucket).Append(bucketId).Append(Path.Object).ToString();
return Uri.UriBuilder(Url.UPLOAD, path, queryParams);
return Utility.Uri.UriBuilder(Url.UPLOAD, path, queryParams);
}
public static string GetUrl(string bucketId, string objectId)
@@ -118,7 +113,7 @@ namespace Duplicati.Library.Backend.WebApi
};
var path = BucketObjectPath(bucketId, objectId);
return Uri.UriBuilder(Url.API, path, queryParams);
return Utility.Uri.UriBuilder(Url.API, path, queryParams);
}
private static class Url
@@ -148,36 +143,27 @@ namespace Duplicati.Library.Backend.WebApi
public const string Object = "o";
}
private static string BucketObjectPath(string bucketId, string objectId = null)
{
return UrlPath.Create(Path.Bucket)
.Append(bucketId)
.Append(Path.Object)
.Append(objectId).ToString();
}
private static string BucketObjectPath(string bucketId, string? objectId = null)
=> UrlPath.Create(Path.Bucket)
.Append(bucketId)
.Append(Path.Object)
.Append(objectId).ToString();
}
public static class GoogleDrive
{
public static string[] Hosts()
{
return new [] { new System.Uri(Url.DRIVE).Host, new System.Uri(Url.UPLOAD).Host };
}
=> [new System.Uri(Url.DRIVE).Host, new System.Uri(Url.UPLOAD).Host];
public static string GetUrl(string fileId)
{
return FileQueryUrl(fileId, new NameValueCollection{
=> FileQueryUrl(fileId, new NameValueCollection{
{ QueryParam.Alt, QueryValue.Media }
});
}
public static string DeleteUrl(string fileId, string teamDriveId)
{
return FileQueryUrl(Uri.UrlPathEncode(fileId), AddTeamDriveParam(teamDriveId));
}
public static string DeleteUrl(string fileId, string? teamDriveId)
=> FileQueryUrl(Utility.Uri.UrlPathEncode(fileId), AddTeamDriveParam(teamDriveId));
public static string PutUrl(string fileId, bool useTeamDrive)
public static string PutUrl(string? fileId, bool useTeamDrive)
{
var queryParams = new NameValueCollection {
{ QueryParam.UploadType,
@@ -189,16 +175,14 @@ namespace Duplicati.Library.Backend.WebApi
}
return !string.IsNullOrWhiteSpace(fileId) ?
FileUploadUrl(Uri.UrlPathEncode(fileId), queryParams) :
FileUploadUrl(Utility.Uri.UrlPathEncode(fileId), queryParams) :
FileUploadUrl(queryParams);
}
public static string ListUrl(string fileQuery, string teamDriveId)
{
return ListUrl(fileQuery, teamDriveId, null);
}
public static string ListUrl(string fileQuery, string teamDriveId, string token)
public static string ListUrl(string fileQuery, string? teamDriveId)
=> ListUrl(fileQuery, teamDriveId, null);
public static string ListUrl(string fileQuery, string? teamDriveId, string? token)
{
var queryParams = new NameValueCollection
{
@@ -207,7 +191,7 @@ namespace Duplicati.Library.Backend.WebApi
};
queryParams.Add(AddTeamDriveParam(teamDriveId));
if (token != null)
{
queryParams.Set(QueryParam.PageToken, token);
@@ -216,15 +200,11 @@ namespace Duplicati.Library.Backend.WebApi
return FileQueryUrl(queryParams);
}
public static string CreateFolderUrl(string teamDriveId)
{
return FileQueryUrl(AddTeamDriveParam(teamDriveId));
}
public static string CreateFolderUrl(string? teamDriveId)
=> FileQueryUrl(AddTeamDriveParam(teamDriveId));
public static string AboutInfoUrl()
{
return Uri.UriBuilder(Url.DRIVE, Path.About);
}
=> Utility.Uri.UriBuilder(Url.DRIVE, Path.About);
private static class Url
{
@@ -259,40 +239,30 @@ namespace Duplicati.Library.Backend.WebApi
}
private static string FileQueryUrl(NameValueCollection values)
{
return Uri.UriBuilder(Url.DRIVE, Path.File, values);
}
=> Utility.Uri.UriBuilder(Url.DRIVE, Path.File, values);
private static string FileQueryUrl(string fileId, NameValueCollection values = null)
{
return Uri.UriBuilder(Url.DRIVE, UrlPath.Create(Path.File).Append(fileId).ToString(),
values);
}
private static string FileQueryUrl(string fileId, NameValueCollection? values = null)
=> Utility.Uri.UriBuilder(Url.DRIVE, UrlPath.Create(Path.File).Append(fileId).ToString(), values);
private static string FileUploadUrl(string fileId, NameValueCollection values)
{
return Uri.UriBuilder(Url.UPLOAD, UrlPath.Create(Path.File).Append(fileId).ToString(),
values);
}
private static string FileUploadUrl(string fileId, NameValueCollection? values)
=> Utility.Uri.UriBuilder(Url.UPLOAD, UrlPath.Create(Path.File).Append(fileId).ToString(), values);
private static string FileUploadUrl(NameValueCollection values)
{
return Uri.UriBuilder(Url.UPLOAD, Path.File, values);
}
=> Utility.Uri.UriBuilder(Url.UPLOAD, Path.File, values);
private static NameValueCollection AddTeamDriveParam(string teamDriveId)
private static NameValueCollection AddTeamDriveParam(string? teamDriveId)
{
return teamDriveId != null ? new NameValueCollection {
{ WebApi.GoogleDrive.QueryParam.SupportsTeamDrive,
WebApi.GoogleDrive.QueryValue.True },
{ WebApi.GoogleDrive.QueryParam.TeamDriveId,
{ QueryParam.SupportsTeamDrive,
QueryValue.True },
{ QueryParam.TeamDriveId,
teamDriveId },
{ WebApi.GoogleDrive.QueryParam.IncludeTeamDrive,
WebApi.GoogleDrive.QueryValue.True },
{ WebApi.GoogleDrive.QueryParam.corpora,
WebApi.GoogleDrive.QueryValue.TeamDrive }
{ QueryParam.IncludeTeamDrive,
QueryValue.True },
{ QueryParam.corpora,
QueryValue.TeamDrive }
} : new NameValueCollection( );
} : new NameValueCollection();
}
}
}
@@ -5,6 +5,8 @@
<OutputType>Library</OutputType>
<RootNamespace>Duplicati.Library.Backend</RootNamespace>
<Copyright>Copyright © 2025 Team Duplicati, MIT license</Copyright>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>
<ItemGroup>
@@ -21,84 +21,71 @@
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Duplicati.Library.Utility;
using Duplicati.Library.Utility.Options;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
namespace Duplicati.Library.Backend
{
public class Idrivee2Backend : IBackend, IStreamingBackend, IFolderEnabledBackend
{
private static readonly string LOGTAG = Logging.Log.LogTagFromType<Idrivee2Backend>();
static Idrivee2Backend()
{
}
// TODO: Non-standard naming, should be access-key-id and access-key-secret
private const string AUTH_USERNAME_OPTION = "access_key_id";
private const string AUTH_PASSWORD_OPTION = "access_key_secret";
private readonly string m_prefix;
private readonly string m_bucket;
private IS3Client m_s3Client;
private IS3Client? m_s3Client;
private readonly TimeoutOptionsHelper.Timeouts m_timeouts;
private readonly AuthOptionsHelper.AuthOptions m_auth;
private readonly Dictionary<string, string?> m_options;
public Idrivee2Backend()
{
m_bucket = null!;
m_prefix = null!;
m_timeouts = null!;
m_auth = null!;
m_options = null!;
}
public Idrivee2Backend(string url, Dictionary<string, string> options)
public Idrivee2Backend(string url, Dictionary<string, string?> options)
{
var uri = new Utility.Uri(url);
m_bucket = uri.Host;
m_prefix = uri.Path;
m_prefix = m_prefix.Trim();
if (m_prefix.Length != 0)
{
m_prefix = Util.AppendDirSeparator(m_prefix, "/");
}
string accessKeyId = null;
string accessKeySecret = null;
if (options.ContainsKey("auth-username"))
accessKeyId = options["auth-username"];
if (options.ContainsKey("auth-password"))
accessKeySecret = options["auth-password"];
if (options.ContainsKey("access_key_id"))
accessKeyId = options["access_key_id"];
if (options.ContainsKey("secret_access_key"))
accessKeySecret = options["secret_access_key"];
if (string.IsNullOrEmpty(accessKeyId))
m_timeouts = TimeoutOptionsHelper.Parse(options);
m_auth = AuthOptionsHelper.ParseWithAlias(options, uri, AUTH_USERNAME_OPTION, AUTH_PASSWORD_OPTION);
if (!m_auth.HasUsername)
throw new UserInformationException(Strings.Idrivee2Backend.NoKeyIdError, "Idrivee2NoKeyId");
if (string.IsNullOrEmpty(accessKeySecret))
if (!m_auth.HasPassword)
throw new UserInformationException(Strings.Idrivee2Backend.NoKeySecretError, "Idrivee2NoKeySecret");
string host = GetRegionEndpoint("https://api.idrivee2.com/api/service/get_region_end_point/" + accessKeyId);
m_s3Client = new S3AwsClient(accessKeyId, accessKeySecret, null, host, null, true, false, options);
m_options = options;
}
public string GetRegionEndpoint(string url)
public async Task<string> GetRegionEndpointAsync(string url, CancellationToken cancellationToken)
{
try
{
System.Net.HttpWebRequest req = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(url);
var req = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(url);
req.Method = System.Net.WebRequestMethods.Http.Get;
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
var areq = new Utility.AsyncHttpRequest(req);
using (System.Net.HttpWebResponse resp = (System.Net.HttpWebResponse)areq.GetResponse())
using (var resp = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancellationToken, _ => (System.Net.HttpWebResponse)areq.GetResponse()).ConfigureAwait(false))
{
int code = (int)resp.StatusCode;
if (code < 200 || code >= 300) //For some reason Mono does not throw this automatically
throw new Exception("Failed to fetch region endpoint");
using (var s = areq.GetResponseStream())
using (var s = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancellationToken, _ => areq.GetResponseStream()))
using (var t = s.ObserveReadTimeout(m_timeouts.ReadWriteTimeout))
{
using (var reader = new StreamReader(s))
using (var reader = new StreamReader(t))
{
string endpoint = reader.ReadToEnd();
return endpoint;
@@ -115,18 +102,14 @@ namespace Duplicati.Library.Backend
#region IBackend Members
public string DisplayName
{
get { return Strings.Idrivee2Backend.DisplayName; }
}
public string DisplayName => Strings.Idrivee2Backend.DisplayName;
public string ProtocolKey => "e2";
public bool SupportsStreaming => true;
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancelToken)
{
await foreach (IFileEntry file in Connection.ListBucketAsync(m_bucket, m_prefix, false, cancelToken).ConfigureAwait(false))
var con = await GetConnection(cancelToken).ConfigureAwait(false);
await foreach (IFileEntry file in con.ListBucketAsync(m_bucket, m_prefix, false, cancelToken).ConfigureAwait(false))
yield return file;
}
@@ -138,7 +121,8 @@ namespace Duplicati.Library.Backend
public async Task PutAsync(string remotename, Stream input, CancellationToken cancelToken)
{
await Connection.AddFileStreamAsync(m_bucket, GetFullKey(remotename), input, cancelToken).ConfigureAwait(false);
var con = await GetConnection(cancelToken).ConfigureAwait(false);
await con.AddFileStreamAsync(m_bucket, GetFullKey(remotename), input, cancelToken).ConfigureAwait(false);
}
public async Task GetAsync(string remotename, string localname, CancellationToken cancellationToken)
@@ -147,55 +131,46 @@ namespace Duplicati.Library.Backend
await GetAsync(remotename, fs, cancellationToken).ConfigureAwait(false);
}
public Task GetAsync(string remotename, Stream output, CancellationToken cancellationToken)
public async Task GetAsync(string remotename, Stream output, CancellationToken cancellationToken)
{
return Connection.GetFileStreamAsync(m_bucket, GetFullKey(remotename), output, cancellationToken);
var con = await GetConnection(cancellationToken).ConfigureAwait(false);
await con.GetFileStreamAsync(m_bucket, GetFullKey(remotename), output, cancellationToken).ConfigureAwait(false);
}
public Task DeleteAsync(string remotename, CancellationToken cancellationToken)
public async Task DeleteAsync(string remotename, CancellationToken cancellationToken)
{
return Connection.DeleteObjectAsync(m_bucket, GetFullKey(remotename), cancellationToken);
var con = await GetConnection(cancellationToken).ConfigureAwait(false);
await con.DeleteObjectAsync(m_bucket, GetFullKey(remotename), cancellationToken).ConfigureAwait(false);
}
public IList<ICommandLineArgument> SupportedCommands
{
get
{
var exts = S3AwsClient.GetAwsExtendedOptions();
public IList<ICommandLineArgument> SupportedCommands =>
[
.. S3AwsClient.GetAwsExtendedOptions(),
new CommandLineArgument(AUTH_USERNAME_OPTION, CommandLineArgument.ArgumentType.String, Strings.Idrivee2Backend.KeyIDDescriptionShort, Strings.Idrivee2Backend.KeyIDDescriptionLong,null, [AuthOptionsHelper.AuthUsernameOption], null),
new CommandLineArgument(AUTH_PASSWORD_OPTION, CommandLineArgument.ArgumentType.Password, Strings.Idrivee2Backend.KeySecretDescriptionShort, Strings.Idrivee2Backend.KeySecretDescriptionLong, null, [AuthOptionsHelper.AuthPasswordOption ], null),
.. TimeoutOptionsHelper.GetOptions(),
];
var normal = new ICommandLineArgument[] {
new CommandLineArgument("access_key_secret", CommandLineArgument.ArgumentType.Password, Strings.Idrivee2Backend.KeySecretDescriptionShort, Strings.Idrivee2Backend.KeySecretDescriptionLong, null, new[]{"auth-password"}, null),
new CommandLineArgument("access_key_id", CommandLineArgument.ArgumentType.String, Strings.Idrivee2Backend.KeyIDDescriptionShort, Strings.Idrivee2Backend.KeyIDDescriptionLong,null, new[]{"auth-username"}, null)
};
return normal.Union(exts).ToList();
}
}
public string Description
{
get
{
return Strings.Idrivee2Backend.Description;
}
}
public string Description => Strings.Idrivee2Backend.Description;
public Task TestAsync(CancellationToken cancelToken)
=> this.TestListAsync(cancelToken);
public Task CreateFolderAsync(CancellationToken cancelToken)
public async Task CreateFolderAsync(CancellationToken cancelToken)
{
var con = await GetConnection(cancelToken).ConfigureAwait(false);
//S3 does not complain if the bucket already exists
return Connection.AddBucketAsync(m_bucket, cancelToken);
await con.AddBucketAsync(m_bucket, cancelToken).ConfigureAwait(false);
}
#endregion
#region IRenameEnabledBackend Members
public Task Rename(string source, string target, CancellationToken cancelToken)
public async Task Rename(string source, string target, CancellationToken cancelToken)
{
return Connection.RenameFileAsync(m_bucket, GetFullKey(source), GetFullKey(target), cancelToken);
var con = await GetConnection(cancelToken).ConfigureAwait(false);
await con.RenameFileAsync(m_bucket, GetFullKey(source), GetFullKey(target), cancelToken);
}
#endregion
@@ -210,29 +185,46 @@ namespace Duplicati.Library.Backend
#endregion
private IS3Client Connection => m_s3Client;
public Task<string[]> GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new[] { m_s3Client.GetDnsHost() });
private string GetFullKey(string name)
private async Task<IS3Client> GetConnection(CancellationToken cancellationToken)
{
//AWS SDK encodes the filenames correctly
return m_prefix + name;
if (m_s3Client == null)
{
(var accessKeyId, var accessKeySecret) = m_auth.GetCredentials();
// TODO: Do not make blocking calls in the constructor
var host = await GetRegionEndpointAsync("https://api.idrivee2.com/api/service/get_region_end_point/" + accessKeyId, cancellationToken).ConfigureAwait(false);
m_s3Client = new S3AwsClient(accessKeyId, accessKeySecret, null, host, null, true, false, m_timeouts, m_options);
}
return m_s3Client;
}
public async Task<string[]> GetDNSNamesAsync(CancellationToken cancelToken)
{
var con = await GetConnection(cancelToken).ConfigureAwait(false);
var dnshost = con.GetDnsHost();
return string.IsNullOrWhiteSpace(dnshost)
? []
: [dnshost];
}
private string GetFullKey(string? name)
//AWS SDK encodes the filenames correctly
=> $"{m_prefix}{name}";
/// <inheritdoc/>
public IAsyncEnumerable<IFileEntry> ListAsync(string path, CancellationToken cancellationToken)
public async IAsyncEnumerable<IFileEntry> ListAsync(string? path, [EnumeratorCancellation] CancellationToken cancellationToken)
{
var filterPath = GetFullKey(path);
if (!string.IsNullOrWhiteSpace(filterPath))
filterPath = Util.AppendDirSeparator(filterPath, "/");
return m_s3Client.ListBucketAsync(m_bucket, filterPath, true, cancellationToken);
var con = await GetConnection(cancellationToken).ConfigureAwait(false);
await foreach (var files in con.ListBucketAsync(m_bucket, filterPath, true, cancellationToken).ConfigureAwait(false))
yield return files;
}
/// <inheritdoc/>
public Task<IFileEntry> GetEntryAsync(string path, CancellationToken cancellationToken)
=> Task.FromResult<IFileEntry>(null);
public Task<IFileEntry?> GetEntryAsync(string path, CancellationToken cancellationToken)
=> Task.FromResult<IFileEntry?>(null);
}
}
@@ -5,6 +5,8 @@
<OutputType>Library</OutputType>
<RootNamespace>Duplicati.Library.Backend</RootNamespace>
<Copyright>Copyright © 2025 Team Duplicati, MIT license</Copyright>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>
<ItemGroup>
+171 -141
View File
@@ -22,12 +22,11 @@
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.Localization.Short;
using System;
using System.Collections.Generic;
using Duplicati.Library.Utility;
using Duplicati.Library.Utility.Options;
using System.Net;
using System.Runtime.CompilerServices;
using System.Security.Cryptography;
using System.Threading;
using System.Threading.Tasks;
namespace Duplicati.Library.Backend
{
@@ -35,7 +34,7 @@ namespace Duplicati.Library.Backend
// This class is instantiated dynamically in the BackendLoader.
public class Jottacloud : IBackend, IStreamingBackend
{
private const string AUTHID_OPTION = "authid";
private static readonly string TOKEN_URL = OAuthHelper.OAUTH_LOGIN_URL("jottacloud");
private const string JFS_ROOT = "https://jfs.jottacloud.com/jfs";
private const string JFS_ROOT_UPLOAD = "https://up.jottacloud.com/jfs"; // Separate host for uploading files
private const string JFS_BUILTIN_DEVICE = "Jotta"; // The built-in device used for the built-in Sync and Archive mount points.
@@ -60,6 +59,7 @@ namespace Duplicati.Library.Backend
private static readonly string JFS_DEFAULT_THREADS = "4";
private readonly int m_threads;
private readonly long m_chunksize;
private readonly TimeoutOptionsHelper.Timeouts m_timeouts;
private readonly JottacloudAuthHelper m_oauth;
@@ -70,23 +70,33 @@ namespace Duplicati.Library.Backend
/// </summary>
static Jottacloud()
{
System.Net.ServicePointManager.DefaultConnectionLimit = 1000;
// TODO: Remove this when HttpClient migration is done
ServicePointManager.DefaultConnectionLimit = 1000;
}
public Jottacloud()
{
m_device = null!;
m_mountPoint = null!;
m_path = null!;
m_url_device = null!;
m_url = null!;
m_url_upload = null!;
m_timeouts = null!;
m_oauth = null!;
}
public Jottacloud(string url, Dictionary<string, string> options)
public Jottacloud(string url, Dictionary<string, string?> options)
{
// Duplicati back-end url for Jottacloud is in format "jottacloud://folder/subfolder", we transform them to
// the Jottacloud REST API (JFS) url format "https://jfs.jottacloud.com/jfs/[username]/[device]/[mountpoint]/[folder]/[subfolder]".
// Find out what JFS device to use.
if (options.ContainsKey(JFS_DEVICE_OPTION))
var device = options.GetValueOrDefault(JFS_DEVICE_OPTION);
if (!string.IsNullOrWhiteSpace(device))
{
// Custom device specified.
m_device = options[JFS_DEVICE_OPTION];
m_device = device;
if (string.Equals(m_device, JFS_BUILTIN_DEVICE, StringComparison.OrdinalIgnoreCase))
{
m_device_builtin = true; // Device is configured, but value set to the built-in device!
@@ -105,10 +115,11 @@ namespace Duplicati.Library.Backend
}
// Find out what JFS mount point to use on the device.
if (options.ContainsKey(JFS_MOUNT_POINT_OPTION))
var mountPoint = options.GetValueOrDefault(JFS_MOUNT_POINT_OPTION);
if (!string.IsNullOrWhiteSpace(mountPoint))
{
// Custom mount point specified.
m_mountPoint = options[JFS_MOUNT_POINT_OPTION];
m_mountPoint = mountPoint;
// If we are using the built-in device make sure we have picked a mount point that we can use.
if (m_device_builtin)
@@ -136,10 +147,10 @@ namespace Duplicati.Library.Backend
m_mountPoint = JFS_DEFAULT_CUSTOM_MOUNT_POINT; // Set a suitable default mount point for custom (backup) devices.
}
string authid = null;
if (options.ContainsKey(AUTHID_OPTION))
authid = options[AUTHID_OPTION];
m_oauth = new JottacloudAuthHelper(authid);
var authId = AuthIdOptionsHelper.Parse(options)
.RequireCredentials(TOKEN_URL);
m_oauth = new JottacloudAuthHelper(authId.AuthId!);
// Build URL
var u = new Utility.Uri(url);
@@ -152,36 +163,32 @@ namespace Duplicati.Library.Backend
m_url = m_url_device + "/" + m_mountPoint + "/" + m_path;
m_url_upload = JFS_ROOT_UPLOAD + "/" + m_oauth.Username + "/" + m_device + "/" + m_mountPoint + "/" + m_path; // Different hostname, else identical to m_url.
m_threads = int.Parse(options.ContainsKey(JFS_THREADS) ? options[JFS_THREADS] : JFS_DEFAULT_THREADS);
var jfsThreads = options.GetValueOrDefault(JFS_THREADS);
if (string.IsNullOrWhiteSpace(jfsThreads))
jfsThreads = JFS_DEFAULT_THREADS;
if (!options.TryGetValue(JFS_CHUNKSIZE, out var tmp))
{
tmp = JFS_DEFAULT_CHUNKSIZE;
}
m_threads = int.Parse(jfsThreads);
var chunksize = Utility.Sizeparser.ParseSize(tmp, "mb");
var jfsChunksize = options.GetValueOrDefault(JFS_CHUNKSIZE);
if (string.IsNullOrWhiteSpace(jfsChunksize))
jfsChunksize = JFS_DEFAULT_CHUNKSIZE;
var chunksize = Sizeparser.ParseSize(jfsChunksize, "mb");
// Chunk size is bound by BinaryReader.ReadBytes(length) where length is an int.
if (chunksize > int.MaxValue || chunksize < 1024)
{
throw new ArgumentOutOfRangeException(nameof(chunksize), string.Format("The chunk size cannot be less than {0}, nor larger than {1}", Utility.Utility.FormatSizeString(1024), Utility.Utility.FormatSizeString(int.MaxValue)));
}
throw new ArgumentOutOfRangeException(nameof(chunksize), $"The chunk size cannot be less than {Utility.Utility.FormatSizeString(1024)}, nor larger than {Utility.Utility.FormatSizeString(int.MaxValue)}");
m_chunksize = chunksize;
m_timeouts = TimeoutOptionsHelper.Parse(options);
}
#region IBackend Members
public string DisplayName
{
get { return Strings.Jottacloud.DisplayName; }
}
public string DisplayName => Strings.Jottacloud.DisplayName;
public string ProtocolKey
{
get { return "jottacloud"; }
}
public string ProtocolKey => "jottacloud";
/// <inheritdoc />
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancelToken)
@@ -193,14 +200,17 @@ namespace Duplicati.Library.Backend
try
{
// Send request and load XML response.
var req = CreateRequest(System.Net.WebRequestMethods.Http.Get, "", "", false);
var areq = new Utility.AsyncHttpRequest(req);
using (var rs = areq.GetResponseStream())
doc.Load(rs);
var req = CreateRequest(WebRequestMethods.Http.Get, "", "", false);
var areq = new AsyncHttpRequest(req);
await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, _ =>
{
using (var rs = areq.GetResponseStream())
doc.Load(rs);
}).ConfigureAwait(false);
}
catch (System.Net.WebException wex)
catch (WebException wex)
{
if (wex.Response is HttpWebResponse response && response.StatusCode == System.Net.HttpStatusCode.NotFound)
if (wex.Response is HttpWebResponse response && response.StatusCode == HttpStatusCode.NotFound)
throw new FolderMissingException(wex);
throw;
}
@@ -209,50 +219,61 @@ namespace Duplicati.Library.Backend
// We must check for "deleted" attribute, because files/folders which has it is deleted (attribute contains the timestamp of deletion)
// so we treat them as non-existent here.
var xRoot = doc.DocumentElement;
if (xRoot.Attributes["deleted"] != null)
{
if (xRoot == null)
throw new FolderMissingException();
}
foreach (System.Xml.XmlNode xFolder in xRoot.SelectNodes("folders/folder[not(@deleted)]"))
{
// Subfolders are only listed with name. We can get a timestamp by sending a request for each folder, but that is probably not necessary?
FileEntry fe = new FileEntry(xFolder.Attributes["name"].Value);
fe.IsFolder = true;
yield return fe;
}
foreach (System.Xml.XmlNode xFile in xRoot.SelectNodes("files/file[not(@deleted)]"))
{
var fe = ToFileEntry(xFile);
if (fe != null)
if (xRoot.Attributes["deleted"] != null)
throw new FolderMissingException();
var folders = xRoot.SelectNodes("folders/folder[not(@deleted)]");
if (folders != null)
foreach (System.Xml.XmlNode xFolder in folders)
{
yield return fe;
// Subfolders are only listed with name. We can get a timestamp by sending a request for each folder, but that is probably not necessary?
var name = xFolder.Attributes?["name"]?.Value;
if (string.IsNullOrEmpty(name))
continue;
yield return new FileEntry(name) { IsFolder = true };
}
var files = xRoot.SelectNodes("files/file[not(@deleted)]");
if (files != null)
foreach (System.Xml.XmlNode xFile in files)
{
var fe = ToFileEntry(xFile);
if (fe != null)
yield return fe;
}
}
}
public static IFileEntry ToFileEntry(System.Xml.XmlNode xFile)
public static IFileEntry? ToFileEntry(System.Xml.XmlNode? xFile)
{
string name = xFile.Attributes["name"].Value;
if (xFile == null)
return null;
var name = xFile.Attributes?["name"]?.Value;
if (string.IsNullOrWhiteSpace(name))
return null;
// Normal files have an "currentRevision", which represent the most recent successfully upload
// (could also checked that currentRevision/state is "COMPLETED", but should not be necessary).
// There might also be a newer "latestRevision" coming from an incomplete or corrupt upload,
// but we ignore that here and use the information about the last valid version.
System.Xml.XmlNode xRevision = xFile.SelectSingleNode("currentRevision");
var xRevision = xFile.SelectSingleNode("currentRevision");
if (xRevision != null)
{
System.Xml.XmlNode xState = xRevision.SelectSingleNode("state");
var xState = xRevision.SelectSingleNode("state");
if (xState != null && xState.InnerText == "COMPLETED") // Think "currentRevision" always is a complete version, but just to be on the safe side..
{
System.Xml.XmlNode xSize = xRevision.SelectSingleNode("size");
var xSize = xRevision.SelectSingleNode("size");
long size;
if (xSize == null || !long.TryParse(xSize.InnerText, out size))
size = -1;
DateTime lastModified;
System.Xml.XmlNode xModified = xRevision.SelectSingleNode("modified"); // There is created, modified and updated time stamps, but not last accessed.
if (xModified == null || !DateTime.TryParseExact(xModified.InnerText, JFS_DATE_FORMAT, System.Globalization.CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.AdjustToUniversal, out lastModified))
var xModified = xRevision.SelectSingleNode("modified"); // There is created, modified and updated time stamps, but not last accessed.
if (xModified == null || !DateTime.TryParseExact(xModified.InnerText, JFS_DATE_FORMAT, System.Globalization.CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.AdjustToUniversal, out var lastModified))
lastModified = new DateTime();
FileEntry fe = new FileEntry(name, size, lastModified, lastModified);
return fe;
return new FileEntry(name, size, lastModified, lastModified);
}
}
return null;
@@ -261,22 +282,26 @@ namespace Duplicati.Library.Backend
/// <summary>
/// Retrieves info for a single file (used to determine file size for chunking)
/// </summary>
/// <param name="remotename"></param>
/// <returns></returns>
public IFileEntry Info(string remotename)
/// <param name="remotename">The remote file name</param>
/// <param name="cancelToken">The cancellation token</param>
/// <returns>The file entry</returns>
public async Task<IFileEntry?> Info(string remotename, CancellationToken cancelToken)
{
var doc = new System.Xml.XmlDocument();
try
{
// Send request and load XML response.
var req = CreateRequest(System.Net.WebRequestMethods.Http.Get, remotename, "", false);
var req = CreateRequest(WebRequestMethods.Http.Get, remotename, "", false);
var areq = new Utility.AsyncHttpRequest(req);
using (var rs = areq.GetResponseStream())
doc.Load(rs);
await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ =>
{
using (var rs = areq.GetResponseStream())
doc.Load(rs);
}).ConfigureAwait(false);
}
catch (System.Net.WebException wex)
catch (WebException wex)
{
if (wex.Response is HttpWebResponse response && response.StatusCode == System.Net.HttpStatusCode.NotFound)
if (wex.Response is HttpWebResponse response && response.StatusCode == HttpStatusCode.NotFound)
throw new FileMissingException(wex);
throw;
}
@@ -285,77 +310,74 @@ namespace Duplicati.Library.Backend
// We must check for "deleted" attribute, because files/folders which has it is deleted (attribute contains the timestamp of deletion)
// so we treat them as non-existent here.
var xFile = doc.DocumentElement;
if (xFile.Attributes["deleted"] != null)
{
if (xFile?.Attributes["deleted"] != null)
throw new FileMissingException(string.Format("{0}: {1}", LC.L("The requested file does not exist"), remotename));
}
return ToFileEntry(xFile);
}
public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken)
{
using (System.IO.FileStream fs = System.IO.File.OpenRead(filename))
using (var fs = File.OpenRead(filename))
await PutAsync(remotename, fs, cancelToken).ConfigureAwait(false);
}
public async Task GetAsync(string remotename, string filename, CancellationToken cancelToken)
{
using (System.IO.FileStream fs = System.IO.File.Create(filename))
using (var fs = File.Create(filename))
await GetAsync(remotename, fs, cancelToken).ConfigureAwait(false);
}
public Task DeleteAsync(string remotename, CancellationToken cancelToken)
{
System.Net.HttpWebRequest req = CreateRequest(System.Net.WebRequestMethods.Http.Post, remotename, "rm=true", false); // rm=true means permanent delete, dl=true would be move to trash.
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (System.Net.HttpWebResponse resp = (System.Net.HttpWebResponse)areq.GetResponse())
{ }
var req = CreateRequest(WebRequestMethods.Http.Post, remotename, "rm=true", false); // rm=true means permanent delete, dl=true would be move to trash.
var areq = new Utility.AsyncHttpRequest(req);
return Task.CompletedTask;
}
public IList<ICommandLineArgument> SupportedCommands
{
get
return Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ =>
{
return new List<ICommandLineArgument>([
new CommandLineArgument(AUTHID_OPTION, CommandLineArgument.ArgumentType.Password, Strings.Jottacloud.AuthidShort, Strings.Jottacloud.AuthidLong(OAuthHelper.OAUTH_LOGIN_URL("jottacloud"))),
new CommandLineArgument(JFS_DEVICE_OPTION, CommandLineArgument.ArgumentType.String, Strings.Jottacloud.DescriptionDeviceShort, Strings.Jottacloud.DescriptionDeviceLong(JFS_MOUNT_POINT_OPTION)),
new CommandLineArgument(JFS_MOUNT_POINT_OPTION, CommandLineArgument.ArgumentType.String, Strings.Jottacloud.DescriptionMountPointShort, Strings.Jottacloud.DescriptionMountPointLong(JFS_DEVICE_OPTION)),
new CommandLineArgument(JFS_THREADS, CommandLineArgument.ArgumentType.Integer, Strings.Jottacloud.ThreadsShort, Strings.Jottacloud.ThreadsLong, JFS_DEFAULT_THREADS),
new CommandLineArgument(JFS_CHUNKSIZE, CommandLineArgument.ArgumentType.Size, Strings.Jottacloud.ChunksizeShort, Strings.Jottacloud.ChunksizeLong, JFS_DEFAULT_CHUNKSIZE),
]);
}
using (var resp = (HttpWebResponse)areq.GetResponse())
{ }
});
}
public string Description
{
get { return Strings.Jottacloud.Description; }
}
public IList<ICommandLineArgument> SupportedCommands => [
.. AuthIdOptionsHelper.GetOptions(TOKEN_URL),
new CommandLineArgument(JFS_DEVICE_OPTION, CommandLineArgument.ArgumentType.String, Strings.Jottacloud.DescriptionDeviceShort, Strings.Jottacloud.DescriptionDeviceLong(JFS_MOUNT_POINT_OPTION)),
new CommandLineArgument(JFS_MOUNT_POINT_OPTION, CommandLineArgument.ArgumentType.String, Strings.Jottacloud.DescriptionMountPointShort, Strings.Jottacloud.DescriptionMountPointLong(JFS_DEVICE_OPTION)),
new CommandLineArgument(JFS_THREADS, CommandLineArgument.ArgumentType.Integer, Strings.Jottacloud.ThreadsShort, Strings.Jottacloud.ThreadsLong, JFS_DEFAULT_THREADS),
new CommandLineArgument(JFS_CHUNKSIZE, CommandLineArgument.ArgumentType.Size, Strings.Jottacloud.ChunksizeShort, Strings.Jottacloud.ChunksizeLong, JFS_DEFAULT_CHUNKSIZE),
.. TimeoutOptionsHelper.GetOptions(),
];
public string Description => Strings.Jottacloud.Description;
public Task TestAsync(CancellationToken cancelToken)
=> this.TestListAsync(cancelToken);
public Task CreateFolderAsync(CancellationToken cancelToken)
public async Task CreateFolderAsync(CancellationToken cancelToken)
{
// When using custom (backup) device we must create the device first (if not already exists).
if (!m_device_builtin)
{
var req = CreateRequest(System.Net.WebRequestMethods.Http.Post, m_url_device, "type=WORKSTATION"); // Hard-coding device type. Must be one of "WORKSTATION", "LAPTOP", "IMAC", "MACBOOK", "IPAD", "ANDROID", "IPHONE" or "WINDOWS_PHONE".
var areq = new Utility.AsyncHttpRequest(req);
using (var resp = (System.Net.HttpWebResponse)areq.GetResponse())
{ }
var req = CreateRequest(WebRequestMethods.Http.Post, m_url_device, "type=WORKSTATION"); // Hard-coding device type. Must be one of "WORKSTATION", "LAPTOP", "IMAC", "MACBOOK", "IPAD", "ANDROID", "IPHONE" or "WINDOWS_PHONE".
var areq = new AsyncHttpRequest(req);
await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ =>
{
using (var resp = (HttpWebResponse)areq.GetResponse())
{ }
}).ConfigureAwait(false);
}
// Create the folder path, and if using custom mount point it will be created as well in the same operation.
{
var req = CreateRequest(System.Net.WebRequestMethods.Http.Post, "", "mkDir=true", false);
var areq = new Utility.AsyncHttpRequest(req);
using (var resp = (System.Net.HttpWebResponse)areq.GetResponse())
{ }
var req = CreateRequest(WebRequestMethods.Http.Post, "", "mkDir=true", false);
var areq = new AsyncHttpRequest(req);
await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ =>
{
using (var resp = (HttpWebResponse)areq.GetResponse())
{ }
}).ConfigureAwait(false);
}
return Task.CompletedTask;
}
#endregion
@@ -368,44 +390,49 @@ namespace Duplicati.Library.Backend
#endregion
private System.Net.HttpWebRequest CreateRequest(string method, string url, string queryparams)
private HttpWebRequest CreateRequest(string method, string url, string queryparams)
{
return m_oauth.CreateRequest(url + (string.IsNullOrEmpty(queryparams) || queryparams.Trim().Length == 0 ? "" : "?" + queryparams), method);
}
private System.Net.HttpWebRequest CreateRequest(string method, string remotename, string queryparams, bool upload)
private HttpWebRequest CreateRequest(string method, string remotename, string queryparams, bool upload)
{
var url = (upload ? m_url_upload : m_url) + Library.Utility.Uri.UrlEncode(remotename).Replace("+", "%20");
return CreateRequest(method, url, queryparams);
}
public Task<string[]> GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new string[] {
new Uri(JFS_ROOT).Host,
new Uri(JFS_ROOT_UPLOAD).Host
new System.Uri(JFS_ROOT).Host,
new System.Uri(JFS_ROOT_UPLOAD).Host
});
public async Task GetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken)
public async Task GetAsync(string remotename, Stream stream, CancellationToken cancelToken)
{
if (m_threads > 1)
{
await ParallelGetAsync(remotename, stream, cancelToken).ConfigureAwait(false);
using (var timeoutStream = stream.ObserveWriteTimeout(m_timeouts.ReadWriteTimeout, false))
await ParallelGetAsync(remotename, stream, cancelToken).ConfigureAwait(false);
return;
}
// Downloading from Jottacloud: Will only succeed if the file has a completed revision,
// and if there are multiple versions of the file we will only get the latest completed version,
// ignoring any incomplete or corrupt versions.
var req = CreateRequest(System.Net.WebRequestMethods.Http.Get, remotename, "mode=bin", false);
var areq = new Utility.AsyncHttpRequest(req);
using (var s = areq.GetResponseStream())
await Utility.Utility.CopyStreamAsync(s, stream, true, cancelToken).ConfigureAwait(false);
var req = CreateRequest(WebRequestMethods.Http.Get, remotename, "mode=bin", false);
var areq = new AsyncHttpRequest(req);
using (var s = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => areq.GetResponseStream()).ConfigureAwait(false))
using (var t = s.ObserveReadTimeout(m_timeouts.ReadWriteTimeout))
await Utility.Utility.CopyStreamAsync(t, stream, true, cancelToken).ConfigureAwait(false);
}
/// <summary>
/// Fetches the file in chunks (parallelized)
/// </summary>
public async Task ParallelGetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken)
public async Task ParallelGetAsync(string remotename, Stream stream, CancellationToken cancelToken)
{
var size = Info(remotename).Size;
var info = await Info(remotename, cancelToken);
if (info == null)
throw new FileMissingException(remotename);
var size = info.Size;
var chunks = new Queue<Tuple<long, long>>(); // Tuple => Position (from), Position (to)
@@ -428,11 +455,11 @@ namespace Duplicati.Library.Backend
var item = chunks.Dequeue();
tasks.Enqueue(Task.Run(() =>
{
var req = CreateRequest(System.Net.WebRequestMethods.Http.Get, remotename, "mode=bin", false);
var req = CreateRequest(WebRequestMethods.Http.Get, remotename, "mode=bin", false);
req.AddRange(item.Item1, item.Item2 - 1);
var areq = new Utility.AsyncHttpRequest(req);
var areq = new AsyncHttpRequest(req);
using (var s = areq.GetResponseStream())
using (var reader = new System.IO.BinaryReader(s))
using (var reader = new BinaryReader(s))
{
var length = item.Item2 - item.Item1;
return reader.ReadBytes((int)length);
@@ -444,7 +471,7 @@ namespace Duplicati.Library.Backend
}
}
public async Task PutAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken)
public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken)
{
// Some challenges with uploading to Jottacloud:
// - Jottacloud supports use of a custom header where we can tell the server the MD5 hash of the file
@@ -461,32 +488,34 @@ namespace Duplicati.Library.Backend
// request the server to remove the file again and throw an exception. But there is a requirement that
// we specify the file size in a custom header. And if the stream is not seek-able we are not able
// to use stream.Length, so we are back at square one.
Duplicati.Library.Utility.TempFile tmpFile = null;
var baseStream = stream;
while (baseStream is Duplicati.Library.Utility.OverrideableStream)
baseStream = typeof(Duplicati.Library.Utility.OverrideableStream).GetField("m_basestream", System.Reflection.BindingFlags.DeclaredOnly | System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic).GetValue(baseStream) as System.IO.Stream;
// TODO: This unwrapping code is shared with CloudFiles
TempFile? tmpFile = null;
Stream? baseStream = stream;
while (baseStream is OverrideableStream)
baseStream = typeof(OverrideableStream).GetField("m_basestream", System.Reflection.BindingFlags.DeclaredOnly | System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic)?.GetValue(baseStream) as System.IO.Stream;
if (baseStream == null)
throw new Exception(string.Format("Unable to unwrap stream from: {0}", stream.GetType()));
string md5Hash;
if (baseStream.CanSeek)
{
var originalPosition = baseStream.Position;
using (var md5 = System.Security.Cryptography.MD5.Create())
md5Hash = Library.Utility.Utility.ByteArrayAsHexString(md5.ComputeHash(baseStream));
using (var md5 = MD5.Create())
md5Hash = Utility.Utility.ByteArrayAsHexString(md5.ComputeHash(baseStream));
baseStream.Position = originalPosition;
}
else
{
// No seeking possible, use a temp file
tmpFile = new Duplicati.Library.Utility.TempFile();
using (var os = System.IO.File.OpenWrite(tmpFile))
tmpFile = new TempFile();
using (var os = File.OpenWrite(tmpFile))
using (var hasher = MD5.Create())
using (var md5 = new Utility.HashCalculatingStream(baseStream, hasher))
using (var md5 = new HashCalculatingStream(baseStream, hasher))
{
await Utility.Utility.CopyStreamAsync(md5, os, true, cancelToken).ConfigureAwait(false);
md5Hash = md5.GetFinalHashString();
}
stream = System.IO.File.OpenRead(tmpFile);
stream = File.OpenRead(tmpFile);
}
try
{
@@ -497,7 +526,7 @@ namespace Duplicati.Library.Backend
// HTTP 404 (Not Found) if file does not exists or it exists with a different hash, in which
// case we must send a new request to upload the new content.
var fileSize = stream.Length;
var req = CreateRequest(System.Net.WebRequestMethods.Http.Post, remotename, "umode=nomultipart", true);
var req = CreateRequest(WebRequestMethods.Http.Post, remotename, "umode=nomultipart", true);
req.Headers.Add("JMd5", md5Hash); // Not required, but it will make the server verify the content and mark the file as corrupt if there is a mismatch.
req.Headers.Add("JSize", fileSize.ToString()); // Required, and used to mark file as incomplete if we upload something be the total size of the original file!
// File time stamp headers: Since we are working with a stream here we do not know the local file's timestamps,
@@ -508,14 +537,15 @@ namespace Duplicati.Library.Backend
req.ContentLength = fileSize;
// Write post data request
var areq = new Utility.AsyncHttpRequest(req);
var areq = new AsyncHttpRequest(req);
using (var rs = areq.GetRequestStream())
using (var ts = rs.ObserveWriteTimeout(m_timeouts.ReadWriteTimeout))
await Utility.Utility.CopyStreamAsync(stream, rs, true, cancelToken).ConfigureAwait(false);
// Send request, and check response
using (var resp = (System.Net.HttpWebResponse)areq.GetResponse())
using (var resp = (HttpWebResponse)areq.GetResponse())
{
if (resp.StatusCode != System.Net.HttpStatusCode.Created)
throw new System.Net.WebException(Strings.Jottacloud.FileUploadError, null, System.Net.WebExceptionStatus.ProtocolError, resp);
if (resp.StatusCode != HttpStatusCode.Created)
throw new WebException(Strings.Jottacloud.FileUploadError, null, WebExceptionStatus.ProtocolError, resp);
// Request seems to be successful, but we must verify the response XML content to be sure that the file
// was correctly uploaded: The server will verify the JSize header and mark the file as incomplete if
@@ -533,7 +563,7 @@ namespace Duplicati.Library.Backend
try { doc.Load(rs); }
catch (System.Xml.XmlException)
{
throw new System.Net.WebException(Strings.Jottacloud.FileUploadError, System.Net.WebExceptionStatus.ProtocolError);
throw new WebException(Strings.Jottacloud.FileUploadError, WebExceptionStatus.ProtocolError);
}
bool uploadCompletedSuccessfully = false;
var xFile = doc["file"];
@@ -548,7 +578,7 @@ namespace Duplicati.Library.Backend
}
}
if (!uploadCompletedSuccessfully) // Report error (and we just let the incomplete/corrupt file revision stay on the server..)
throw new System.Net.WebException(Strings.Jottacloud.FileUploadError, System.Net.WebExceptionStatus.ProtocolError);
throw new WebException(Strings.Jottacloud.FileUploadError, WebExceptionStatus.ProtocolError);
}
}
}
@@ -52,23 +52,23 @@ namespace Duplicati.Library.Backend
private class UserInfo
{
[JsonProperty("sub")]
public string Subject { get; set; }
public string? Subject { get; set; }
[JsonProperty("email_verified")]
public bool EmailVerified { get; set; }
[JsonProperty("name")]
public string Name { get; set; }
public string? Name { get; set; }
[JsonProperty("realm")]
public string Realm { get; set; }
public string? Realm { get; set; }
[JsonProperty("preferred_username")]
public string PreferredUsername { get; set; } // The numeric internal username, same as Username
public string? PreferredUsername { get; set; } // The numeric internal username, same as Username
[JsonProperty("given_name")]
public string GivenName { get; set; }
public string? GivenName { get; set; }
[JsonProperty("family_name")]
public string FamilyName { get; set; }
public string? FamilyName { get; set; }
[JsonProperty("email")]
public string Email { get; set; }
public string? Email { get; set; }
[JsonProperty("username")]
public string Username { get; set; } // The numeric internal username, same as PreferredUsername
public string? Username { get; set; } // The numeric internal username, same as PreferredUsername
}
}
@@ -19,13 +19,12 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Localization.Short;
namespace Duplicati.Library.Backend.Strings {
namespace Duplicati.Library.Backend.Strings
{
internal static class Jottacloud
{
public static string Description { get { return LC.L(@"This backend can read and write data to Jottacloud using its REST protocol. Allowed format is ""jottacloud://folder/subfolder""."); } }
public static string DisplayName { get { return LC.L(@"Jottacloud"); } }
public static string AuthidLong(string url) { return LC.L(@"The authorization token retrieved from {0}", url); }
public static string AuthidShort { get { return LC.L(@"The authorization code"); } }
public static string NoUsernameError { get { return LC.L(@"No username found"); } }
public static string NoPathError { get { return LC.L(@"No path given. Files cannot be uploaded to the root folder"); } }
public static string IllegalMountPoint { get { return LC.L(@"Illegal mount point given."); } }
@@ -4,6 +4,8 @@
<TargetFramework>net8.0</TargetFramework>
<OutputType>Library</OutputType>
<Copyright>Copyright © 2025 Team Duplicati, MIT license</Copyright>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="MegaApiClient" Version="1.10.4" />
+79 -90
View File
@@ -22,11 +22,8 @@
using CG.Web.MegaApiClient;
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Duplicati.Library.Utility;
using Duplicati.Library.Utility.Options;
using OtpNet;
using System.Runtime.CompilerServices;
@@ -36,68 +33,67 @@ namespace Duplicati.Library.Backend.Mega
// This class is instantiated dynamically in the BackendLoader.
public class MegaBackend : IBackend, IStreamingBackend
{
private readonly string m_username = null;
private readonly string m_password = null;
private readonly string m_twoFactorKey = null;
private Dictionary<string, List<INode>> m_filecache;
private INode m_currentFolder = null;
private readonly string m_prefix = null;
private readonly string m_username;
private readonly string m_password;
private readonly string? m_twoFactorKey;
private Dictionary<string, List<INode>>? m_filecache;
private INode? m_currentFolder = null;
private readonly string m_prefix;
private readonly TimeoutOptionsHelper.Timeouts m_timeouts;
private MegaApiClient m_client;
private MegaApiClient? m_client;
public MegaBackend()
{
m_username = null!;
m_password = null!;
m_twoFactorKey = null;
m_prefix = null!;
m_timeouts = null!;
}
private MegaApiClient Client
private async Task<MegaApiClient> GetClient(CancellationToken cancelToken)
{
get
{
if (m_client == null)
if (m_client == null)
m_client = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, async ct =>
{
var cl = new MegaApiClient();
if (m_twoFactorKey == null)
cl.Login(m_username, m_password);
await cl.LoginAsync(m_username, m_password).ConfigureAwait(false);
else
{
var totp = new Totp(Base32Encoding.ToBytes(m_twoFactorKey)).ComputeTotp();
cl.Login(m_username, m_password, totp);
await cl.LoginAsync(m_username, m_password, totp).ConfigureAwait(false);
}
m_client = cl;
}
return cl;
}).ConfigureAwait(false);
return m_client;
}
return m_client;
}
public MegaBackend(string url, Dictionary<string, string> options)
public MegaBackend(string url, Dictionary<string, string?> options)
{
var uri = new Utility.Uri(url);
if (options.ContainsKey("auth-username"))
m_username = options["auth-username"];
if (options.ContainsKey("auth-password"))
m_password = options["auth-password"];
var auth = AuthOptionsHelper.Parse(options, uri);
if (options.ContainsKey("auth-two-factor-key"))
m_twoFactorKey = options["auth-two-factor-key"];
if (!string.IsNullOrEmpty(uri.Username))
m_username = uri.Username;
if (!string.IsNullOrEmpty(uri.Password))
m_password = uri.Password;
if (string.IsNullOrEmpty(m_username))
throw new UserInformationException(Strings.MegaBackend.NoUsernameError, "MegaNoUsername");
if (string.IsNullOrEmpty(m_password))
throw new UserInformationException(Strings.MegaBackend.NoPasswordError, "MegaNoPassword");
(m_username, m_password) = auth.GetCredentials();
m_prefix = uri.HostAndPath ?? "";
m_timeouts = TimeoutOptionsHelper.Parse(options);
}
private async Task FetchCurrentFolderAsync(bool autocreate, CancellationToken cancelToken)
private async Task<INode> FetchCurrentFolderAsync(bool autocreate, CancellationToken cancelToken)
{
var client = await GetClient(cancelToken).ConfigureAwait(false);
var parts = m_prefix.Split(new string[] { "/" }, StringSplitOptions.RemoveEmptyEntries);
var nodes = Client.GetNodes();
var nodes = await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, _ => client.GetNodes()).ConfigureAwait(false);
INode parent = nodes.First(x => x.Type == NodeType.Root);
foreach (var n in parts)
@@ -108,7 +104,7 @@ namespace Duplicati.Library.Backend.Mega
if (!autocreate)
throw new FolderMissingException();
item = await Client.CreateFolderAsync(n, parent).ConfigureAwait(false);
item = await client.CreateFolderAsync(n, parent).ConfigureAwait(false);
}
parent = item;
@@ -117,14 +113,17 @@ namespace Duplicati.Library.Backend.Mega
m_currentFolder = parent;
await ResetFileCacheAsync(nodes, cancelToken).ConfigureAwait(false);
return m_currentFolder;
}
private async Task<INode> GetCurrentFolderAsync(CancellationToken cancelToken)
{
if (m_currentFolder == null)
await FetchCurrentFolderAsync(false, cancelToken).ConfigureAwait(false);
var folder = m_currentFolder;
if (folder == null)
folder = await FetchCurrentFolderAsync(false, cancelToken).ConfigureAwait(false);
return m_currentFolder;
return folder;
}
private async Task<INode> GetFileNodeAsync(string name, CancellationToken cancelToken)
@@ -140,17 +139,23 @@ namespace Duplicati.Library.Backend.Mega
throw new FileMissingException();
}
private async Task ResetFileCacheAsync(IEnumerable<INode> list, CancellationToken cancelToken)
private async Task<Dictionary<string, List<INode>>> ResetFileCacheAsync(IEnumerable<INode>? list, CancellationToken cancelToken)
{
if (m_currentFolder == null)
{
await FetchCurrentFolderAsync(false, cancelToken).ConfigureAwait(false);
// We build it as part of the fetch step
return m_filecache!;
}
else
{
var client = await GetClient(cancelToken).ConfigureAwait(false);
var currentFolder = await GetCurrentFolderAsync(cancelToken).ConfigureAwait(false);
m_filecache =
(list ?? Client.GetNodes()).Where(x => x.Type == NodeType.File && x.ParentId == currentFolder.Id)
if (list == null)
list = await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, _ => client.GetNodes()).ConfigureAwait(false);
return m_filecache =
list.Where(x => x.Type == NodeType.File && x.ParentId == currentFolder.Id)
.GroupBy(x => x.Name, x => x, (k, g) => new KeyValuePair<string, List<INode>>(k, g.ToList()))
.ToDictionary(x => x.Key, x => x.Value);
}
@@ -158,19 +163,22 @@ namespace Duplicati.Library.Backend.Mega
#region IStreamingBackend implementation
public async Task PutAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken)
public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken)
{
try
{
if (m_filecache == null)
await ResetFileCacheAsync(null, cancelToken).ConfigureAwait(false);
var filecache = m_filecache;
if (filecache == null)
filecache = await ResetFileCacheAsync(null, cancelToken).ConfigureAwait(false);
var client = await GetClient(cancelToken).ConfigureAwait(false);
var currentFolder = await GetCurrentFolderAsync(cancelToken).ConfigureAwait(false);
var el = await Client.UploadAsync(stream, remotename, currentFolder, new Progress(), null, cancelToken).ConfigureAwait(false);
if (m_filecache.ContainsKey(remotename))
using var ts = stream.ObserveReadTimeout(m_timeouts.ReadWriteTimeout, false);
var el = await client.UploadAsync(ts, remotename, currentFolder, new Progress(), null, cancelToken).ConfigureAwait(false);
if (filecache.ContainsKey(remotename))
await DeleteAsync(remotename, cancelToken).ConfigureAwait(false);
m_filecache[remotename] = [el];
filecache[remotename] = [el];
}
catch
{
@@ -181,9 +189,11 @@ namespace Duplicati.Library.Backend.Mega
public async Task GetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken)
{
var client = await GetClient(cancelToken).ConfigureAwait(false);
var node = await GetFileNodeAsync(remotename, cancelToken).ConfigureAwait(false);
using (var s = Client.Download(node))
await Library.Utility.Utility.CopyStreamAsync(s, stream, cancelToken).ConfigureAwait(false);
using (var s = await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, _ => client.Download(node)).ConfigureAwait(false))
using (var t = s.ObserveReadTimeout(m_timeouts.ReadWriteTimeout))
await Utility.Utility.CopyStreamAsync(t, stream, cancelToken).ConfigureAwait(false);
}
#endregion
@@ -193,10 +203,11 @@ namespace Duplicati.Library.Backend.Mega
/// <inheritdoc />
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancelToken)
{
if (m_filecache == null)
await ResetFileCacheAsync(null, cancelToken).ConfigureAwait(false);
var filecache = m_filecache;
if (filecache == null)
filecache = await ResetFileCacheAsync(null, cancelToken).ConfigureAwait(false);
foreach (var n in m_filecache.Values)
foreach (var n in filecache.Values)
{
var item = n.OrderByDescending(x => x.ModificationDate).First();
yield return new FileEntry(item.Name, item.Size, item.ModificationDate ?? new DateTime(0), item.ModificationDate ?? new DateTime(0));
@@ -219,16 +230,18 @@ namespace Duplicati.Library.Backend.Mega
{
try
{
if (m_filecache == null || !m_filecache.ContainsKey(remotename))
await ResetFileCacheAsync(null, cancelToken).ConfigureAwait(false);
var filecache = m_filecache;
if (filecache == null || !filecache.ContainsKey(remotename))
filecache = await ResetFileCacheAsync(null, cancelToken).ConfigureAwait(false);
if (!m_filecache.ContainsKey(remotename))
if (!filecache.ContainsKey(remotename))
throw new FileMissingException();
foreach (var n in m_filecache[remotename])
await Client.DeleteAsync(n, false).ConfigureAwait(false);
var client = await GetClient(cancelToken).ConfigureAwait(false);
foreach (var n in filecache[remotename])
await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, _ => client.DeleteAsync(n, false)).ConfigureAwait(false);
m_filecache.Remove(remotename);
filecache.Remove(remotename);
}
catch
{
@@ -245,41 +258,17 @@ namespace Duplicati.Library.Backend.Mega
return FetchCurrentFolderAsync(true, cancelToken);
}
public string DisplayName
{
get
{
return Strings.MegaBackend.DisplayName;
}
}
public string DisplayName => Strings.MegaBackend.DisplayName;
public string ProtocolKey
{
get
{
return "mega";
}
}
public string ProtocolKey => "mega";
public IList<ICommandLineArgument> SupportedCommands
{
get
{
return new List<ICommandLineArgument>(new ICommandLineArgument[] {
new CommandLineArgument("auth-password", CommandLineArgument.ArgumentType.Password, Strings.MegaBackend.AuthPasswordDescriptionShort, Strings.MegaBackend.AuthPasswordDescriptionLong),
new CommandLineArgument("auth-username", CommandLineArgument.ArgumentType.String, Strings.MegaBackend.AuthUsernameDescriptionShort, Strings.MegaBackend.AuthUsernameDescriptionLong),
new CommandLineArgument("auth-two-factor-key", CommandLineArgument.ArgumentType.Password, Strings.MegaBackend.AuthTwoFactorKeyDescriptionShort, Strings.MegaBackend.AuthTwoFactorKeyDescriptionLong),
});
}
}
public IList<ICommandLineArgument> SupportedCommands => [
.. AuthOptionsHelper.GetOptions(),
new CommandLineArgument("auth-two-factor-key", CommandLineArgument.ArgumentType.Password, Strings.MegaBackend.AuthTwoFactorKeyDescriptionShort, Strings.MegaBackend.AuthTwoFactorKeyDescriptionLong),
.. TimeoutOptionsHelper.GetOptions()
];
public string Description
{
get
{
return Strings.MegaBackend.Description;
}
}
public string Description => Strings.MegaBackend.Description;
public Task<string[]> GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(Array.Empty<string>());
@@ -38,7 +38,7 @@ namespace Duplicati.Library;
public class JsonWebHelperHttpClient(HttpClient httpClient)
{
/// <summary>
/// HttpClient reference
/// HttpClient reference for inheritors
/// </summary>
protected readonly HttpClient _httpClient = httpClient;
@@ -46,21 +46,47 @@ public class JsonWebHelperHttpClient(HttpClient httpClient)
/// Useragent string building method
/// </summary>
protected string UserAgent => $"Duplicati v{System.Reflection.Assembly.GetExecutingAssembly().GetName().Version}";
public event Action<HttpRequestMessage> CreateSetupHelper;
/// <summary>
/// Centralized method to prepare a request with the given URL and method setting useragent
/// </summary>
/// <param name="url">Url</param>
/// <param name="method">Method</param>
public virtual HttpRequestMessage CreateRequest(string url, string method = null)
public virtual Task<HttpRequestMessage> CreateRequestAsync(string url, HttpMethod method, CancellationToken cancelToken)
{
HttpRequestMessage request = new HttpRequestMessage(string.IsNullOrEmpty(method) ? HttpMethod.Get : new HttpMethod(method), url);
var request = new HttpRequestMessage(method, url);
request.Headers.Add("User-Agent", UserAgent);
CreateSetupHelper?.Invoke(request);
return Task.FromResult(request);
}
return request;
/// <summary>
/// Performs a multipart post and parses the response as JSON
/// </summary>
/// <returns>The parsed JSON item.</returns>
/// <param name="url">The url to post to.</param>
/// <param name="cancellationToken">Token to cancel the operation.</param>
/// <param name="parts">The multipart items.</param>
/// <typeparam name="T">The return type parameter.</typeparam>
public virtual async Task<T> PostMultipartAndGetJsonDataAsync<T>(string url, CancellationToken cancellationToken, MultipartContent parts)
{
using var response = await PostMultipartAsync(url, cancellationToken, parts).ConfigureAwait(false);
return await ReadJsonResponseAsync<T>(response, cancellationToken).ConfigureAwait(false);
}
/// <summary>
/// Performs a multipart post
/// </summary>
/// <returns>The response.</returns>
/// <param name="url">The url to post to.</param>
/// <param name="cancellationToken">Token to cancel the operation.</param>
/// <param name="parts">The multipart items.</param>
protected virtual async Task<HttpResponseMessage> PostMultipartAsync(string url, CancellationToken cancellationToken, MultipartContent parts)
{
using var req = await CreateRequestAsync(url, HttpMethod.Post, cancellationToken).ConfigureAwait(false);
req.Content = parts;
return await _httpClient.SendAsync(req, cancellationToken).ConfigureAwait(false);
}
/// <summary>
@@ -72,7 +98,7 @@ public class JsonWebHelperHttpClient(HttpClient httpClient)
/// <typeparam name="T">Destination Type</typeparam>
protected virtual async Task<T> GetJsonDataAsync<T>(string url, CancellationToken cancellationToken, Action<HttpRequestMessage> setup = null)
{
var req = CreateRequest(url);
using var req = await CreateRequestAsync(url, HttpMethod.Get, cancellationToken).ConfigureAwait(false);
if (setup != null)
setup(req);
@@ -104,6 +130,26 @@ public class JsonWebHelperHttpClient(HttpClient httpClient)
).ConfigureAwait(false);
}
/// <summary>
/// Executes a web request and json-deserializes the results as the specified type
/// </summary>
/// <returns>The deserialized JSON data.</returns>
/// <param name="url">The remote URL</param>
/// <param name="cancellationToken">Token to cancel the operation.</param>
/// <typeparam name="T">The type of data to return.</typeparam>
public virtual async Task<T> GetJsonDataAsync<T>(string url, CancellationToken cancellationToken)
{
return await GetJsonDataAsync<T>(
url,
cancellationToken,
request =>
{
request.Method = HttpMethod.Get;
}
).ConfigureAwait(false);
}
/// <summary>
/// Reads the JSON response from the server and deserializes it into the given type
/// </summary>
@@ -112,35 +158,8 @@ public class JsonWebHelperHttpClient(HttpClient httpClient)
/// <typeparam name="T">Destination Type</typeparam>
protected virtual async Task<T> ReadJsonResponseAsync<T>(HttpRequestMessage req, CancellationToken cancellationToken)
{
using var resp = await GetResponseAsync(req, cancellationToken).ConfigureAwait(false);
return await ReadJsonResponsAsync<T>(resp, cancellationToken).ConfigureAwait(false);
}
/// <summary>
/// Read the JSON response from the server and deserialize it into the given type
/// </summary>
/// <param name="response">Response object</param>
/// <typeparam name="T">Type to cast to</typeparam>
/// <returns></returns>
/// <exception cref="IOException">Exception when failing to deserialize the JSON to the Type</exception>
protected virtual T ReadJsonResponse<T>(HttpResponseMessage response)
{
using var rs = response.Content.ReadAsStream();
using var ps = new StreamPeekReader(rs);
try
{
using var tr = new StreamReader(ps);
using var jr = new JsonTextReader(tr);
return new JsonSerializer().Deserialize<T>(jr);
}
catch (Exception ex)
{
// If we get invalid JSON, report the peek value
if (ex is JsonReaderException)
throw new IOException($"Invalid JSON data: \"{ps.PeekData()}\"", ex);
// Otherwise, we have no additional help to offer
throw;
}
using var resp = await GetResponseAsync(req, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
return await ReadJsonResponseAsync<T>(resp, cancellationToken).ConfigureAwait(false);
}
/// <summary>
@@ -151,7 +170,7 @@ public class JsonWebHelperHttpClient(HttpClient httpClient)
/// <typeparam name="T">Type to cast to</typeparam>
/// <returns></returns>
/// <exception cref="IOException">Exception when failing to deserialize the JSON to the Type</exception>
protected virtual async Task<T> ReadJsonResponsAsync<T>(HttpResponseMessage response, CancellationToken cancellationToken)
protected virtual async Task<T> ReadJsonResponseAsync<T>(HttpResponseMessage response, CancellationToken cancellationToken)
{
await using var rs = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
await using var ps = new StreamPeekReader(rs);
@@ -175,9 +194,9 @@ public class JsonWebHelperHttpClient(HttpClient httpClient)
/// Use this method to register an exception handler,
/// which can throw another, more meaningful exception
/// </summary>
public virtual void AttemptParseAndThrowException(Exception ex, HttpResponseMessage responseContext = null)
public virtual Task AttemptParseAndThrowExceptionAsync(Exception ex, HttpResponseMessage responseContext, CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
/// <summary>
@@ -186,18 +205,19 @@ public class JsonWebHelperHttpClient(HttpClient httpClient)
/// <param name="req">Request object</param>
/// <param name="cancellationToken">Cancellation Token</param>
/// <returns></returns>
private async Task<HttpResponseMessage> GetResponseAsync(HttpRequestMessage req, CancellationToken cancellationToken)
public async Task<HttpResponseMessage> GetResponseAsync(HttpRequestMessage req, HttpCompletionOption httpCompletionOption, CancellationToken cancellationToken)
{
HttpResponseMessage response = null;
try
{
response = await _httpClient.SendAsync(req, cancellationToken).ConfigureAwait(false);
response = await _httpClient.SendAsync(req, httpCompletionOption, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
return response;
}
catch (Exception ex)
{
AttemptParseAndThrowException(ex, response);
await AttemptParseAndThrowExceptionAsync(ex, response, cancellationToken).ConfigureAwait(false);
throw;
}
}
@@ -277,6 +297,5 @@ public class JsonWebHelperHttpClient(HttpClient httpClient)
return source.Read(buffer, offset, count) + br;
}
}
}
@@ -1,60 +1,58 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Net;
using Duplicati.Library.Utility;
using System.Collections.Generic;
using System.Web;
namespace Duplicati.Library
{
/// <summary>
/// Class for providing call-context access to http settings
/// </summary>
public static class OAuthContextSettings
{
/// <summary>
/// The struct wrapping the OAuth settings
/// </summary>
private struct OAuthSettings
{
/// <summary>
/// The server url
/// </summary>
public string ServerURL;
}
/// <summary>
/// Class for providing call-context access to http settings
/// </summary>
public static class OAuthContextSettings
{
/// <summary>
/// The struct wrapping the OAuth settings
/// </summary>
private struct OAuthSettings
{
/// <summary>
/// The server url
/// </summary>
public string ServerURL;
}
/// <summary>
/// Starts the session.
/// </summary>
/// <returns>The session.</returns>
/// <param name="serverurl">The url to use for the server.</param>
public static IDisposable StartSession(string serverurl)
{
/// <summary>
/// Starts the session.
/// </summary>
/// <returns>The session.</returns>
/// <param name="serverurl">The url to use for the server.</param>
public static IDisposable StartSession(string serverurl)
{
return CallContextSettings<OAuthSettings>.StartContext(new OAuthSettings { ServerURL = serverurl });
}
}
/// <summary>
/// Gets the server URL to use for OAuth.
/// </summary>
/// <summary>
/// Gets the server URL to use for OAuth.
/// </summary>
public static string ServerURL
{
get
@@ -63,7 +61,7 @@ namespace Duplicati.Library
return string.IsNullOrWhiteSpace(r) ? OAuthHelper.DUPLICATI_OAUTH_SERVICE : r;
}
}
}
}
public class OAuthHelper : JSONWebHelper
{
@@ -73,11 +71,11 @@ namespace Duplicati.Library
public const string DUPLICATI_OAUTH_SERVICE = "https://duplicati-oauth-handler.appspot.com/refresh";
public static string OAUTH_LOGIN_URL(string modulename)
public static string OAUTH_LOGIN_URL(string modulename)
{
var u = new Library.Utility.Uri(OAuthContextSettings.ServerURL);
var addr = u.SetPath("").SetQuery((u.Query ?? "") + (string.IsNullOrWhiteSpace(u.Query) ? "" : "&") + "type={0}");
return string.Format(addr.ToString(), modulename);
return string.Format(addr.ToString(), modulename);
}
/// <summary>
@@ -106,7 +104,7 @@ namespace Duplicati.Library
OAuthLoginUrl = OAUTH_LOGIN_URL(servicename);
if (string.IsNullOrEmpty(authid))
throw new Duplicati.Library.Interface.UserInformationException(Strings.OAuthHelper.MissingAuthID(OAuthLoginUrl), "MissingAuthID");
throw new Interface.UserInformationException(Strings.OAuthHelper.MissingAuthID(OAuthLoginUrl), "MissingAuthID");
}
public T GetTokenResponse<T>()
@@ -129,7 +127,7 @@ namespace Duplicati.Library
if (!noAuthorization && AutoAuthHeader && !string.Equals(OAuthContextSettings.ServerURL, url))
r.Headers["Authorization"] = string.Format("Bearer {0}", AccessToken);
return r;
}
}
public string AccessToken
{
@@ -142,7 +140,7 @@ namespace Duplicati.Library
{
var retries = 0;
while(true)
while (true)
{
try
{
@@ -0,0 +1,196 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Threading;
using System.Threading.Tasks;
using Duplicati.Library.Utility;
namespace Duplicati.Library;
public class OAuthHelperHttpClient : JsonWebHelperHttpClient
{
private string _Token;
private string _Authid;
private DateTime _mTokenExpires = DateTime.UtcNow;
private string OAuthLoginUrl { get; }
/// <summary>
/// Timeout for authentication requests
/// </summary>
private static readonly TimeSpan AUTHENTICATION_TIMEOUT = TimeSpan.FromSeconds(25);
/// <summary>
/// Maximum number of retries for authorization
/// </summary>
private const int MAX_AUTHORIZATION_RETRIES = 5;
/// <summary>
/// URL to use for OAuth login
/// </summary>
/// <param name="modulename">The name of the module</param>
/// <returns>The URL to use for OAuth login</returns>
private static string OAUTH_LOGIN_URL(string modulename)
{
var u = new Utility.Uri(OAuthContextSettings.ServerURL);
var addr = u.SetPath("")
.SetQuery((u.Query ?? "") + (string.IsNullOrWhiteSpace(u.Query) ? "" : "&") + "type={0}");
return string.Format(addr.ToString(), modulename);
}
/// <summary>
/// Set to true to automatically add the Authorization header to requests
/// </summary>
public bool AutoAuthHeader { get; set; }
/// <summary>
/// Set to true if the provider does not use refresh tokens, but only access tokens
/// </summary>
public bool AccessTokenOnly { get; set; }
/// <summary>
/// If true (the default), when a v1 authid is being used it will be swapped
/// with a v2 authid, when the OAuth service returns one (which it dypically
/// does after a provider token refresh has been performed). Some providers
/// are not compatible with v2 authid, tyically because they generate a new
/// refresh token with every access token refresh and invalidates the old.
/// If the oauth service still returns a v2 authid for such a provider,
/// set this property to false to make Duplicati ignore it.
/// </summary>
public bool AutoV2 { get; set; } = true;
public OAuthHelperHttpClient(string authid, string servicename, HttpClient httpClient = null, string useragent = null)
: base(httpClient ?? HttpClientHelper.CreateClient())
{
_Authid = authid;
OAuthLoginUrl = OAUTH_LOGIN_URL(servicename);
if (string.IsNullOrEmpty(authid))
throw new Interface.UserInformationException(
Strings.OAuthHelper.MissingAuthID(OAuthLoginUrl), "MissingAuthID");
}
private async Task<HttpRequestMessage> CreateRequestAsync(string url, HttpMethod method, bool noAuthorization, CancellationToken cancellationToken)
{
var request = new HttpRequestMessage(method, url);
request.Headers.Add("User-Agent", UserAgent);
if (!noAuthorization && AutoAuthHeader && !string.Equals(OAuthContextSettings.ServerURL, url)) request.Headers.Authorization = new AuthenticationHeaderValue("Bearer",
await GetAccessTokenAsync(cancellationToken).ConfigureAwait(false));
return request;
}
public override Task<HttpRequestMessage> CreateRequestAsync(string url, HttpMethod method, CancellationToken cancellationToken)
=> CreateRequestAsync(url, method, false, cancellationToken);
private async Task<string> GetAccessTokenAsync(CancellationToken cancellationToken)
{
if (AccessTokenOnly)
return _Authid;
if (_Token == null || _mTokenExpires < DateTime.UtcNow)
{
var retries = 0;
while (true)
{
HttpResponseMessage response = null;
try
{
using var request = await CreateRequestAsync(OAuthContextSettings.ServerURL, HttpMethod.Get, false, cancellationToken).ConfigureAwait(false);
return await Utility.Utility.WithTimeout(AUTHENTICATION_TIMEOUT, cancellationToken, async ct =>
{
request.Headers.TryAddWithoutValidation("X-AuthID", _Authid);
response = await _httpClient.SendAsync(request, ct).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
var res = await ReadJsonResponseAsync<OAuthServiceResponse>(response, ct).ConfigureAwait(false);
_mTokenExpires = DateTime.UtcNow.AddSeconds(res.expires - 30);
if (AutoV2 && !string.IsNullOrWhiteSpace(res.v2_authid))
_Authid = res.v2_authid;
return _Token = res.access_token;
}).ConfigureAwait(false);
}
catch (Exception ex)
{
var clientError = false;
try
{
// Only retry once on client errors
if (ex is HttpRequestException { StatusCode: not null } exception)
{
var sc = (int)exception.StatusCode;
clientError = sc is >= 400 and <= 499;
}
}
catch
{
// ignored
}
if (response != null && response.Headers.Contains("X-Reason"))
{
var msg = response.Headers.GetValues("X-Reason").FirstOrDefault();
if (string.IsNullOrWhiteSpace(msg))
msg = response.StatusCode.ToString();
if (response.StatusCode == HttpStatusCode.ServiceUnavailable)
{
string errorKey = msg == response.StatusCode.ToString()
? "OAuthOverQuotaError"
: "OAuthLoginError";
string errorMessage = errorKey == "OAuthOverQuotaError"
? Strings.OAuthHelper.OverQuotaError
: Strings.OAuthHelper.AuthorizationFailure(msg, OAuthLoginUrl);
throw new Interface.UserInformationException(errorMessage, errorKey,
errorKey == "OAuthLoginError" ? ex : null);
}
}
if (retries >= (clientError ? 1 : MAX_AUTHORIZATION_RETRIES))
{
await AttemptParseAndThrowExceptionAsync(ex, response, cancellationToken).ConfigureAwait(false);
throw;
}
Thread.Sleep(TimeSpan.FromSeconds(Math.Pow(2, retries)));
retries++;
}
finally
{
response?.Dispose();
}
}
}
return _Token;
}
}
@@ -0,0 +1,33 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
namespace Duplicati.Library;
public class OAuthServiceResponse
{
public string access_token { get; set; }
[Newtonsoft.Json.JsonProperty(NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
public int expires { get; set; }
[Newtonsoft.Json.JsonProperty(NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
public string v2_authid { get; set; }
}
@@ -5,6 +5,8 @@
<OutputType>Library</OutputType>
<RootNamespace>Duplicati.Library.Backend</RootNamespace>
<Copyright>Copyright © 2025 Team Duplicati, MIT license</Copyright>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>
<ItemGroup>
@@ -1,30 +1,25 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.IO;
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Duplicati.Library.Utility;
@@ -36,11 +31,11 @@ namespace Duplicati.Library.Backend.MicrosoftGraph
{
private static readonly Regex authorizationHeaderRemover = new Regex(@"Authorization:\s*Bearer\s+\S+", RegexOptions.IgnoreCase);
private readonly HttpResponseMessage responseMessage;
private readonly HttpWebResponse webResponse;
private readonly HttpResponseMessage? responseMessage;
private readonly HttpWebResponse? webResponse;
public MicrosoftGraphException(HttpResponseMessage response)
: this(string.Format("{0}: {1} error from request {2}", response.StatusCode, response.ReasonPhrase, response.RequestMessage.RequestUri), response)
: this(string.Format("{0}: {1} error from request {2}", response.StatusCode, response.ReasonPhrase, response.RequestMessage?.RequestUri), response)
{
}
@@ -49,7 +44,7 @@ namespace Duplicati.Library.Backend.MicrosoftGraph
{
}
public MicrosoftGraphException(string message, HttpResponseMessage response, Exception innerException)
public MicrosoftGraphException(string message, HttpResponseMessage response, Exception? innerException)
: base(BuildFullMessage(message, response), innerException)
{
this.responseMessage = response;
@@ -65,59 +60,40 @@ namespace Duplicati.Library.Backend.MicrosoftGraph
{
}
public MicrosoftGraphException(string message, HttpWebResponse response, Exception innerException)
public MicrosoftGraphException(string message, HttpWebResponse response, Exception? innerException)
: base(BuildFullMessage(message, response), innerException)
{
this.webResponse = response;
}
public string RequestUrl
{
get
{
if (this.responseMessage != null)
{
return this.responseMessage.RequestMessage.RequestUri.ToString();
}
else
{
return this.webResponse.ResponseUri.ToString();
}
}
}
public HttpStatusCode StatusCode
{
get
{
if (this.responseMessage != null)
{
return this.responseMessage.StatusCode;
}
else
{
return this.webResponse.StatusCode;
}
if (responseMessage != null)
return responseMessage.StatusCode;
if (webResponse != null)
return webResponse.StatusCode;
return HttpStatusCode.InternalServerError;
}
}
protected static string ResponseToString(HttpResponseMessage response)
protected static string? ResponseToString(HttpResponseMessage response)
{
if (response != null)
{
// Start to read the content
using (Task<string> content = response.Content.ReadAsStringAsync())
{
// Since the exception message may be saved / sent in logs, we want to prevent the authorization header from being included.
// it wouldn't be as bad as recording the username/password in logs, since the token will expire, but it doesn't hurt to be safe.
// So we replace anything in the request that looks like the auth header with a safe version.
string requestMessage = authorizationHeaderRemover.Replace(response.RequestMessage.ToString(), "Authorization: Bearer ABC...XYZ");
return string.Format("{0}\n{1}\n{2}", requestMessage, response, PrettifyJson(content.Await()));
}
}
else
{
if (response == null)
return null;
// Start to read the content
using (Task<string> content = response.Content.ReadAsStringAsync())
{
// Since the exception message may be saved / sent in logs, we want to prevent the authorization header from being included.
// it wouldn't be as bad as recording the username/password in logs, since the token will expire, but it doesn't hurt to be safe.
// So we replace anything in the request that looks like the auth header with a safe version.
var requestMessage = response.RequestMessage == null
? ""
: authorizationHeaderRemover.Replace(response.RequestMessage.ToString(), "Authorization: Bearer ABC...XYZ");
return string.Format("{0}\n{1}\n{2}", requestMessage, response, PrettifyJson(content.Await()));
}
}
@@ -133,41 +109,28 @@ namespace Duplicati.Library.Backend.MicrosoftGraph
}
}
protected static string ResponseToString(HttpWebResponse response)
protected static string? ResponseToString(HttpWebResponse response)
{
if (response != null)
{
// Start to read the content
using (var responseStream = response.GetResponseStream())
using (var textReader = new StreamReader(responseStream))
{
return string.Format("{0}\n{1}", response, PrettifyJson(textReader.ReadToEnd()));
}
}
else
{
if (response == null)
return null;
}
// Start to read the content
using (var responseStream = response.GetResponseStream())
using (var textReader = new StreamReader(responseStream))
return string.Format("{0}\n{1}", response, PrettifyJson(textReader.ReadToEnd()));
}
private static string BuildFullMessage(string message, HttpWebResponse response)
{
if (response != null)
{
return string.Format("{0}\n{1}", message, ResponseToString(response));
}
else
{
if (response == null)
return message;
}
return string.Format("{0}\n{1}", message, ResponseToString(response));
}
private static string PrettifyJson(string json)
{
if (string.IsNullOrWhiteSpace(json))
{
return json;
}
if (json[0] == '<')
{
@@ -191,12 +154,12 @@ namespace Duplicati.Library.Backend.MicrosoftGraph
public class DriveItemNotFoundException : MicrosoftGraphException
{
public DriveItemNotFoundException(HttpResponseMessage response)
: base(string.Format("Item at {0} was not found", response?.RequestMessage?.RequestUri?.ToString() ?? "<unknown>"), response)
: base(string.Format("Item at {0} was not found", response.RequestMessage?.RequestUri?.ToString() ?? "<unknown>"), response)
{
}
public DriveItemNotFoundException(HttpWebResponse response)
: base(string.Format("Item at {0} was not found", response?.ResponseUri?.ToString() ?? "<unknown>"), response)
: base(string.Format("Item at {0} was not found", response.ResponseUri?.ToString() ?? "<unknown>"), response)
{
}
}
@@ -209,7 +172,7 @@ namespace Duplicati.Library.Backend.MicrosoftGraph
int fragmentCount,
Exception fragmentException)
: base(
string.Format("Error uploading fragment {0} of {1} for {2}", fragment, fragmentCount, originalResponse?.RequestMessage?.RequestUri?.ToString() ?? "<unknown>"),
string.Format("Error uploading fragment {0} of {1} for {2}", fragment, fragmentCount, originalResponse.RequestMessage?.RequestUri?.ToString() ?? "<unknown>"),
originalResponse,
fragmentException)
{
@@ -223,7 +186,7 @@ namespace Duplicati.Library.Backend.MicrosoftGraph
int fragmentCount,
Exception fragmentException)
: base(
string.Format("Error uploading fragment {0} of {1} for {2}", fragment, fragmentCount, originalResponse?.ResponseUri?.ToString() ?? "<unknown>"),
string.Format("Error uploading fragment {0} of {1} for {2}", fragment, fragmentCount, originalResponse.ResponseUri?.ToString() ?? "<unknown>"),
originalResponse,
fragmentException)
{
@@ -23,18 +23,12 @@ using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.Logging;
using Duplicati.Library.Utility;
using Duplicati.Library.Utility.Options;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace Duplicati.Library.Backend
{
@@ -59,7 +53,6 @@ namespace Duplicati.Library.Backend
private const string BASE_ADDRESS = "https://graph.microsoft.com";
private const string AUTHID_OPTION = "authid";
private const string UPLOAD_SESSION_FRAGMENT_SIZE_OPTION = "fragment-size";
private const string UPLOAD_SESSION_FRAGMENT_RETRY_COUNT_OPTION = "fragment-retry-count";
private const string UPLOAD_SESSION_FRAGMENT_RETRY_DELAY_OPTION = "fragment-retry-delay";
@@ -116,23 +109,29 @@ namespace Duplicati.Library.Backend
// send a request. And before sending any requests, we'll make sure to wait until at least this time.
// Since this may be read and written by multiple threads, it is stored as a long and updated using Interlocked.Exchange.
private readonly RetryAfterHelper m_retryAfter;
protected readonly TimeoutOptionsHelper.Timeouts m_timeouts;
private string[] dnsNames = null;
private string[]? dnsNames = null;
private readonly Lazy<string> rootPathFromURL;
private string RootPath => this.rootPathFromURL.Value;
private string TokenUrl => OAuthHelper.OAUTH_LOGIN_URL(ProtocolKey);
protected MicrosoftGraphBackend() { } // Constructor needed for dynamic loading to find it
protected MicrosoftGraphBackend(string url, string protocolKey, Dictionary<string, string> options)
protected MicrosoftGraphBackend()
{
string authid;
options.TryGetValue(AUTHID_OPTION, out authid);
if (string.IsNullOrEmpty(authid))
throw new UserInformationException(Strings.MicrosoftGraph.MissingAuthId(OAuthHelper.OAUTH_LOGIN_URL(protocolKey)), "MicrosoftGraphBackendMissingAuthId");
// Constructor needed for dynamic loading to find it
m_client = null!;
m_retryAfter = null!;
m_timeouts = null!;
rootPathFromURL = null!;
}
string fragmentSizeStr;
if (options.TryGetValue(UPLOAD_SESSION_FRAGMENT_SIZE_OPTION, out fragmentSizeStr) && int.TryParse(fragmentSizeStr, out this.fragmentSize))
protected MicrosoftGraphBackend(string url, string protocolKey, Dictionary<string, string?> options)
{
var authId = AuthIdOptionsHelper.Parse(options)
.RequireCredentials(TokenUrl);
if (options.TryGetValue(UPLOAD_SESSION_FRAGMENT_SIZE_OPTION, out var fragmentSizeStr) && int.TryParse(fragmentSizeStr, out this.fragmentSize))
{
// Make sure the fragment size is a multiple of the desired multiple size.
// If it isn't, we round down to the nearest multiple below it.
@@ -146,22 +145,17 @@ namespace Duplicati.Library.Backend
this.fragmentSize = UPLOAD_SESSION_FRAGMENT_DEFAULT_SIZE;
}
string fragmentRetryCountStr;
if (!(options.TryGetValue(UPLOAD_SESSION_FRAGMENT_RETRY_COUNT_OPTION, out fragmentRetryCountStr) && int.TryParse(fragmentRetryCountStr, out this.fragmentRetryCount)))
{
if (!(options.TryGetValue(UPLOAD_SESSION_FRAGMENT_RETRY_COUNT_OPTION, out var fragmentRetryCountStr) && int.TryParse(fragmentRetryCountStr, out this.fragmentRetryCount)))
this.fragmentRetryCount = UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_COUNT;
}
string fragmentRetryDelayStr;
if (!(options.TryGetValue(UPLOAD_SESSION_FRAGMENT_RETRY_DELAY_OPTION, out fragmentRetryDelayStr) && int.TryParse(fragmentRetryDelayStr, out this.fragmentRetryDelay)))
{
if (!(options.TryGetValue(UPLOAD_SESSION_FRAGMENT_RETRY_DELAY_OPTION, out var fragmentRetryDelayStr) && int.TryParse(fragmentRetryDelayStr, out this.fragmentRetryDelay)))
this.fragmentRetryDelay = UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_DELAY;
}
this.m_client = new OAuthHttpClient(authid, protocolKey);
this.m_client = new OAuthHttpClient(authId.AuthId!, protocolKey);
this.m_client.BaseAddress = new System.Uri(BASE_ADDRESS);
this.m_retryAfter = RetryAfterHelper.CreateOrGetRetryAfterHelper(url);
this.m_timeouts = TimeoutOptionsHelper.Parse(options);
// Extract out the path to the backup root folder from the given URI. Since this can be an expensive operation,
// we will cache the value using a lazy initializer.
@@ -173,78 +167,67 @@ namespace Duplicati.Library.Backend
public abstract string DisplayName { get; }
public string Description
{
get
{
return this.DescriptionTemplate(
public string Description => this.DescriptionTemplate(
"Microsoft Service Agreement",
SERVICES_AGREEMENT,
"Microsoft Online Privacy Statement",
PRIVACY_STATEMENT);
}
}
public IList<ICommandLineArgument> SupportedCommands
{
get
{
return new[]
{
new CommandLineArgument(AUTHID_OPTION, CommandLineArgument.ArgumentType.Password, Strings.MicrosoftGraph.AuthIdShort, Strings.MicrosoftGraph.AuthIdLong(OAuthHelper.OAUTH_LOGIN_URL(this.ProtocolKey))),
new CommandLineArgument(UPLOAD_SESSION_FRAGMENT_SIZE_OPTION, CommandLineArgument.ArgumentType.Integer, Strings.MicrosoftGraph.FragmentSizeShort, Strings.MicrosoftGraph.FragmentSizeLong, Library.Utility.Utility.FormatSizeString(UPLOAD_SESSION_FRAGMENT_DEFAULT_SIZE)),
new CommandLineArgument(UPLOAD_SESSION_FRAGMENT_RETRY_COUNT_OPTION, CommandLineArgument.ArgumentType.Integer, Strings.MicrosoftGraph.FragmentRetryCountShort, Strings.MicrosoftGraph.FragmentRetryCountLong, UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_COUNT.ToString()),
new CommandLineArgument(UPLOAD_SESSION_FRAGMENT_RETRY_DELAY_OPTION, CommandLineArgument.ArgumentType.Integer, Strings.MicrosoftGraph.FragmentRetryDelayShort, Strings.MicrosoftGraph.FragmentRetryDelayLong, UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_DELAY.ToString()),
new CommandLineArgument("use-http-client", CommandLineArgument.ArgumentType.Boolean, Strings.MicrosoftGraph.UseHttpClientShort, Strings.MicrosoftGraph.UseHttpClientLong, "true", null, null, Strings.MicrosoftGraph.UseHttpClientDeprecated),
}
.Concat(this.AdditionalSupportedCommands).ToList();
}
}
public IList<ICommandLineArgument> SupportedCommands => [
.. AuthIdOptionsHelper.GetOptions(TokenUrl),
new CommandLineArgument(UPLOAD_SESSION_FRAGMENT_SIZE_OPTION, CommandLineArgument.ArgumentType.Integer, Strings.MicrosoftGraph.FragmentSizeShort, Strings.MicrosoftGraph.FragmentSizeLong, Library.Utility.Utility.FormatSizeString(UPLOAD_SESSION_FRAGMENT_DEFAULT_SIZE)),
new CommandLineArgument(UPLOAD_SESSION_FRAGMENT_RETRY_COUNT_OPTION, CommandLineArgument.ArgumentType.Integer, Strings.MicrosoftGraph.FragmentRetryCountShort, Strings.MicrosoftGraph.FragmentRetryCountLong, UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_COUNT.ToString()),
new CommandLineArgument(UPLOAD_SESSION_FRAGMENT_RETRY_DELAY_OPTION, CommandLineArgument.ArgumentType.Integer, Strings.MicrosoftGraph.FragmentRetryDelayShort, Strings.MicrosoftGraph.FragmentRetryDelayLong, UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_DELAY.ToString()),
new CommandLineArgument("use-http-client", CommandLineArgument.ArgumentType.Boolean, Strings.MicrosoftGraph.UseHttpClientShort, Strings.MicrosoftGraph.UseHttpClientLong, "true", null, null, Strings.MicrosoftGraph.UseHttpClientDeprecated),
.. TimeoutOptionsHelper.GetOptions()
];
public async Task<string[]> GetDNSNamesAsync(CancellationToken cancelToken)
{
if (this.dnsNames == null)
if (dnsNames == null)
{
// The DNS names that this instance may need to access include:
// - Core graph API endpoint
// - Upload session endpoint (which seems to be different depending on the drive being accessed - not sure if it can vary for a single drive)
// To get the upload session endpoint, we can start an upload session and then immediately cancel it.
// We pick a random file name (using a guid) to make sure we don't conflict with an existing file
string dnsTestFile = string.Format("DNSNameTest-{0}", Guid.NewGuid());
var uploadSession = await this.PostAsync<UploadSession>(string.Format("{0}/root:{1}{2}:/createUploadSession", this.DrivePrefix, this.RootPath, NormalizeSlashes(dnsTestFile)), MicrosoftGraphBackend.dummyUploadSession, cancelToken).ConfigureAwait(false);
var dnsTestFile = string.Format("DNSNameTest-{0}", Guid.NewGuid());
var drivePrefix = await GetDrivePrefix(cancelToken).ConfigureAwait(false);
var uploadSession = await this.PostAsync<UploadSession>(string.Format("{0}/root:{1}{2}:/createUploadSession", drivePrefix, this.RootPath, NormalizeSlashes(dnsTestFile)), dummyUploadSession, cancelToken).ConfigureAwait(false);
// Canceling an upload session is done by sending a DELETE to the upload URL
await m_retryAfter.WaitForRetryAfterAsync(cancelToken).ConfigureAwait(false);
using (var request = new HttpRequestMessage(HttpMethod.Delete, uploadSession.UploadUrl))
using (var response = await this.m_client.SendAsync(request, false, cancelToken).ConfigureAwait(false))
{
this.CheckResponse(response);
}
using (var response = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken,
ct => m_client.SendAsync(request, false, ct)).ConfigureAwait(false))
CheckResponse(response);
this.dnsNames = new[]
{
new System.Uri(BASE_ADDRESS).Host,
new System.Uri(uploadSession.UploadUrl).Host,
}
dnsNames = new[] {
new System.Uri(BASE_ADDRESS).Host,
string.IsNullOrWhiteSpace(uploadSession.UploadUrl) ? null : new System.Uri(uploadSession.UploadUrl).Host,
}
.WhereNotNullOrWhiteSpace()
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray();
}
return this.dnsNames;
return dnsNames;
}
public async Task<IQuotaInfo> GetQuotaInfoAsync(CancellationToken cancelToken)
public async Task<IQuotaInfo?> GetQuotaInfoAsync(CancellationToken cancelToken)
{
Drive driveInfo = await this.GetAsync<Drive>(this.DrivePrefix, cancelToken).ConfigureAwait(false);
if (driveInfo.Quota != null)
var drivePrefix = await GetDrivePrefix(cancelToken).ConfigureAwait(false);
var driveInfo = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken,
ct => GetAsync<Drive>(drivePrefix, ct)
).ConfigureAwait(false);
if (driveInfo?.Quota != null)
{
// Some sources (SharePoint for example) seem to return 0 for these values even when the quota isn't exceeded..
// As a special test, if all the returned values are 0, we pretend that no quota was reported.
// This way we don't send spurious warnings because the quota looks like it is exceeded.
if (driveInfo.Quota.Total != 0 || driveInfo.Quota.Remaining != 0 || driveInfo.Quota.Used != 0)
{
return new QuotaInfo(driveInfo.Quota.Total, driveInfo.Quota.Remaining);
}
}
return null;
@@ -254,54 +237,36 @@ namespace Duplicati.Library.Backend
/// <summary>
/// Override-able fragment indicating the API version to use each query
/// </summary>
protected virtual string ApiVersion
{
get { return "/v1.0"; }
}
protected virtual string ApiVersion => "/v1.0";
/// <summary>
/// Normalized fragment (starting with a slash and ending without one) for the path to the drive to be used.
/// For example: "/me/drive" for the default drive for a user.
/// </summary>
protected abstract string DrivePath { get; }
protected abstract Task<string> GetDrivePath(CancellationToken cancelToken);
protected abstract DescriptionTemplateDelegate DescriptionTemplate { get; }
protected virtual IList<ICommandLineArgument> AdditionalSupportedCommands
{
get
{
return new ICommandLineArgument[0];
}
}
protected virtual IList<ICommandLineArgument> AdditionalSupportedCommands => [];
private string DrivePrefix
{
get
{
if (this.m_client != null)
{
return this.ApiVersion + this.DrivePath;
}
else
{
// When not using the HttpClient for requests, the base address needs to be included in this prefix
return BASE_ADDRESS + this.ApiVersion + this.DrivePath;
}
}
}
private async Task<string> GetDrivePrefix(CancellationToken cancelToken)
=> $"{ApiVersion}{await GetDrivePath(cancelToken).ConfigureAwait(false)}";
public async Task CreateFolderAsync(CancellationToken cancelToken)
{
string parentFolder = "root";
string parentFolderPath = string.Empty;
var parentFolder = "root";
var parentFolderPath = string.Empty;
var drivePrefix = await GetDrivePrefix(cancelToken).ConfigureAwait(false);
foreach (string folder in this.RootPath.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries))
{
string nextPath = parentFolderPath + "/" + folder;
DriveItem folderItem;
try
{
folderItem = await this.GetAsync<DriveItem>(string.Format("{0}/root:{1}", this.DrivePrefix, NormalizeSlashes(nextPath)), cancelToken).ConfigureAwait(false);
folderItem = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken,
ct => GetAsync<DriveItem>(string.Format("{0}/root:{1}", drivePrefix, NormalizeSlashes(nextPath)), ct)
).ConfigureAwait(false);
}
catch (DriveItemNotFoundException)
{
@@ -311,7 +276,9 @@ namespace Duplicati.Library.Backend
Folder = new FolderFacet(),
};
folderItem = await this.PostAsync(string.Format("{0}/items/{1}/children", this.DrivePrefix, parentFolder), newFolder, cancelToken).ConfigureAwait(false);
folderItem = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken,
ct => PostAsync(string.Format("{0}/items/{1}/children", drivePrefix, parentFolder), newFolder, ct)
).ConfigureAwait(false);
}
parentFolder = folderItem.Id;
@@ -321,9 +288,9 @@ namespace Duplicati.Library.Backend
/// <inheritdoc />
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancelToken)
{
await foreach (var item in this.Enumerate<DriveItem>(string.Format("{0}/root:{1}:/children", this.DrivePrefix, this.RootPath), cancelToken).ConfigureAwait(false))
var drivePrefix = await GetDrivePrefix(cancelToken).ConfigureAwait(false);
await foreach (var item in this.Enumerate<DriveItem>(string.Format("{0}/root:{1}:/children", drivePrefix, this.RootPath), cancelToken).ConfigureAwait(false))
{
// Exclude non-files and deleted items (not sure if they show up in this listing, but make sure anyway)
if (item.IsFile && !item.IsDeleted)
@@ -347,13 +314,18 @@ namespace Duplicati.Library.Backend
{
try
{
var drivePrefix = await GetDrivePrefix(cancelToken).ConfigureAwait(false);
m_retryAfter.WaitForRetryAfter();
string getUrl = string.Format("{0}/root:{1}{2}:/content", this.DrivePrefix, this.RootPath, NormalizeSlashes(remotename));
using (var response = await this.m_client.GetAsync(getUrl, HttpCompletionOption.ResponseHeadersRead, cancelToken).ConfigureAwait(false))
string getUrl = string.Format("{0}/root:{1}{2}:/content", drivePrefix, this.RootPath, NormalizeSlashes(remotename));
using (var response = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken,
ct => m_client.GetAsync(getUrl, HttpCompletionOption.ResponseHeadersRead, ct)
).ConfigureAwait(false))
{
this.CheckResponse(response);
using (var responseStream = await response.Content.ReadAsStreamAsync(cancelToken).ConfigureAwait(false))
await Library.Utility.Utility.CopyStreamAsync(responseStream, stream, cancelToken).ConfigureAwait(false);
CheckResponse(response);
using (var responseStream = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken,
ct => response.Content.ReadAsStreamAsync(ct)).ConfigureAwait(false))
using (var timeoutSream = responseStream.ObserveReadTimeout(m_timeouts.ReadWriteTimeout))
await Utility.Utility.CopyStreamAsync(responseStream, stream, cancelToken).ConfigureAwait(false);
}
}
catch (DriveItemNotFoundException ex)
@@ -367,7 +339,10 @@ namespace Duplicati.Library.Backend
{
try
{
await this.PatchAsync(string.Format("{0}/root:{1}{2}", this.DrivePrefix, this.RootPath, NormalizeSlashes(oldname)), new DriveItem() { Name = newname }, cancelToken).ConfigureAwait(false);
var drivePrefix = await GetDrivePrefix(cancelToken).ConfigureAwait(false);
await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken,
ct => PatchAsync(string.Format("{0}/root:{1}{2}", drivePrefix, this.RootPath, NormalizeSlashes(oldname)), new DriveItem() { Name = newname }, ct)
).ConfigureAwait(false);
}
catch (DriveItemNotFoundException ex)
{
@@ -379,19 +354,20 @@ namespace Duplicati.Library.Backend
public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken)
{
using (FileStream fileStream = File.OpenRead(filename))
{
await PutAsync(remotename, fileStream, cancelToken).ConfigureAwait(false);
}
}
public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken)
{
var drivePrefix = await GetDrivePrefix(cancelToken).ConfigureAwait(false);
// PUT only supports up to 4 MB file uploads. There's a separate process for larger files.
if (stream.Length < PUT_MAX_SIZE)
{
await m_retryAfter.WaitForRetryAfterAsync(cancelToken).ConfigureAwait(false);
string putUrl = string.Format("{0}/root:{1}{2}:/content", this.DrivePrefix, this.RootPath, NormalizeSlashes(remotename));
using (StreamContent streamContent = new StreamContent(stream))
string putUrl = string.Format("{0}/root:{1}{2}:/content", drivePrefix, this.RootPath, NormalizeSlashes(remotename));
using (var timeoutStream = stream.ObserveReadTimeout(m_timeouts.ReadWriteTimeout, false))
using (var streamContent = new StreamContent(timeoutStream))
{
streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
using (var response = await this.m_client.PutAsync(putUrl, streamContent, cancelToken).ConfigureAwait(false))
@@ -408,10 +384,10 @@ namespace Duplicati.Library.Backend
// The documentation seems somewhat contradictory - it states that uploads must be done sequentially,
// but also states that the nextExpectedRanges value returned may indicate multiple ranges...
// For now, this plays it safe and does a sequential upload.
string createSessionUrl = string.Format("{0}/root:{1}{2}:/createUploadSession", this.DrivePrefix, this.RootPath, NormalizeSlashes(remotename));
string createSessionUrl = string.Format("{0}/root:{1}{2}:/createUploadSession", drivePrefix, this.RootPath, NormalizeSlashes(remotename));
await m_retryAfter.WaitForRetryAfterAsync(cancelToken).ConfigureAwait(false);
using (HttpRequestMessage createSessionRequest = new HttpRequestMessage(HttpMethod.Post, createSessionUrl))
using (HttpResponseMessage createSessionResponse = await this.m_client.SendAsync(createSessionRequest, cancelToken).ConfigureAwait(false))
using (var createSessionRequest = new HttpRequestMessage(HttpMethod.Post, createSessionUrl))
using (var createSessionResponse = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => this.m_client.SendAsync(createSessionRequest, ct)).ConfigureAwait(false))
{
var uploadSession = await this.ParseResponseAsync<UploadSession>(createSessionResponse, cancelToken).ConfigureAwait(false);
@@ -428,7 +404,7 @@ namespace Duplicati.Library.Backend
currentBufferSize = stream.Length - offset;
}
using (Stream subStream = new ReadLimitLengthStream(stream, offset, currentBufferSize))
using (var subStream = new ReadLimitLengthStream(stream, offset, currentBufferSize))
{
read = subStream.Length;
@@ -447,8 +423,9 @@ namespace Duplicati.Library.Backend
fragmentCount,
remotename);
using (HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Put, uploadSession.UploadUrl))
using (StreamContent fragmentContent = new StreamContent(subStream))
using (var request = new HttpRequestMessage(HttpMethod.Put, uploadSession.UploadUrl))
using (var timeoutStream = subStream.ObserveReadTimeout(m_timeouts.ReadWriteTimeout, false))
using (var fragmentContent = new StreamContent(timeoutStream))
{
fragmentContent.Headers.ContentLength = read;
fragmentContent.Headers.ContentRange = new ContentRangeHeaderValue(offset, offset + read - 1, stream.Length);
@@ -458,7 +435,7 @@ namespace Duplicati.Library.Backend
try
{
// The uploaded put requests will error if they are authenticated
using (HttpResponseMessage response = await this.m_client.SendAsync(request, false, cancelToken).ConfigureAwait(false))
using (var response = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => this.m_client.SendAsync(request, false, ct)).ConfigureAwait(false))
{
// Note: On the last request, the json result includes the default properties of the item that was uploaded
await this.ParseResponseAsync<UploadSession>(response, cancelToken).ConfigureAwait(false);
@@ -585,12 +562,11 @@ namespace Duplicati.Library.Backend
{
try
{
var drivePrefix = await GetDrivePrefix(cancelToken).ConfigureAwait(false);
await m_retryAfter.WaitForRetryAfterAsync(cancelToken).ConfigureAwait(false);
string deleteUrl = string.Format("{0}/root:{1}{2}", this.DrivePrefix, this.RootPath, NormalizeSlashes(remotename));
using (var response = await this.m_client.DeleteAsync(deleteUrl, cancelToken).ConfigureAwait(false))
{
string deleteUrl = string.Format("{0}/root:{1}{2}", drivePrefix, this.RootPath, NormalizeSlashes(remotename));
using (var response = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => this.m_client.DeleteAsync(deleteUrl, ct)).ConfigureAwait(false))
this.CheckResponse(response);
}
}
catch (DriveItemNotFoundException ex)
{
@@ -603,7 +579,8 @@ namespace Duplicati.Library.Backend
{
try
{
string rootPath = string.Format("{0}/root:{1}", this.DrivePrefix, this.RootPath);
var drivePrefix = await GetDrivePrefix(cancelToken).ConfigureAwait(false);
string rootPath = string.Format("{0}/root:{1}", drivePrefix, this.RootPath);
await this.GetAsync<DriveItem>(rootPath, cancelToken).ConfigureAwait(false);
}
catch (DriveItemNotFoundException ex)
@@ -672,13 +649,13 @@ namespace Duplicati.Library.Backend
private async IAsyncEnumerable<T> Enumerate<T>(string url, [EnumeratorCancellation] CancellationToken cancelToken)
{
string nextUrl = url;
var nextUrl = url;
while (!string.IsNullOrEmpty(nextUrl))
{
GraphCollection<T> results;
try
{
results = await this.GetAsync<GraphCollection<T>>(nextUrl, cancelToken).ConfigureAwait(false);
results = await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, ct => this.GetAsync<GraphCollection<T>>(nextUrl, ct)).ConfigureAwait(false);
}
catch (DriveItemNotFoundException ex)
{
@@ -686,10 +663,8 @@ namespace Duplicati.Library.Backend
throw new FolderMissingException(ex);
}
foreach (T result in results.Value)
{
foreach (T result in results.Value ?? [])
yield return result;
}
nextUrl = results.ODataNextLink;
}
@@ -714,49 +689,15 @@ namespace Duplicati.Library.Backend
}
}
private void CheckResponse(HttpWebResponse response)
{
string retryAfterHeader = response.Headers[HttpResponseHeader.RetryAfter];
if (retryAfterHeader != null && RetryConditionHeaderValue.TryParse(retryAfterHeader, out RetryConditionHeaderValue retryAfter))
{
m_retryAfter.SetRetryAfter(retryAfter);
}
if (!((int)response.StatusCode >= 200 && (int)response.StatusCode < 300))
{
if (response.StatusCode == HttpStatusCode.NotFound)
{
// It looks like this is an 'item not found' exception, so wrap it in a new exception class to make it easier to pick things out.
throw new DriveItemNotFoundException(response);
}
else
{
// Throw a wrapper exception to make it easier for the caller to look at specific status codes, etc.
throw new MicrosoftGraphException(response);
}
}
}
private async Task<T> ParseResponseAsync<T>(HttpResponseMessage response, CancellationToken cancelToken)
{
this.CheckResponse(response);
using (var responseStream = await response.Content.ReadAsStreamAsync(cancelToken).ConfigureAwait(false))
using (var reader = new StreamReader(responseStream))
CheckResponse(response);
using (var responseStream = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => response.Content.ReadAsStreamAsync(ct)).ConfigureAwait(false))
using (var timeoutStream = responseStream.ObserveReadTimeout(m_timeouts.ReadWriteTimeout))
using (var reader = new StreamReader(timeoutStream))
using (var jsonReader = new JsonTextReader(reader))
{
return this.m_serializer.Deserialize<T>(jsonReader);
}
}
private Task<T> ParseResponseAsync<T>(HttpWebResponse response, CancellationToken cancelToken)
{
this.CheckResponse(response);
using (Stream responseStream = response.GetResponseStream())
using (StreamReader reader = new StreamReader(responseStream))
using (JsonTextReader jsonReader = new JsonTextReader(reader))
{
return Task.FromResult(this.m_serializer.Deserialize<T>(jsonReader));
}
return m_serializer.Deserialize<T>(jsonReader)
?? throw new MicrosoftGraphException("Failed to parse response", response);
}
private async Task ThrowUploadSessionException(
@@ -801,16 +742,11 @@ namespace Duplicati.Library.Backend
return url;
}
private StringContent PrepareContent<T>(T body)
private StringContent? PrepareContent<T>(T body)
{
if (body != null)
{
return new StringContent(JsonConvert.SerializeObject(body), Encoding.UTF8, "application/json");
}
else
{
if (body == null)
return null;
}
return new StringContent(JsonConvert.SerializeObject(body), Encoding.UTF8, "application/json");
}
}
}
@@ -18,9 +18,6 @@
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Linq;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
@@ -34,10 +31,10 @@ namespace Duplicati.Library.Backend.MicrosoftGraph
public class Identity
{
[JsonProperty("id", NullValueHandling = NullValueHandling.Ignore)]
public string Id { get; set; }
public string? Id { get; set; }
[JsonProperty("displayName", NullValueHandling = NullValueHandling.Ignore)]
public string DisplayName { get; set; }
public string? DisplayName { get; set; }
}
public class IdentitySet
@@ -46,37 +43,37 @@ namespace Duplicati.Library.Backend.MicrosoftGraph
/// The optional application associated with this action
/// </summary>
[JsonProperty("application", NullValueHandling = NullValueHandling.Ignore)]
public Identity Application { get; set; }
public Identity? Application { get; set; }
/// <summary>
/// The optional device associated with this action
/// </summary>
[JsonProperty("device", NullValueHandling = NullValueHandling.Ignore)]
public Identity Device { get; set; }
public Identity? Device { get; set; }
/// <summary>
/// The optional user associated with this action
/// </summary>
[JsonProperty("user", NullValueHandling = NullValueHandling.Ignore)]
public Identity User { get; set; }
public Identity? User { get; set; }
}
public class BaseItem
{
[JsonProperty("id", NullValueHandling = NullValueHandling.Ignore)]
public string Id { get; set; }
public string? Id { get; set; }
[JsonProperty("createdBy", NullValueHandling = NullValueHandling.Ignore)]
public IdentitySet CreatedBy { get; set; }
public IdentitySet? CreatedBy { get; set; }
[JsonProperty("createdDateTime", NullValueHandling = NullValueHandling.Ignore)]
public DateTimeOffset? CreatedDateTime { get; set; }
[JsonProperty("description", NullValueHandling = NullValueHandling.Ignore)]
public string Description { get; set; }
public string? Description { get; set; }
[JsonProperty("lastModifiedBy", NullValueHandling = NullValueHandling.Ignore)]
public IdentitySet LastModifiedBy { get; set; }
public IdentitySet? LastModifiedBy { get; set; }
[JsonProperty("lastModifiedDateTime", NullValueHandling = NullValueHandling.Ignore)]
public DateTimeOffset? LastModifiedDateTime { get; set; }
@@ -90,10 +87,10 @@ namespace Duplicati.Library.Backend.MicrosoftGraph
/// https://support.microsoft.com/en-us/help/2829981/support-for-whitespace-characters-in-file-and-folder-names-for-windows
/// </summary>
[JsonProperty("name", NullValueHandling = NullValueHandling.Ignore)]
public string Name { get; set; }
public string? Name { get; set; }
[JsonProperty("webUrl", NullValueHandling = NullValueHandling.Ignore)]
public string WebUrl { get; set; }
public string? WebUrl { get; set; }
}
public enum QuotaState
@@ -176,10 +173,10 @@ namespace Duplicati.Library.Backend.MicrosoftGraph
public DriveType DriveType { get; set; }
[JsonProperty("owner", NullValueHandling = NullValueHandling.Ignore)]
public IdentitySet Owner { get; set; }
public IdentitySet? Owner { get; set; }
[JsonProperty("quota", NullValueHandling = NullValueHandling.Ignore)]
public Quota Quota { get; set; }
public Quota? Quota { get; set; }
}
public class HashesType
@@ -189,30 +186,30 @@ namespace Duplicati.Library.Backend.MicrosoftGraph
/// Available only in OneDrive personal
/// </summary>
[JsonProperty("sha1Hash", NullValueHandling = NullValueHandling.Ignore)]
public string Sha1Hash { get; set; }
public string? Sha1Hash { get; set; }
/// <summary>
/// CRC32 hash
/// Available only in OneDrive personal
/// </summary>
[JsonProperty("crc32Hash", NullValueHandling = NullValueHandling.Ignore)]
public string Crc32Hash { get; set; }
public string? Crc32Hash { get; set; }
/// <summary>
/// A proprietary hash of the file that can be used to determine if the file's contents have changed.
/// Available on in OneDrive for Business and SharePoint Server 2016.
/// </summary>
[JsonProperty("quickXorHash", NullValueHandling = NullValueHandling.Ignore)]
public string QuickXorHash { get; set; }
public string? QuickXorHash { get; set; }
}
public class FileFacet
{
[JsonProperty("hashes", NullValueHandling = NullValueHandling.Ignore)]
public HashesType Hashes { get; set; }
public HashesType? Hashes { get; set; }
[JsonProperty("mimeType", NullValueHandling = NullValueHandling.Ignore)]
public string MimeType { get; set; }
public string? MimeType { get; set; }
}
public enum SortBy
@@ -307,7 +304,7 @@ namespace Duplicati.Library.Backend.MicrosoftGraph
public long? ChildCount { get; set; }
[JsonProperty("view", NullValueHandling = NullValueHandling.Ignore)]
public FolderView View { get; set; }
public FolderView? View { get; set; }
}
/// <summary>
@@ -337,19 +334,19 @@ namespace Duplicati.Library.Backend.MicrosoftGraph
/// Package type - currently only oneNote is defined, but others could be used.
/// </summary>
[JsonProperty("type", NullValueHandling = NullValueHandling.Ignore)]
public string Type { get; set; }
public string? Type { get; set; }
}
public class DeletedFacet
{
[JsonProperty("state", NullValueHandling = NullValueHandling.Ignore)]
public string State { get; set; }
public string? State { get; set; }
}
public class RemoteItemFacet
{
[JsonProperty("remoteItem", NullValueHandling = NullValueHandling.Ignore)]
public RemoteItemFacet RemoteItem { get; set; }
public RemoteItemFacet? RemoteItem { get; set; }
}
public class RootFacet
@@ -362,7 +359,7 @@ namespace Duplicati.Library.Backend.MicrosoftGraph
/// The unique identifier which can be used under /drive/special/
/// </summary>
[JsonProperty("name", NullValueHandling = NullValueHandling.Ignore)]
public string Name { get; set; }
public string? Name { get; set; }
}
/// <summary>
@@ -372,19 +369,19 @@ namespace Duplicati.Library.Backend.MicrosoftGraph
public class ItemReference
{
[JsonProperty("driveId", NullValueHandling = NullValueHandling.Ignore)]
public string DriveId { get; set; }
public string? DriveId { get; set; }
[JsonProperty("driveType", NullValueHandling = NullValueHandling.Ignore)]
public DriveType DriveType { get; set; }
[JsonProperty("id", NullValueHandling = NullValueHandling.Ignore)]
public string Id { get; set; }
public string? Id { get; set; }
[JsonProperty("name", NullValueHandling = NullValueHandling.Ignore)]
public string Name { get; set; }
public string? Name { get; set; }
[JsonProperty("path", NullValueHandling = NullValueHandling.Ignore)]
public string Path { get; set; }
public string? Path { get; set; }
}
public enum ConflictBehavior
@@ -403,41 +400,41 @@ namespace Duplicati.Library.Backend.MicrosoftGraph
public class DriveItem : BaseItem
{
[JsonProperty("parentReference", NullValueHandling = NullValueHandling.Ignore)]
public ItemReference ParentReference { get; set; }
public ItemReference? ParentReference { get; set; }
[JsonProperty("size", NullValueHandling = NullValueHandling.Ignore)]
public long? Size { get; set; }
[JsonProperty("file", NullValueHandling = NullValueHandling.Ignore)]
public FileFacet File { get; set; }
public FileFacet? File { get; set; }
[JsonProperty("folder", NullValueHandling = NullValueHandling.Ignore)]
public FolderFacet Folder { get; set; }
public FolderFacet? Folder { get; set; }
[JsonProperty("fileSystemInfo", NullValueHandling = NullValueHandling.Ignore)]
public FileSystemInfoFacet FileSystemInfo { get; set; }
public FileSystemInfoFacet? FileSystemInfo { get; set; }
[JsonProperty("package", NullValueHandling = NullValueHandling.Ignore)]
public PackageFacet Package { get; set; }
public PackageFacet? Package { get; set; }
[JsonProperty("deleted", NullValueHandling = NullValueHandling.Ignore)]
public DeletedFacet Deleted { get; set; }
public DeletedFacet? Deleted { get; set; }
[JsonProperty("remoteItem", NullValueHandling = NullValueHandling.Ignore)]
public RemoteItemFacet RemoteItem { get; set; }
public RemoteItemFacet? RemoteItem { get; set; }
[JsonProperty("root", NullValueHandling = NullValueHandling.Ignore)]
public RootFacet Root { get; set; }
public RootFacet? Root { get; set; }
[JsonProperty("specialFolder", NullValueHandling = NullValueHandling.Ignore)]
public SpecialFolderFacet SpecialFolder { get; set; }
public SpecialFolderFacet? SpecialFolder { get; set; }
[JsonProperty("@microsoft.graph.conflictBehavior", NullValueHandling = NullValueHandling.Ignore)]
[JsonConverter(typeof(StringEnumConverter))]
public ConflictBehavior? ConflictBehavior { get; set; }
[JsonProperty("@microsoft.graph.downloadUrl", NullValueHandling = NullValueHandling.Ignore)]
public string DownloadUrl { get; set; }
public string? DownloadUrl { get; set; }
[JsonIgnore]
public bool IsFile
@@ -515,25 +512,25 @@ namespace Duplicati.Library.Backend.MicrosoftGraph
public class GraphCollection<T>
{
[JsonProperty("value", NullValueHandling = NullValueHandling.Ignore)]
public T[] Value { get; set; }
public T[]? Value { get; set; }
[JsonProperty("@odata.nextLink", NullValueHandling = NullValueHandling.Ignore)]
public string ODataNextLink { get; set; }
public string? ODataNextLink { get; set; }
}
public class UploadSession
{
[JsonProperty("uploadUrl", NullValueHandling = NullValueHandling.Ignore)]
public string UploadUrl { get; set; }
public string? UploadUrl { get; set; }
[JsonProperty("expirationDateTime", NullValueHandling = NullValueHandling.Ignore)]
public DateTimeOffset? ExpirationDateTime { get; set; }
[JsonProperty("nextExpectedRanges", NullValueHandling = NullValueHandling.Ignore)]
public string[] NextExpectedRanges { get; set; }
public string[]? NextExpectedRanges { get; set; }
[JsonProperty("item", NullValueHandling = NullValueHandling.Ignore)]
public DriveItem Item { get; set; }
public DriveItem? Item { get; set; }
}
public class SharePointSite : BaseItem
@@ -543,16 +540,16 @@ namespace Duplicati.Library.Backend.MicrosoftGraph
public class Group : BaseItem
{
[JsonProperty("mail", NullValueHandling = NullValueHandling.Ignore)]
public string Mail { get; set; }
public string? Mail { get; set; }
[JsonProperty("proxyAddresses", NullValueHandling = NullValueHandling.Ignore)]
public string[] ProxyAddresses { get; set; }
public string[]? ProxyAddresses { get; set; }
[JsonProperty("mailNickname", NullValueHandling = NullValueHandling.Ignore)]
public string MailNickname { get; set; }
public string? MailNickname { get; set; }
[JsonProperty("groupTypes", NullValueHandling = NullValueHandling.Ignore)]
public string[] GroupTypes { get; set; }
public string[]? GroupTypes { get; set; }
[JsonIgnore]
public bool IsUnifiedGroup
@@ -18,13 +18,9 @@
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Duplicati.Library.Backend.MicrosoftGraph;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
namespace Duplicati.Library.Backend
{
@@ -34,93 +30,76 @@ namespace Duplicati.Library.Backend
private const string GROUP_ID_OPTION = "group-id";
private const string PROTOCOL_KEY = "msgroup";
private readonly string drivePath;
private readonly Dictionary<string, string?> options;
private string? drivePath;
public MicrosoftGroup() { } // Constructor needed for dynamic loading to find it
public MicrosoftGroup(string url, Dictionary<string, string> options)
: base(url, MicrosoftGroup.PROTOCOL_KEY, options)
public MicrosoftGroup()
{
string groupId = null;
string groupEmail;
if (options.TryGetValue(GROUP_EMAIL_OPTION, out groupEmail))
{
// TODO: Should not make network requests in constructor
groupId = this.GetGroupIdFromEmailAsync(groupEmail, CancellationToken.None).Await();
}
// Constructor needed for dynamic loading to find it
options = null!;
}
string groupIdOption;
if (options.TryGetValue(GROUP_ID_OPTION, out groupIdOption))
public MicrosoftGroup(string url, Dictionary<string, string?> options)
: base(url, PROTOCOL_KEY, options)
{
this.options = options;
}
public override string ProtocolKey => PROTOCOL_KEY;
public override string DisplayName => Strings.MicrosoftGroup.DisplayName;
protected override async Task<string> GetDrivePath(CancellationToken cancelToken)
{
if (string.IsNullOrEmpty(drivePath))
{
if (!string.IsNullOrEmpty(groupId) && !string.Equals(groupId, groupIdOption))
string? groupId = null;
var groupEmail = options.GetValueOrDefault(GROUP_EMAIL_OPTION);
if (!string.IsNullOrWhiteSpace(groupEmail))
groupId = await this.GetGroupIdFromEmailAsync(groupEmail, cancelToken).ConfigureAwait(false);
var groupIdOption = options.GetValueOrDefault(GROUP_ID_OPTION);
if (!string.IsNullOrWhiteSpace(groupIdOption))
{
throw new UserInformationException(Strings.MicrosoftGroup.ConflictingGroupId(groupIdOption, groupId), "MicrosoftGroupConflictingGroupId");
if (!string.IsNullOrEmpty(groupId) && !string.Equals(groupId, groupIdOption))
throw new UserInformationException(Strings.MicrosoftGroup.ConflictingGroupId(groupIdOption, groupId), "MicrosoftGroupConflictingGroupId");
groupId = groupIdOption;
}
groupId = groupIdOption;
if (string.IsNullOrWhiteSpace(groupId))
throw new UserInformationException(Strings.MicrosoftGroup.MissingGroupIdAndEmailAddress, "MicrosoftGroupMissingGroupIdAndEmailAddress");
drivePath = string.Format("/groups/{0}/drive", groupId);
}
if (string.IsNullOrEmpty(groupId))
{
throw new UserInformationException(Strings.MicrosoftGroup.MissingGroupIdAndEmailAddress, "MicrosoftGroupMissingGroupIdAndEmailAddress");
}
this.drivePath = string.Format("/groups/{0}/drive", groupId);
return drivePath;
}
public override string ProtocolKey
{
get { return MicrosoftGroup.PROTOCOL_KEY; }
}
protected override DescriptionTemplateDelegate DescriptionTemplate => Strings.MicrosoftGroup.Description;
public override string DisplayName
{
get { return Strings.MicrosoftGroup.DisplayName; }
}
protected override string DrivePath
{
get { return this.drivePath; }
}
protected override DescriptionTemplateDelegate DescriptionTemplate
{
get
{
return Strings.MicrosoftGroup.Description;
}
}
protected override IList<ICommandLineArgument> AdditionalSupportedCommands
{
get
{
return
[
protected override IList<ICommandLineArgument> AdditionalSupportedCommands => [
new CommandLineArgument(GROUP_ID_OPTION, CommandLineArgument.ArgumentType.String, Strings.MicrosoftGroup.GroupIdShort, Strings.MicrosoftGroup.GroupIdLong),
new CommandLineArgument(GROUP_EMAIL_OPTION, CommandLineArgument.ArgumentType.String, Strings.MicrosoftGroup.GroupEmailShort, Strings.MicrosoftGroup.GroupEmailLong),
];
}
}
private async Task<string> GetGroupIdFromEmailAsync(string email, CancellationToken cancelToken)
{
// We can get all groups that have the given email as one of their addresses with:
// https://graph.microsoft.com/v1.0/groups?$filter=mail eq '{email}' or proxyAddresses/any(x:x eq 'smtp:{email}')
string request = string.Format("{0}/groups?$filter=mail eq '{1}' or proxyAddresses/any(x:x eq 'smtp:{1}')", this.ApiVersion, email);
var groups = await this.GetAsync<GraphCollection<Group>>(request, cancelToken).ConfigureAwait(false);
if (groups.Value.Length == 0)
{
var groups = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => GetAsync<GraphCollection<Group>>(request, ct)).ConfigureAwait(false);
if (groups.Value == null || groups.Value.Length == 0)
throw new UserInformationException(Strings.MicrosoftGroup.NoGroupsWithEmail(email), "MicrosoftGroupNoGroupsWithEmail");
}
else if (groups.Value.Length > 1)
{
if (groups.Value.Length > 1)
throw new UserInformationException(Strings.MicrosoftGroup.MultipleGroupsWithEmail(email), "MicrosoftGroupMultipleGroupsWithEmail");
}
else
{
return groups.Value.Single().Id;
}
var id = groups.Value.Single().Id;
if (string.IsNullOrEmpty(id))
throw new UserInformationException(Strings.MicrosoftGroup.NoGroupsWithEmail(email), "MicrosoftGroupNoGroupsWithEmail");
return id;
}
}
}
+18 -42
View File
@@ -18,7 +18,6 @@
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System.Collections.Generic;
using Duplicati.Library.Interface;
@@ -30,56 +29,33 @@ namespace Duplicati.Library.Backend
private const string DEFAULT_DRIVE_PATH = "/me/drive";
private const string PROTOCOL_KEY = "onedrivev2";
private readonly string drivePath;
private readonly Task<string> drivePath;
public OneDrive() { } // Constructor needed for dynamic loading to find it
public OneDrive(string url, Dictionary<string, string> options)
: base(url, OneDrive.PROTOCOL_KEY, options)
public OneDrive()
{
string driveId;
if (options.TryGetValue(DRIVE_ID_OPTION, out driveId))
{
this.drivePath = string.Format("/drives/{0}", driveId);
}
else
{
this.drivePath = DEFAULT_DRIVE_PATH;
}
// Constructor needed for dynamic loading to find it
drivePath = null!;
}
public override string ProtocolKey
public OneDrive(string url, Dictionary<string, string?> options)
: base(url, PROTOCOL_KEY, options)
{
get { return OneDrive.PROTOCOL_KEY; }
var driveId = options.GetValueOrDefault(DRIVE_ID_OPTION);
drivePath = Task.FromResult(string.IsNullOrWhiteSpace(driveId)
? DEFAULT_DRIVE_PATH
: string.Format("/drives/{0}", driveId));
}
public override string DisplayName
{
get { return Strings.OneDrive.DisplayName; }
}
public override string ProtocolKey => PROTOCOL_KEY;
protected override string DrivePath
{
get { return this.drivePath; }
}
public override string DisplayName => Strings.OneDrive.DisplayName;
protected override DescriptionTemplateDelegate DescriptionTemplate
{
get
{
return Strings.OneDrive.Description;
}
}
protected override Task<string> GetDrivePath(CancellationToken cancelToken) => drivePath;
protected override IList<ICommandLineArgument> AdditionalSupportedCommands
{
get
{
return new ICommandLineArgument[]
{
new CommandLineArgument(DRIVE_ID_OPTION, CommandLineArgument.ArgumentType.String, Strings.OneDrive.DriveIdShort, Strings.OneDrive.DriveIdLong(DEFAULT_DRIVE_PATH)),
};
}
}
protected override DescriptionTemplateDelegate DescriptionTemplate => Strings.OneDrive.Description;
protected override IList<ICommandLineArgument> AdditionalSupportedCommands => [
new CommandLineArgument(DRIVE_ID_OPTION, CommandLineArgument.ArgumentType.String, Strings.OneDrive.DriveIdShort, Strings.OneDrive.DriveIdLong(DEFAULT_DRIVE_PATH)),
];
}
}
@@ -18,12 +18,7 @@
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Duplicati.Library.Backend.MicrosoftGraph;
using Duplicati.Library.Interface;
@@ -34,67 +29,40 @@ namespace Duplicati.Library.Backend
private const string SITE_ID_OPTION = "site-id";
private const string PROTOCOL_KEY = "sharepoint";
private readonly string drivePath;
private string siteId = null;
private readonly Task<string> drivePath;
private string siteId;
public SharePointV2() { } // Constructor needed for dynamic loading to find it
public SharePointV2()
{
// Constructor needed for dynamic loading to find it
drivePath = null!;
siteId = null!;
}
public SharePointV2(string url, Dictionary<string, string> options)
: base(url, SharePointV2.PROTOCOL_KEY, options)
public SharePointV2(string url, Dictionary<string, string?> options)
: base(url, PROTOCOL_KEY, options)
{
// Check to see if a site ID was explicitly provided
string siteIdOption;
if (options.TryGetValue(SITE_ID_OPTION, out siteIdOption))
{
if (!string.IsNullOrEmpty(this.siteId) && !string.Equals(this.siteId, siteIdOption))
{
throw new UserInformationException(Strings.SharePointV2.ConflictingSiteId(siteIdOption, this.siteId), "SharePointConflictingSiteId");
}
this.siteId = siteIdOption;
}
if (string.IsNullOrEmpty(this.siteId))
{
var siteIdOption = options.GetValueOrDefault(SITE_ID_OPTION);
if (!string.IsNullOrWhiteSpace(siteIdOption))
siteId = siteIdOption;
else
throw new UserInformationException(Strings.SharePointV2.MissingSiteId, "SharePointMissingSiteId");
}
this.drivePath = string.Format("/sites/{0}/drive", this.siteId);
drivePath = Task.FromResult(string.Format("/sites/{0}/drive", siteId));
}
public override string ProtocolKey
{
get { return SharePointV2.PROTOCOL_KEY; }
}
public override string ProtocolKey => PROTOCOL_KEY;
public override string DisplayName
{
get { return Strings.SharePointV2.DisplayName; }
}
public override string DisplayName => Strings.SharePointV2.DisplayName;
protected override string DrivePath
{
get { return this.drivePath; }
}
protected override Task<string> GetDrivePath(CancellationToken cancelToken) => drivePath;
protected override DescriptionTemplateDelegate DescriptionTemplate
{
get
{
return Strings.SharePointV2.Description;
}
}
protected override DescriptionTemplateDelegate DescriptionTemplate => Strings.SharePointV2.Description;
protected override IList<ICommandLineArgument> AdditionalSupportedCommands
{
get
{
return new ICommandLineArgument[]
{
new CommandLineArgument(SITE_ID_OPTION, CommandLineArgument.ArgumentType.String, Strings.SharePointV2.SiteIdShort, Strings.SharePointV2.SiteIdLong),
};
}
}
protected override IList<ICommandLineArgument> AdditionalSupportedCommands => [
new CommandLineArgument(SITE_ID_OPTION, CommandLineArgument.ArgumentType.String, Strings.SharePointV2.SiteIdShort, Strings.SharePointV2.SiteIdLong),
];
/// <summary>
/// This method takes an input URL, which could be either in the format that the old SharePoint backend accepted or
@@ -107,7 +75,7 @@ namespace Duplicati.Library.Backend
/// <returns>Path within the drive</returns>
protected override async Task<string> GetRootPathFromUrlAsync(string url, CancellationToken cancelToken)
{
Uri uri = new Uri(url);
var uri = new Uri(url);
// If the user gave a URL like "https://{tenant}.sharepoint.com/path" in the UI,
// it might appear here as "sharepoint://https://{tenant}.sharepoint.com/path".
@@ -118,19 +86,22 @@ namespace Duplicati.Library.Backend
uri = new Uri(string.Format("{0}:{1}", uri.Scheme, uri.LocalPath));
}
var site = await this.GetSharePointSite(uri, cancelToken).ConfigureAwait(false);
var site = await GetSharePointSite(uri, cancelToken).ConfigureAwait(false);
if (site != null)
{
// Get the web URL of the site's main drive
try
{
var drive = await this.GetAsync<Drive>(string.Format("{0}/sites/{1}/drive", this.ApiVersion, site.Id), cancelToken).ConfigureAwait(false);
var drive = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => this.GetAsync<Drive>(string.Format("{0}/sites/{1}/drive", this.ApiVersion, site.Id), ct)).ConfigureAwait(false);
this.siteId = site.Id;
var driveWebUrl = new Uri(drive.WebUrl);
if (drive != null && !string.IsNullOrEmpty(drive.WebUrl) && !string.IsNullOrWhiteSpace(site.Id))
{
siteId = site.Id;
var driveWebUrl = new Uri(drive.WebUrl);
// Make sure to replace any "//" in the original path with "/", so the substrings line up.
return uri.LocalPath.Replace("//", "/").Substring(driveWebUrl.LocalPath.Length);
// Make sure to replace any "//" in the original path with "/", so the substrings line up.
return uri.LocalPath.Replace("//", "/").Substring(driveWebUrl.LocalPath.Length);
}
}
catch (MicrosoftGraphException)
{
@@ -141,7 +112,7 @@ namespace Duplicati.Library.Backend
return await base.GetRootPathFromUrlAsync(url, cancelToken).ConfigureAwait(false);
}
private async Task<SharePointSite> GetSharePointSite(Uri url, CancellationToken cancelToken)
private async Task<SharePointSite?> GetSharePointSite(Uri url, CancellationToken cancelToken)
{
var uri = new UriBuilder(url);
@@ -157,7 +128,7 @@ namespace Duplicati.Library.Backend
try
{
var request = string.Format("{0}:/{1}", requestBase, uri.Path.Substring(0, siteHint));
return await this.GetAsync<SharePointSite>(request, cancelToken).ConfigureAwait(false);
return await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => this.GetAsync<SharePointSite>(request, ct)).ConfigureAwait(false);
}
catch (MicrosoftGraphException)
{
@@ -171,7 +142,7 @@ namespace Duplicati.Library.Backend
try
{
string request = string.Format("{0}:/{1}", requestBase, string.Join("/", pathPieces.Take(i)));
return await this.GetAsync<SharePointSite>(request, cancelToken).ConfigureAwait(false);
return await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => this.GetAsync<SharePointSite>(request, ct)).ConfigureAwait(false);
}
catch (MicrosoftGraphException)
{
+20 -23
View File
@@ -1,31 +1,28 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Localization.Short;
namespace Duplicati.Library.Backend.Strings
{
internal static class MicrosoftGraph
{
public static string AuthIdLong(string url) { return LC.L(@"The authorization token retrieved from {0}", url); }
public static string AuthIdShort { get { return LC.L(@"The authorization code"); } }
public static string MissingAuthId(string url) { return LC.L(@"No Auth-ID was provided - you can get one from {0}", url); }
public static string FragmentSizeLong { get { return LC.L(@"Size of individual fragments which are uploaded separately for large files. It is recommended to be between 5-10 MiB (though a smaller value may work better on a slower or less reliable connection), and to be a multiple of 320 KiB."); } }
public static string FragmentSizeShort { get { return LC.L(@"Fragment size for large uploads"); } }
public static string FragmentRetryCountLong { get { return LC.L(@"Number of retry attempts made for each fragment before failing the overall file upload."); } }
@@ -52,7 +49,7 @@ namespace Duplicati.Library.Backend.Strings
public static string SiteIdLong { get { return LC.L(@"ID of the site to store data in."); } }
public static string SiteIdShort { get { return LC.L(@"ID of the site"); } }
public static string MissingSiteId { get { return LC.L(@"No site ID was provided"); } }
public static string ConflictingSiteId(string given, string found) { return LC.L(@"Conflicting site IDs used: given {0} but found {1}", given, found); }
public static string MissingDriveInfo { get { return LC.L(@"No drive information was returned"); } }
}
internal static class MicrosoftGroup
@@ -4,6 +4,8 @@
<TargetFramework>net8.0</TargetFramework>
<OutputType>Library</OutputType>
<Copyright>Copyright © 2025 Team Duplicati, MIT license</Copyright>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
@@ -1,36 +1,32 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using Duplicati.Library.Interface;
using System.Collections.Generic;
using System.Linq;
using Duplicati.Library.Localization.Short;
namespace Duplicati.Library.Backend.OpenStack
{
public class SwiftConfig : IWebModule
{
private const ConfigType DEFAULT_CONFIG_TYPE = ConfigType.Providers;
private static readonly string DEFAULT_CONFIG_TYPE_STR = Enum.GetName(typeof(ConfigType), DEFAULT_CONFIG_TYPE);
private static readonly string DEFAULT_CONFIG_TYPE_STR = DEFAULT_CONFIG_TYPE.ToString();
private const string KEY_CONFIGTYPE = "openstack-config";
public enum ConfigType
@@ -42,42 +38,32 @@ namespace Duplicati.Library.Backend.OpenStack
{
}
public System.Collections.Generic.IDictionary<string, string> Execute(System.Collections.Generic.IDictionary<string, string> options)
public IDictionary<string, string> Execute(IDictionary<string, string> options)
{
string k;
options.TryGetValue(KEY_CONFIGTYPE, out k);
options.TryGetValue(KEY_CONFIGTYPE, out var k);
if (string.IsNullOrWhiteSpace(k))
k = DEFAULT_CONFIG_TYPE_STR;
ConfigType ct;
if (!Enum.TryParse<ConfigType>(k, true, out ct))
if (!Enum.TryParse<ConfigType>(k, true, out var ct))
ct = DEFAULT_CONFIG_TYPE;
switch (ct)
{
case ConfigType.Versions:
return OpenStack.OpenStackStorage.OPENSTACK_VERSIONS.ToDictionary((x) => x.Key, (y) => y.Value);
return OpenStackStorage.OPENSTACK_VERSIONS.ToDictionary((x) => x.Key, (y) => y.Value);
default:
return OpenStack.OpenStackStorage.KNOWN_OPENSTACK_PROVIDERS.ToDictionary((x) => x.Key, (y) => y.Value);
return OpenStackStorage.KNOWN_OPENSTACK_PROVIDERS.ToDictionary((x) => x.Key, (y) => y.Value);
}
}
public string Key { get { return "openstack-getconfig"; } }
public string Key => "openstack-getconfig";
public string DisplayName { get { return LC.L("OpenStack configuration module"); } }
public string DisplayName => Strings.SwiftConfig.DisplayName;
public string Description { get { return LC.L("Expose OpenStack configuration as a web module"); } }
public string Description => Strings.SwiftConfig.Description;
public System.Collections.Generic.IList<ICommandLineArgument> SupportedCommands
{
get
{
return new List<ICommandLineArgument>([
new CommandLineArgument(KEY_CONFIGTYPE, CommandLineArgument.ArgumentType.Enumeration, LC.L("The config to get"), LC.L("Provide different config values"), DEFAULT_CONFIG_TYPE_STR, Enum.GetNames(typeof(ConfigType)))
]);
}
}
public IList<ICommandLineArgument> SupportedCommands => [
new CommandLineArgument(KEY_CONFIGTYPE, CommandLineArgument.ArgumentType.Enumeration, Strings.SwiftConfig.ConfigTypeOptionShort, Strings.SwiftConfig.ConfigTypeOptionLong, DEFAULT_CONFIG_TYPE_STR, Enum.GetNames(typeof(ConfigType)))
];
}
}
@@ -19,28 +19,22 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Collections.Generic;
using Duplicati.Library.Interface;
using System.Linq;
using System.IO;
using Duplicati.Library.Utility;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
using System.Net;
using System.Text;
using Duplicati.Library.Common.IO;
using System.Threading;
using System.Threading.Tasks;
using System.Runtime.CompilerServices;
using Duplicati.Library.Utility.Options;
using System.Threading.Tasks;
namespace Duplicati.Library.Backend.OpenStack
{
public class OpenStackStorage : IBackend, IStreamingBackend
{
private const string DOMAINNAME_OPTION = "openstack-domain-name";
private const string USERNAME_OPTION = "auth-username";
private const string PASSWORD_OPTION = "auth-password";
private const string TENANTNAME_OPTION = "openstack-tenant-name";
private const string AUTHURI_OPTION = "openstack-authuri";
private const string VERSION_OPTION = "openstack-version";
@@ -53,19 +47,20 @@ namespace Duplicati.Library.Backend.OpenStack
private readonly string m_container;
private readonly string m_prefix;
private readonly string m_domainName;
private readonly string? m_domainName;
private readonly string m_username;
private readonly string m_password;
private readonly string? m_password;
private readonly string m_authUri;
private readonly string m_version;
private readonly string m_tenantName;
private readonly string m_apikey;
private readonly string m_region;
private readonly string? m_version;
private readonly string? m_tenantName;
private readonly string? m_apikey;
private readonly string? m_region;
private readonly TimeoutOptionsHelper.Timeouts m_timeouts;
protected string m_simplestorageendpoint;
protected string? m_simplestorageendpoint;
private readonly WebHelper m_helper;
private OpenStackAuthResponse.TokenClass m_accessToken;
private OpenStackAuthResponse.TokenClass? m_accessToken;
public static readonly KeyValuePair<string, string>[] KNOWN_OPENSTACK_PROVIDERS = {
new KeyValuePair<string, string>("Rackspace US", "https://identity.api.rackspacecloud.com/v2.0"),
@@ -93,8 +88,8 @@ namespace Duplicati.Library.Backend.OpenStack
{
public class AuthContainer
{
public Identity identity { get; set; }
public Scope scope { get; set; }
public Identity? identity { get; set; }
public Scope? scope { get; set; }
}
public class Identity
@@ -103,7 +98,7 @@ namespace Duplicati.Library.Backend.OpenStack
public IdentityMethods[] methods { get; set; }
[JsonProperty("password", NullValueHandling = NullValueHandling.Ignore)]
public PasswordBasedRequest PasswordCredentials { get; set; }
public PasswordBasedRequest? PasswordCredentials { get; set; }
public Identity()
{
@@ -113,7 +108,7 @@ namespace Duplicati.Library.Backend.OpenStack
public class Scope
{
public Project project;
public Project? project;
}
public enum IdentityMethods
@@ -123,7 +118,7 @@ namespace Duplicati.Library.Backend.OpenStack
public class PasswordBasedRequest
{
public UserCredentials user { get; set; }
public UserCredentials? user { get; set; }
}
public class UserCredentials
@@ -134,6 +129,9 @@ namespace Duplicati.Library.Backend.OpenStack
public UserCredentials()
{
domain = null!;
name = null!;
password = null!;
}
public UserCredentials(Domain domain, string name, string password)
{
@@ -146,9 +144,9 @@ namespace Duplicati.Library.Backend.OpenStack
public class Domain
{
public string name { get; set; }
public string? name { get; set; }
public Domain(string name)
public Domain(string? name)
{
this.name = name;
}
@@ -168,16 +166,16 @@ namespace Duplicati.Library.Backend.OpenStack
public AuthContainer auth { get; set; }
public Keystone3AuthRequest(string domain_name, string username, string password, string project_name)
public Keystone3AuthRequest(string? domain_name, string username, string? password, string? project_name)
{
Domain domain = new Domain(domain_name);
this.auth = new AuthContainer();
this.auth.identity = new Identity();
this.auth.identity.PasswordCredentials = new PasswordBasedRequest();
this.auth.identity.PasswordCredentials.user = new UserCredentials(domain, username, password);
this.auth.identity.PasswordCredentials.user = new UserCredentials(domain, username, password!);
this.auth.scope = new Scope();
this.auth.scope.project = new Project(domain, project_name);
this.auth.scope.project = new Project(domain, project_name!);
}
}
@@ -186,41 +184,41 @@ namespace Duplicati.Library.Backend.OpenStack
public class AuthContainer
{
[JsonProperty("RAX-KSKEY:apiKeyCredentials", NullValueHandling = NullValueHandling.Ignore)]
public ApiKeyBasedRequest ApiCredentials { get; set; }
public ApiKeyBasedRequest? ApiCredentials { get; set; }
[JsonProperty("passwordCredentials", NullValueHandling = NullValueHandling.Ignore)]
public PasswordBasedRequest PasswordCredentials { get; set; }
public PasswordBasedRequest? PasswordCredentials { get; set; }
[JsonProperty("tenantName", NullValueHandling = NullValueHandling.Ignore)]
public string TenantName { get; set; }
public string? TenantName { get; set; }
[JsonProperty("token", NullValueHandling = NullValueHandling.Ignore)]
public TokenBasedRequest Token { get; set; }
public TokenBasedRequest? Token { get; set; }
}
public class ApiKeyBasedRequest
{
public string username { get; set; }
public string apiKey { get; set; }
public string? username { get; set; }
public string? apiKey { get; set; }
}
public class PasswordBasedRequest
{
public string username { get; set; }
public string password { get; set; }
public string tenantName { get; set; }
public string? username { get; set; }
public string? password { get; set; }
public string? tenantName { get; set; }
}
public class TokenBasedRequest
{
public string id { get; set; }
public string? id { get; set; }
}
public AuthContainer auth { get; set; }
public OpenStackAuthRequest(string tenantname, string username, string password, string apikey)
public OpenStackAuthRequest(string? tenantname, string username, string? password, string? apikey)
{
this.auth = new AuthContainer();
this.auth.TenantName = tenantname;
@@ -247,70 +245,70 @@ namespace Duplicati.Library.Backend.OpenStack
private class Keystone3AuthResponse
{
public TokenClass token { get; set; }
public TokenClass? token { get; set; }
public class EndpointItem
{
// 'interface' is a reserved keyword, so we need this decorator to map it
[JsonProperty(PropertyName = "interface")]
public string interface_name { get; set; }
public string region { get; set; }
public string url { get; set; }
public string? interface_name { get; set; }
public string? region { get; set; }
public string? url { get; set; }
}
public class CatalogItem
{
public EndpointItem[] endpoints { get; set; }
public string name { get; set; }
public string type { get; set; }
public EndpointItem[]? endpoints { get; set; }
public string? name { get; set; }
public string? type { get; set; }
}
public class TokenClass
{
public CatalogItem[] catalog { get; set; }
public CatalogItem[]? catalog { get; set; }
public DateTime? expires_at { get; set; }
}
}
private class OpenStackAuthResponse
{
public AccessClass access { get; set; }
public AccessClass? access { get; set; }
public class TokenClass
{
public string id { get; set; }
public string? id { get; set; }
public DateTime? expires { get; set; }
}
public class EndpointItem
{
public string region { get; set; }
public string tenantId { get; set; }
public string publicURL { get; set; }
public string internalURL { get; set; }
public string? region { get; set; }
public string? tenantId { get; set; }
public string? publicURL { get; set; }
public string? internalURL { get; set; }
}
public class ServiceItem
{
public string name { get; set; }
public string type { get; set; }
public EndpointItem[] endpoints { get; set; }
public string? name { get; set; }
public string? type { get; set; }
public EndpointItem[]? endpoints { get; set; }
}
public class AccessClass
{
public TokenClass token { get; set; }
public ServiceItem[] serviceCatalog { get; set; }
public TokenClass? token { get; set; }
public ServiceItem[]? serviceCatalog { get; set; }
}
}
private class OpenStackStorageItem
{
public string name { get; set; }
public string? name { get; set; }
public DateTime? last_modified { get; set; }
public long? bytes { get; set; }
public string content_type { get; set; }
public string subdir { get; set; }
public string? content_type { get; set; }
public string? subdir { get; set; }
}
private class WebHelper : JSONWebHelper
@@ -319,43 +317,54 @@ namespace Duplicati.Library.Backend.OpenStack
public WebHelper(OpenStackStorage parent) { m_parent = parent; }
public override HttpWebRequest CreateRequest(string url, string method = null)
public override HttpWebRequest CreateRequest(string url, string? method = null)
{
var req = base.CreateRequest(url, method);
req.Headers["X-Auth-Token"] = m_parent.AccessToken;
req.Headers["X-Auth-Token"] = m_parent.GetAccessToken(CancellationToken.None).Await();
return req;
}
}
public OpenStackStorage()
{
m_container = null!;
m_prefix = null!;
m_timeouts = null!;
m_username = null!;
m_authUri = null!;
m_helper = null!;
}
public OpenStackStorage(string url, Dictionary<string, string> options)
public OpenStackStorage(string url, Dictionary<string, string?> options)
{
var uri = new Utility.Uri(url);
m_container = uri.Host;
m_prefix = Util.AppendDirSeparator("/" + uri.Path, "/");
m_timeouts = TimeoutOptionsHelper.Parse(options);
// For OpenStack we do not use a leading slash
if (m_prefix.StartsWith("/", StringComparison.Ordinal))
m_prefix = m_prefix.Substring(1);
var auth = AuthOptionsHelper.Parse(options, uri);
options.TryGetValue(DOMAINNAME_OPTION, out m_domainName);
options.TryGetValue(USERNAME_OPTION, out m_username);
options.TryGetValue(PASSWORD_OPTION, out m_password);
options.TryGetValue(TENANTNAME_OPTION, out m_tenantName);
options.TryGetValue(AUTHURI_OPTION, out m_authUri);
options.TryGetValue(VERSION_OPTION, out m_version);
options.TryGetValue(APIKEY_OPTION, out m_apikey);
options.TryGetValue(REGION_OPTION, out m_region);
if (string.IsNullOrWhiteSpace(m_username))
throw new UserInformationException(Strings.OpenStack.MissingOptionError(USERNAME_OPTION), "OpenStackMissingUsername");
if (string.IsNullOrWhiteSpace(m_authUri))
if (!auth.HasUsername)
throw new UserInformationException(Strings.OpenStack.MissingOptionError(AuthOptionsHelper.AuthUsernameOption), "OpenStackMissingUsername");
m_username = auth.Username!;
m_password = auth.Password;
var authUri = options.GetValueOrDefault(AUTHURI_OPTION);
if (string.IsNullOrWhiteSpace(authUri))
throw new UserInformationException(Strings.OpenStack.MissingOptionError(AUTHURI_OPTION), "OpenStackMissingAuthUri");
m_authUri = authUri;
if (string.IsNullOrWhiteSpace(m_version) && m_authUri.EndsWith("/v3", StringComparison.OrdinalIgnoreCase))
m_version = "v3";
@@ -363,7 +372,7 @@ namespace Duplicati.Library.Backend.OpenStack
{
case "v3":
if (string.IsNullOrWhiteSpace(m_password))
throw new UserInformationException(Strings.OpenStack.MissingOptionError(PASSWORD_OPTION), "OpenStackMissingPassword");
throw new UserInformationException(Strings.OpenStack.MissingOptionError(AuthOptionsHelper.AuthPasswordOption), "OpenStackMissingPassword");
if (string.IsNullOrWhiteSpace(m_domainName))
throw new UserInformationException(Strings.OpenStack.MissingOptionError(DOMAINNAME_OPTION), "OpenStackMissingDomainName");
if (string.IsNullOrWhiteSpace(m_tenantName))
@@ -374,7 +383,7 @@ namespace Duplicati.Library.Backend.OpenStack
if (string.IsNullOrWhiteSpace(m_apikey))
{
if (string.IsNullOrWhiteSpace(m_password))
throw new UserInformationException(Strings.OpenStack.MissingOptionError(PASSWORD_OPTION), "OpenStackMissingPassword");
throw new UserInformationException(Strings.OpenStack.MissingOptionError(AuthOptionsHelper.AuthPasswordOption), "OpenStackMissingPassword");
if (string.IsNullOrWhiteSpace(m_tenantName))
throw new UserInformationException(Strings.OpenStack.MissingOptionError(TENANTNAME_OPTION), "OpenStackMissingTenantName");
}
@@ -384,15 +393,12 @@ namespace Duplicati.Library.Backend.OpenStack
m_helper = new WebHelper(this);
}
protected virtual string AccessToken
protected async Task<string> GetAccessToken(CancellationToken cancelToken)
{
get
{
if (m_accessToken == null || (m_accessToken.expires.HasValue && (m_accessToken.expires.Value - DateTime.UtcNow).TotalSeconds < 30))
GetAuthResponse();
if (m_accessToken == null || (m_accessToken.expires.HasValue && (m_accessToken.expires.Value - DateTime.UtcNow).TotalSeconds < 30))
await GetAuthResponse(cancelToken).ConfigureAwait(false);
return m_accessToken.id;
}
return m_accessToken!.id!;
}
private static string JoinUrls(string uri, string fragment)
@@ -405,21 +411,19 @@ namespace Duplicati.Library.Backend.OpenStack
return JoinUrls(JoinUrls(uri, fragment1), fragment2);
}
private void GetAuthResponse()
private Task GetAuthResponse(CancellationToken cancellationToken)
{
switch (this.m_version)
{
case "v3":
GetKeystone3AuthResponse();
break;
return GetKeystone3AuthResponse(cancellationToken);
case "v2":
default:
GetOpenstackAuthResponse();
break;
return GetOpenstackAuthResponse(cancellationToken);
}
}
private Keystone3AuthResponse GetKeystone3AuthResponse()
private async Task<Keystone3AuthResponse> GetKeystone3AuthResponse(CancellationToken cancellationToken)
{
var helper = new JSONWebHelper();
@@ -435,35 +439,50 @@ namespace Duplicati.Library.Backend.OpenStack
req.ContentLength = data.Length;
req.ContentType = "application/json; charset=UTF-8";
using (var rs = req.GetRequestStream())
rs.Write(data, 0, data.Length);
WebResponse http_response = req.GetResponse();
Keystone3AuthResponse resp;
using (var reader = new StreamReader(http_response.GetResponseStream()))
await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancellationToken, async ct =>
{
resp = Newtonsoft.Json.JsonConvert.DeserializeObject<Keystone3AuthResponse>(
reader.ReadToEnd());
}
using (var rs = req.GetRequestStream())
await rs.WriteAsync(data, 0, data.Length, ct);
}).ConfigureAwait(false);
string token = http_response.Headers["X-Subject-Token"];
this.m_accessToken = new OpenStackAuthResponse.TokenClass();
this.m_accessToken.id = token;
this.m_accessToken.expires = resp.token.expires_at;
var http_response = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancellationToken, ct => req.GetResponse()).ConfigureAwait(false);
var resp = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancellationToken, ct =>
{
using (var reader = new StreamReader(http_response.GetResponseStream()))
return JsonConvert.DeserializeObject<Keystone3AuthResponse>(reader.ReadToEnd())
?? throw new Exception("Failed to parse response");
}).ConfigureAwait(false);
if (resp.token == null)
throw new Exception("No token received");
var token = http_response.Headers["X-Subject-Token"];
if (string.IsNullOrWhiteSpace(token))
throw new Exception("No token received");
this.m_accessToken = new OpenStackAuthResponse.TokenClass()
{
id = token,
expires = resp.token.expires_at
};
// Grab the endpoint now that we have received it anyway
var fileservice = resp.token.catalog.FirstOrDefault(x => string.Equals(x.type, "object-store", StringComparison.OrdinalIgnoreCase));
var fileservice = (resp.token.catalog ?? []).FirstOrDefault(x => string.Equals(x.type, "object-store", StringComparison.OrdinalIgnoreCase));
if (fileservice == null)
throw new Exception("No object-store service found, is this service supported by the provider?");
if (fileservice.endpoints == null || fileservice.endpoints.Length == 0)
throw new Exception("No endpoints found for object-store service");
var endpoint = fileservice.endpoints.FirstOrDefault(x => (string.Equals(m_region, x.region) && string.Equals(x.interface_name, "public", StringComparison.OrdinalIgnoreCase))) ?? fileservice.endpoints.First();
m_simplestorageendpoint = endpoint.url;
return resp;
}
private OpenStackAuthResponse GetOpenstackAuthResponse()
private async Task<OpenStackAuthResponse> GetOpenstackAuthResponse(CancellationToken cancellationToken)
{
var helper = new JSONWebHelper();
@@ -471,53 +490,60 @@ namespace Duplicati.Library.Backend.OpenStack
req.Accept = "application/json";
req.Method = "POST";
var resp = helper.ReadJSONResponse<OpenStackAuthResponse>(
var resp = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancellationToken, ct => helper.ReadJSONResponse<OpenStackAuthResponse>(
req,
new OpenStackAuthRequest(m_tenantName, m_username, m_password, m_apikey)
);
)).ConfigureAwait(false);
if (resp.access?.token == null)
throw new Exception("No token received");
m_accessToken = resp.access.token;
// Grab the endpoint now that we have received it anyway
var fileservice = resp.access.serviceCatalog.FirstOrDefault(x => string.Equals(x.type, "object-store", StringComparison.OrdinalIgnoreCase));
var fileservice = (resp.access.serviceCatalog ?? []).FirstOrDefault(x => string.Equals(x.type, "object-store", StringComparison.OrdinalIgnoreCase));
if (fileservice == null)
throw new Exception("No object-store service found, is this service supported by the provider?");
var endpoint = fileservice.endpoints.FirstOrDefault(x => string.Equals(m_region, x.region)) ?? fileservice.endpoints.First();
var fileserviceendpoints = fileservice.endpoints ?? [];
var endpoint = fileserviceendpoints.FirstOrDefault(x => string.Equals(m_region, x.region))
?? fileserviceendpoints.FirstOrDefault();
if (endpoint == null)
throw new Exception("No endpoint found for object-store service");
m_simplestorageendpoint = endpoint.publicURL;
return resp;
}
protected virtual string SimpleStorageEndPoint
protected async Task<string> GetSimpleStorageEndPoint(CancellationToken cancelToken)
{
get
{
if (m_simplestorageendpoint == null)
GetAuthResponse();
if (m_simplestorageendpoint == null)
await GetAuthResponse(cancelToken);
return m_simplestorageendpoint;
}
return m_simplestorageendpoint!;
}
#region IStreamingBackend implementation
public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken)
{
var url = JoinUrls(SimpleStorageEndPoint, m_container, Utility.Uri.UrlPathEncode(m_prefix + remotename));
using (await m_helper.GetResponseAsync(url, cancelToken, stream, "PUT").ConfigureAwait(false))
var url = JoinUrls(await GetSimpleStorageEndPoint(cancelToken).ConfigureAwait(false), m_container, Utility.Uri.UrlPathEncode(m_prefix + remotename));
using (var ts = stream.ObserveReadTimeout(m_timeouts.ShortTimeout, false))
using (await m_helper.GetResponseAsync(url, cancelToken, ts, "PUT").ConfigureAwait(false))
{ }
}
public async Task GetAsync(string remotename, Stream stream, CancellationToken cancelToken)
{
var url = JoinUrls(SimpleStorageEndPoint, m_container, Utility.Uri.UrlPathEncode(m_prefix + remotename));
var url = JoinUrls(await GetSimpleStorageEndPoint(cancelToken).ConfigureAwait(false), m_container, Utility.Uri.UrlPathEncode(m_prefix + remotename));
try
{
using (var resp = m_helper.GetResponse(url))
using (var rs = AsyncHttpRequest.TrySetTimeout(resp.GetResponseStream()))
await Library.Utility.Utility.CopyStreamAsync(rs, stream, cancelToken).ConfigureAwait(false);
using (var resp = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => m_helper.GetResponse(url)).ConfigureAwait(false))
using (var rs = AsyncHttpRequest.TrySetTimeout(await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => resp.GetResponseStream()).ConfigureAwait(false)))
using (var ts = rs.ObserveReadTimeout(m_timeouts.ShortTimeout))
await Utility.Utility.CopyStreamAsync(ts, stream, cancelToken).ConfigureAwait(false);
}
catch (WebException wex)
{
@@ -549,9 +575,9 @@ namespace Duplicati.Library.Backend.OpenStack
/// <inheritdoc />
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancelToken)
{
var plainurl = JoinUrls(SimpleStorageEndPoint, m_container) + string.Format("?format=json&delimiter=/&limit={0}", PAGE_LIMIT);
var plainurl = JoinUrls(await GetSimpleStorageEndPoint(cancelToken).ConfigureAwait(false), m_container) + string.Format("?format=json&delimiter=/&limit={0}", PAGE_LIMIT);
if (!string.IsNullOrEmpty(m_prefix))
plainurl += "&prefix=" + Library.Utility.Uri.UrlEncode(m_prefix);
plainurl += "&prefix=" + Utility.Uri.UrlEncode(m_prefix);
var url = plainurl;
@@ -560,10 +586,13 @@ namespace Duplicati.Library.Backend.OpenStack
var req = m_helper.CreateRequest(url);
req.Accept = "application/json";
var items = await HandleListExceptions(() => m_helper.ReadJSONResponseAsync<OpenStackStorageItem[]>(req, cancelToken)).ConfigureAwait(false);
var items = await HandleListExceptions(async () => await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, ct => m_helper.ReadJSONResponseAsync<OpenStackStorageItem[]>(req, ct))).ConfigureAwait(false);
foreach (var n in items)
{
var name = n.name;
if (string.IsNullOrWhiteSpace(name))
continue;
if (name.StartsWith(m_prefix, StringComparison.Ordinal))
name = name.Substring(m_prefix.Length);
@@ -594,34 +623,22 @@ namespace Duplicati.Library.Backend.OpenStack
using (FileStream fs = File.Create(filename))
await GetAsync(remotename, fs, cancelToken).ConfigureAwait(false);
}
public Task DeleteAsync(string remotename, CancellationToken cancelToken)
public async Task DeleteAsync(string remotename, CancellationToken cancelToken)
{
var url = JoinUrls(SimpleStorageEndPoint, m_container, Library.Utility.Uri.UrlPathEncode(m_prefix + remotename));
return m_helper.ReadJSONResponseAsync<object>(url, cancelToken, null, "DELETE");
var url = JoinUrls(await GetSimpleStorageEndPoint(cancelToken).ConfigureAwait(false), m_container, Library.Utility.Uri.UrlPathEncode(m_prefix + remotename));
await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => m_helper.ReadJSONResponseAsync<object>(url, ct, null, "DELETE")).ConfigureAwait(false);
}
public Task TestAsync(CancellationToken cancelToken)
=> this.TestListAsync(cancelToken);
public async Task CreateFolderAsync(CancellationToken cancelToken)
{
var url = JoinUrls(SimpleStorageEndPoint, m_container);
using (await m_helper.GetResponseAsync(url, cancelToken, null, "PUT").ConfigureAwait(false))
var url = JoinUrls(await GetSimpleStorageEndPoint(cancelToken).ConfigureAwait(false), m_container);
using (await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => m_helper.GetResponseAsync(url, cancelToken, null, "PUT")).ConfigureAwait(false))
{ }
}
public string DisplayName
{
get
{
return Strings.OpenStack.DisplayName;
}
}
public string ProtocolKey
{
get
{
return "openstack";
}
}
public string DisplayName => Strings.OpenStack.DisplayName;
public string ProtocolKey => "openstack";
public IList<ICommandLineArgument> SupportedCommands
{
get
@@ -630,30 +647,27 @@ namespace Duplicati.Library.Backend.OpenStack
foreach (var s in KNOWN_OPENSTACK_PROVIDERS)
authuris.AppendLine(string.Format("{0}: {1}", s.Key, s.Value));
return new List<ICommandLineArgument>(new[] {
return [
new CommandLineArgument(DOMAINNAME_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.DomainnameOptionShort, Strings.OpenStack.DomainnameOptionLong),
new CommandLineArgument(USERNAME_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.UsernameOptionShort, Strings.OpenStack.UsernameOptionLong),
new CommandLineArgument(PASSWORD_OPTION, CommandLineArgument.ArgumentType.Password, Strings.OpenStack.PasswordOptionShort, Strings.OpenStack.PasswordOptionLong(TENANTNAME_OPTION)),
new CommandLineArgument(AuthOptionsHelper.AuthUsernameOption, CommandLineArgument.ArgumentType.String, Strings.OpenStack.UsernameOptionShort, Strings.OpenStack.UsernameOptionLong),
new CommandLineArgument(AuthOptionsHelper.AuthPasswordOption, CommandLineArgument.ArgumentType.Password, Strings.OpenStack.PasswordOptionShort, Strings.OpenStack.PasswordOptionLong(TENANTNAME_OPTION)),
new CommandLineArgument(TENANTNAME_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.TenantnameOptionShort, Strings.OpenStack.TenantnameOptionLong),
new CommandLineArgument(APIKEY_OPTION, CommandLineArgument.ArgumentType.Password, Strings.OpenStack.ApikeyOptionShort, Strings.OpenStack.ApikeyOptionLong),
new CommandLineArgument(AUTHURI_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.AuthuriOptionShort, Strings.OpenStack.AuthuriOptionLong(authuris.ToString())),
new CommandLineArgument(VERSION_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.VersionOptionShort, Strings.OpenStack.VersionOptionLong),
new CommandLineArgument(REGION_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.RegionOptionShort, Strings.OpenStack.RegionOptionLong),
});
}
}
public string Description
{
get
{
return Strings.OpenStack.Description;
.. TimeoutOptionsHelper.GetOptions(),
];
}
}
public string Description => Strings.OpenStack.Description;
public Task<string[]> GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new[] {
new System.Uri(m_authUri).Host,
string.IsNullOrWhiteSpace(m_simplestorageendpoint) ? null : new System.Uri(m_simplestorageendpoint).Host
});
}
.WhereNotNullOrWhiteSpace()
.ToArray());
#endregion
#region IDisposable implementation
+27 -19
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Localization.Short;
@@ -44,6 +44,14 @@ namespace Duplicati.Library.Backend.Strings
public static string VersionOptionShort { get { return LC.L(@"The keystone API version to use"); } }
public static string RegionOptionLong { get { return LC.L(@"By default, the first reported endpoint will be used for file transfers. To select a specific region, provide the region name. If no such region is supported, the default (first reported) endpoint is used."); } }
public static string RegionOptionShort { get { return LC.L(@"Supply the prefered region for endpoints"); } }
}
internal static class SwiftConfig
{
public static string Description { get { return LC.L(@"Expose OpenStack configuration as a web module"); } }
public static string DisplayName { get { return LC.L(@"OpenStack configuration module"); } }
public static string ConfigTypeOptionLong { get { return LC.L(@"Provide different config values"); } }
public static string ConfigTypeOptionShort { get { return LC.L(@"The config to get"); } }
}
}
@@ -4,6 +4,8 @@
<TargetFramework>net8.0</TargetFramework>
<OutputType>Library</OutputType>
<Copyright>Copyright © 2025 Team Duplicati, MIT license</Copyright>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+66 -67
View File
@@ -21,16 +21,12 @@
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility.Options;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace Duplicati.Library.Backend
{
@@ -51,22 +47,28 @@ namespace Duplicati.Library.Backend
private readonly string remote_path;
private readonly string opt_rclone;
private readonly string rclone_executable;
private readonly TimeoutOptionsHelper.Timeouts timeouts;
public Rclone()
{
local_repo = null!;
remote_repo = null!;
remote_path = null!;
opt_rclone = null!;
rclone_executable = null!;
timeouts = null!;
}
public Rclone(string url, Dictionary<string, string> options)
public Rclone(string url, Dictionary<string, string?> options)
{
var uri = new Utility.Uri(url);
/*should check here if program is installed */
local_repo = options.GetValueOrDefault(OPTION_LOCAL_REPO, local_repo);
remote_repo = options.GetValueOrDefault(OPTION_REMOTE_REPO, remote_repo);
remote_path = options.GetValueOrDefault(OPTION_REMOTE_PATH, remote_path);
opt_rclone = options.GetValueOrDefault(OPTION_RCLONE, opt_rclone) ?? "";
rclone_executable = options.GetValueOrDefault(OPTION_RCLONE_EXECUTABLE, rclone_executable);
local_repo = options.GetValueOrDefault(OPTION_LOCAL_REPO) ?? "";
remote_repo = options.GetValueOrDefault(OPTION_REMOTE_REPO) ?? "";
remote_path = options.GetValueOrDefault(OPTION_REMOTE_PATH) ?? "";
opt_rclone = options.GetValueOrDefault(OPTION_RCLONE) ?? "";
rclone_executable = options.GetValueOrDefault(OPTION_RCLONE_EXECUTABLE) ?? "";
if (string.IsNullOrWhiteSpace(local_repo))
local_repo = "local";
@@ -77,6 +79,8 @@ namespace Duplicati.Library.Backend
if (string.IsNullOrWhiteSpace(rclone_executable))
rclone_executable = "rclone";
timeouts = TimeoutOptionsHelper.Parse(options);
#if DEBUG
Console.WriteLine("Constructor {0}: {1}:{2} {3}", local_repo, remote_repo, remote_path, opt_rclone);
#endif
@@ -84,17 +88,11 @@ namespace Duplicati.Library.Backend
#region IBackendInterface Members
public string DisplayName
{
get { return Strings.Rclone.DisplayName; }
}
public string DisplayName => Strings.Rclone.DisplayName;
public string ProtocolKey
{
get { return "rclone"; }
}
public string ProtocolKey => "rclone";
private async Task<string> RcloneCommandExecuter(String command, String arguments, CancellationToken cancelToken)
private async Task<string> RcloneCommandExecuter(string command, string arguments, TimeSpan timeout, CancellationToken cancelToken)
{
StringBuilder outputBuilder = new StringBuilder();
StringBuilder errorBuilder = new StringBuilder();
@@ -122,11 +120,11 @@ namespace Duplicati.Library.Backend
EnableRaisingEvents = true
};
// attach the event handler for OutputDataReceived before starting the process
process.OutputDataReceived += new System.Diagnostics.DataReceivedEventHandler
process.OutputDataReceived += new DataReceivedEventHandler
(
delegate (object sender, System.Diagnostics.DataReceivedEventArgs e)
delegate (object sender, DataReceivedEventArgs e)
{
if (!String.IsNullOrEmpty(e.Data))
if (!string.IsNullOrEmpty(e.Data))
{
#if DEBUG
// Console.Error.WriteLine(String.Format("output {0}", e.Data));
@@ -137,12 +135,12 @@ namespace Duplicati.Library.Backend
}
);
process.ErrorDataReceived += new System.Diagnostics.DataReceivedEventHandler
process.ErrorDataReceived += new DataReceivedEventHandler
(
delegate (object sender, System.Diagnostics.DataReceivedEventArgs e)
delegate (object sender, DataReceivedEventArgs e)
{
if (!String.IsNullOrEmpty(e.Data))
if (!string.IsNullOrEmpty(e.Data))
{
#if DEBUG
Console.Error.WriteLine("error {0}", e.Data);
@@ -160,11 +158,17 @@ namespace Duplicati.Library.Backend
process.BeginOutputReadLine();
process.BeginErrorReadLine();
using var timer = timeout == Timeout.InfiniteTimeSpan
? new TaskCompletionSource<bool>().Task
: Task.Delay(timeout);
var killed = false;
while (!process.HasExited)
{
await Task.Delay(500).ConfigureAwait(false);
if (cancelToken.IsCancellationRequested)
if (cancelToken.IsCancellationRequested || timer.IsCompleted)
{
killed = true;
process.Kill();
process.WaitForExit();
}
@@ -174,18 +178,20 @@ namespace Duplicati.Library.Backend
process.CancelErrorRead();
if (errorBuilder.ToString().Contains(RCLONE_ERROR_DIRECTORY_NOT_FOUND))
{
throw new FolderMissingException(errorBuilder.ToString());
}
if (errorBuilder.ToString().Contains(RCLONE_ERROR_CONFIG_NOT_FOUND))
{
throw new Exception($"Missing config file? {errorBuilder}");
}
if (errorBuilder.Length > 0)
{
throw new Exception(errorBuilder.ToString());
if (killed)
{
if (timer.IsCompleted)
throw new TimeoutException();
throw new TaskCanceledException();
}
return outputBuilder.ToString();
@@ -200,16 +206,15 @@ namespace Duplicati.Library.Backend
try
{
str_result = await RcloneCommandExecuter(rclone_executable, $"lsjson {remote_repo}:{remote_path}", cancelToken).ConfigureAwait(false);
str_result = await RcloneCommandExecuter(rclone_executable, $"lsjson {remote_repo}:{remote_path}", timeouts.ListTimeout, cancelToken).ConfigureAwait(false);
// this will give an error if the executable does not exist.
}
catch (FolderMissingException ex)
{
throw new FolderMissingException(ex);
}
using (JsonReader jsonReader = new JsonTextReader(new StringReader(str_result)))
using (var jsonReader = new JsonTextReader(new StringReader(str_result)))
{
//no date parsing by JArray needed, will be parsed later
jsonReader.DateParseHandling = DateParseHandling.None;
@@ -220,14 +225,19 @@ namespace Duplicati.Library.Backend
#if DEBUG
Console.Error.WriteLine(item);
#endif
FileEntry fe = new FileEntry(
item.GetValue("Name").Value<string>(),
item.GetValue("Size").Value<long>(),
DateTime.Parse(item.GetValue("ModTime").Value<string>()),
DateTime.Parse(item.GetValue("ModTime").Value<string>())
var modTimeString = item.GetValue("ModTime")?.Value<string>();
var modTime = string.IsNullOrWhiteSpace(modTimeString)
? new DateTime(0)
: DateTime.Parse(modTimeString);
var fe = new FileEntry(
item.GetValue("Name")?.Value<string>(),
item.GetValue("Size")?.Value<long>() ?? -1,
modTime,
modTime
)
{
IsFolder = item.GetValue("IsDir").Value<bool>()
IsFolder = item.GetValue("IsDir")?.Value<bool>() ?? false
};
yield return fe;
}
@@ -238,7 +248,7 @@ namespace Duplicati.Library.Backend
{
try
{
await RcloneCommandExecuter(rclone_executable, $"copyto {local_repo}:{filename} {remote_repo}:{remote_path}/{remotename}", cancelToken).ConfigureAwait(false);
await RcloneCommandExecuter(rclone_executable, $"copyto {local_repo}:{filename} {remote_repo}:{remote_path}/{remotename}", Timeout.InfiniteTimeSpan, cancelToken).ConfigureAwait(false);
}
catch (FolderMissingException ex)
{
@@ -250,7 +260,7 @@ namespace Duplicati.Library.Backend
{
try
{
await RcloneCommandExecuter(rclone_executable, $"copyto {remote_repo}:{Path.Combine(this.remote_path, remotename)} {local_repo}:{filename}", cancelToken).ConfigureAwait(false);
await RcloneCommandExecuter(rclone_executable, $"copyto {remote_repo}:{Path.Combine(this.remote_path, remotename)} {local_repo}:{filename}", Timeout.InfiniteTimeSpan, cancelToken).ConfigureAwait(false);
}
catch (FolderMissingException ex)
{
@@ -264,7 +274,7 @@ namespace Duplicati.Library.Backend
// Will give a "directory not found" error if the file does not exist, need to change that to a missing file exception
try
{
await RcloneCommandExecuter(rclone_executable, $"delete {remote_repo}:{Path.Combine(remote_path, remotename)}", cancelToken).ConfigureAwait(false);
await RcloneCommandExecuter(rclone_executable, $"delete {remote_repo}:{Path.Combine(remote_path, remotename)}", timeouts.ShortTimeout, cancelToken).ConfigureAwait(false);
}
catch (FolderMissingException ex)
{
@@ -272,28 +282,17 @@ namespace Duplicati.Library.Backend
}
}
public IList<ICommandLineArgument> SupportedCommands
{
get
{
return new List<ICommandLineArgument>([
new CommandLineArgument(OPTION_LOCAL_REPO, CommandLineArgument.ArgumentType.String, Strings.Rclone.RcloneLocalRepoShort, Strings.Rclone.RcloneLocalRepoLong, "local"),
new CommandLineArgument(OPTION_REMOTE_REPO, CommandLineArgument.ArgumentType.String, Strings.Rclone.RcloneRemoteRepoShort, Strings.Rclone.RcloneRemoteRepoLong, "remote"),
new CommandLineArgument(OPTION_REMOTE_PATH, CommandLineArgument.ArgumentType.String, Strings.Rclone.RcloneRemotePathShort, Strings.Rclone.RcloneRemotePathLong, "backup"),
new CommandLineArgument(OPTION_RCLONE, CommandLineArgument.ArgumentType.String, Strings.Rclone.RcloneOptionRcloneShort, Strings.Rclone.RcloneOptionRcloneLong, ""),
new CommandLineArgument(OPTION_RCLONE_EXECUTABLE, CommandLineArgument.ArgumentType.String, Strings.Rclone.RcloneExecutableShort, Strings.Rclone.RcloneExecutableLong, "rclone")
]);
public IList<ICommandLineArgument> SupportedCommands => [
new CommandLineArgument(OPTION_LOCAL_REPO, CommandLineArgument.ArgumentType.String, Strings.Rclone.RcloneLocalRepoShort, Strings.Rclone.RcloneLocalRepoLong, "local"),
new CommandLineArgument(OPTION_REMOTE_REPO, CommandLineArgument.ArgumentType.String, Strings.Rclone.RcloneRemoteRepoShort, Strings.Rclone.RcloneRemoteRepoLong, "remote"),
new CommandLineArgument(OPTION_REMOTE_PATH, CommandLineArgument.ArgumentType.String, Strings.Rclone.RcloneRemotePathShort, Strings.Rclone.RcloneRemotePathLong, "backup"),
new CommandLineArgument(OPTION_RCLONE, CommandLineArgument.ArgumentType.String, Strings.Rclone.RcloneOptionRcloneShort, Strings.Rclone.RcloneOptionRcloneLong, ""),
new CommandLineArgument(OPTION_RCLONE_EXECUTABLE, CommandLineArgument.ArgumentType.String, Strings.Rclone.RcloneExecutableShort, Strings.Rclone.RcloneExecutableLong, "rclone"),
.. TimeoutOptionsHelper.GetOptions()
.Where(x => x.Name != TimeoutOptionsHelper.ReadWriteTimeoutOption)
];
}
}
public string Description
{
get
{
return Strings.Rclone.Description;
}
}
public string Description => Strings.Rclone.Description;
public Task<string[]> GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new[] { remote_repo });
@@ -302,7 +301,7 @@ namespace Duplicati.Library.Backend
public Task CreateFolderAsync(CancellationToken cancelToken)
{
return RcloneCommandExecuter(rclone_executable, $"mkdir {remote_repo}:{remote_path}", cancelToken);
return RcloneCommandExecuter(rclone_executable, $"mkdir {remote_repo}:{remote_path}", timeouts.ShortTimeout, cancelToken);
}
#endregion
@@ -5,6 +5,8 @@
<OutputType>Library</OutputType>
<RootNamespace>Duplicati.Library.Backend</RootNamespace>
<Copyright>Copyright © 2025 Team Duplicati, MIT license</Copyright>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>
<ItemGroup>
+3 -7
View File
@@ -18,10 +18,6 @@
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Duplicati.Library.Interface;
namespace Duplicati.Library.Backend
@@ -36,10 +32,10 @@ namespace Duplicati.Library.Backend
Task RenameFileAsync(string bucketName, string source, string target, CancellationToken cancelToken);
Task GetFileStreamAsync(string bucketName, string keyName, System.IO.Stream target, CancellationToken cancelToken);
Task GetFileStreamAsync(string bucketName, string keyName, Stream target, CancellationToken cancelToken);
string GetDnsHost();
string? GetDnsHost();
Task AddFileStreamAsync(string bucketName, string keyName, System.IO.Stream source, CancellationToken cancelToken);
Task AddFileStreamAsync(string bucketName, string keyName, Stream source, CancellationToken cancelToken);
}
}
+24 -32
View File
@@ -25,12 +25,8 @@ using Amazon.S3.Model;
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
using System;
using System.Collections.Generic;
using System.Linq;
using Duplicati.Library.Utility.Options;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
namespace Duplicati.Library.Backend
{
@@ -39,23 +35,23 @@ namespace Duplicati.Library.Backend
/// </summary>
public class S3AwsClient : IS3Client
{
private static readonly string LOGTAG = Logging.Log.LogTagFromType<S3AwsClient>();
private const int ITEM_LIST_LIMIT = 1000;
private const string EXT_OPTION_PREFIX = "s3-ext-";
private readonly string m_locationConstraint;
private readonly string m_storageClass;
private AmazonS3Client m_client;
private readonly string? m_locationConstraint;
private readonly string? m_storageClass;
private readonly AmazonS3Client m_client;
private readonly bool m_useChunkEncoding;
private readonly string m_dnsHost;
private readonly string? m_dnsHost;
private readonly bool m_useV2ListApi;
private readonly TimeoutOptionsHelper.Timeouts m_timeouts;
public S3AwsClient(string awsID, string awsKey, string locationConstraint, string servername,
string storageClass, bool useSSL, bool disableChunkEncoding, Dictionary<string, string> options)
public S3AwsClient(string awsID, string awsKey, string? locationConstraint, string servername,
string? storageClass, bool useSSL, bool disableChunkEncoding, TimeoutOptionsHelper.Timeouts timeouts, Dictionary<string, string?> options)
{
var cfg = S3AwsClient.GetDefaultAmazonS3Config();
var cfg = GetDefaultAmazonS3Config();
cfg.UseHttp = !useSSL;
cfg.ServiceURL = (useSSL ? "https://" : "http://") + servername;
@@ -63,6 +59,7 @@ namespace Duplicati.Library.Backend
m_client = new AmazonS3Client(awsID, awsKey, cfg);
m_timeouts = timeouts;
m_useV2ListApi = string.Equals(options.GetValueOrDefault("list-api-version", "v1"), "v2", StringComparison.OrdinalIgnoreCase);
m_locationConstraint = locationConstraint;
m_storageClass = storageClass;
@@ -80,7 +77,7 @@ namespace Duplicati.Library.Backend
if (!string.IsNullOrEmpty(m_locationConstraint))
request.BucketRegionName = m_locationConstraint;
return m_client.PutBucketAsync(request, cancelToken);
return Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => m_client.PutBucketAsync(request, ct));
}
public static AmazonS3Config GetDefaultAmazonS3Config()
@@ -123,7 +120,7 @@ namespace Duplicati.Library.Backend
return x;
});
public virtual async Task GetFileStreamAsync(string bucketName, string keyName, System.IO.Stream target, CancellationToken cancelToken)
public virtual async Task GetFileStreamAsync(string bucketName, string keyName, Stream target, CancellationToken cancelToken)
{
try
{
@@ -133,15 +130,10 @@ namespace Duplicati.Library.Backend
Key = keyName
};
using (var objectGetResponse = await m_client.GetObjectAsync(objectGetRequest).ConfigureAwait(false))
using (var objectGetResponse = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => m_client.GetObjectAsync(objectGetRequest, ct)).ConfigureAwait(false))
using (var s = objectGetResponse.ResponseStream)
{
// TODO: This does not work and throws InvalidOperationException()
try { s.ReadTimeout = (int)TimeSpan.FromMinutes(1).TotalMilliseconds; }
catch { }
await Utility.Utility.CopyStreamAsync(s, target, cancelToken).ConfigureAwait(false);
}
using (var t = s.ObserveReadTimeout(m_timeouts.ReadWriteTimeout))
await Utility.Utility.CopyStreamAsync(t, target, cancelToken).ConfigureAwait(false);
}
catch (AmazonS3Exception s3Ex)
{
@@ -151,19 +143,20 @@ namespace Duplicati.Library.Backend
}
public string GetDnsHost()
public string? GetDnsHost()
{
return m_dnsHost;
}
public virtual async Task AddFileStreamAsync(string bucketName, string keyName, System.IO.Stream source,
public virtual async Task AddFileStreamAsync(string bucketName, string keyName, Stream source,
CancellationToken cancelToken)
{
using var ts = source.ObserveReadTimeout(m_timeouts.ReadWriteTimeout, false);
var objectAddRequest = new PutObjectRequest
{
BucketName = bucketName,
Key = keyName,
InputStream = source,
InputStream = ts,
UseChunkEncoding = m_useChunkEncoding
};
if (!string.IsNullOrWhiteSpace(m_storageClass))
@@ -192,7 +185,7 @@ namespace Duplicati.Library.Backend
Key = keyName
};
return m_client.DeleteObjectAsync(objectDeleteRequest, cancellationToken);
return Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancellationToken, ct => m_client.DeleteObjectAsync(objectDeleteRequest, ct));
}
/// <summary>
@@ -206,7 +199,7 @@ namespace Duplicati.Library.Backend
public async IAsyncEnumerable<IFileEntry> ListBucketAsync(string bucketName, string prefix, bool recursive, [EnumeratorCancellation] CancellationToken cancellationToken)
{
var isTruncated = true;
string filename = null;
string? filename = null;
var delimiter = recursive ? "" : "/";
if (!string.IsNullOrWhiteSpace(prefix))
prefix = Util.AppendDirSeparator(prefix, "/");
@@ -232,7 +225,7 @@ namespace Duplicati.Library.Backend
MaxKeys = ITEM_LIST_LIMIT,
Delimiter = delimiter
};
listResponse = await m_client.ListObjectsV2Async(listRequest, cancellationToken).ConfigureAwait(false);
listResponse = await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancellationToken, ct => m_client.ListObjectsV2Async(listRequest, ct)).ConfigureAwait(false);
}
else
{
@@ -245,7 +238,7 @@ namespace Duplicati.Library.Backend
MaxKeys = ITEM_LIST_LIMIT,
Delimiter = delimiter
};
var listResponsev1 = await m_client.ListObjectsAsync(listRequest, cancellationToken).ConfigureAwait(false);
var listResponsev1 = await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancellationToken, ct => m_client.ListObjectsAsync(listRequest, ct)).ConfigureAwait(false);
// Map the V1 response to the V2 response
listResponse = new ListObjectsV2Response
@@ -314,7 +307,7 @@ namespace Duplicati.Library.Backend
DestinationKey = target
};
await m_client.CopyObjectAsync(copyObjectRequest, cancelToken).ConfigureAwait(false);
await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, ct => m_client.CopyObjectAsync(copyObjectRequest, ct)).ConfigureAwait(false);
await DeleteObjectAsync(bucketName, source, cancelToken).ConfigureAwait(false);
}
@@ -324,7 +317,6 @@ namespace Duplicati.Library.Backend
{
if (m_client != null)
m_client.Dispose();
m_client = null;
}
#endregion
+41 -45
View File
@@ -21,15 +21,9 @@
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
using Duplicati.Library.Utility.Options;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
namespace Duplicati.Library.Backend
{
@@ -37,6 +31,9 @@ namespace Duplicati.Library.Backend
{
private static readonly string LOGTAG = Logging.Log.LogTagFromType<S3>();
private const string AUTH_USERNAME_OPTION = "aws-access-key-id";
private const string AUTH_PASSWORD_OPTION = "aws-secret-access-key";
private const string STORAGECLASS_OPTION = "s3-storage-class";
private const string SERVER_NAME = "s3-server-name";
private const string LOCATION_OPTION = "s3-location-constraint";
@@ -180,33 +177,29 @@ namespace Duplicati.Library.Backend
private readonly bool m_recurseLists;
private const string DEFAULT_S3_HOST = "s3.amazonaws.com";
private IS3Client s3Client;
private readonly IS3Client m_s3Client;
public S3()
{
m_bucket = null!;
m_prefix = null!;
m_s3Client = null!;
}
public S3(string url, Dictionary<string, string> options)
public S3(string url, Dictionary<string, string?> options)
{
var uri = new Utility.Uri(url);
uri.RequireHost();
m_bucket = uri.Host;
m_prefix = uri.Path;
var timeout = TimeoutOptionsHelper.Parse(options);
if (!options.TryGetValue("aws-access-key-id", out var awsID))
options.TryGetValue("auth-username", out awsID);
if (!options.TryGetValue("aws-secret-access-key", out var awsKey))
options.TryGetValue("auth-password", out awsKey);
var auth = AuthOptionsHelper.ParseWithAlias(options, uri, AUTH_USERNAME_OPTION, AUTH_PASSWORD_OPTION);
if (!string.IsNullOrEmpty(uri.Username))
awsID = uri.Username;
if (!string.IsNullOrEmpty(uri.Password))
awsKey = uri.Password;
if (string.IsNullOrEmpty(awsID))
if (!auth.HasUsername)
throw new UserInformationException(Strings.S3Backend.NoAMZUserIDError, "S3NoAmzUserID");
if (string.IsNullOrEmpty(awsKey))
if (!auth.HasPassword)
throw new UserInformationException(Strings.S3Backend.NoAMZKeyError, "S3NoAmzKey");
var useSSL = Utility.Utility.ParseBoolOption(options, SSL_OPTION);
@@ -240,13 +233,14 @@ namespace Duplicati.Library.Backend
var s3ClientOptionValue = options.GetValueOrDefault(S3_CLIENT_OPTION);
(var awsID, var awsKey) = auth.GetCredentials();
if (string.IsNullOrWhiteSpace(s3ClientOptionValue) || string.Equals(s3ClientOptionValue, "aws", StringComparison.OrdinalIgnoreCase))
{
s3Client = new S3AwsClient(awsID, awsKey, locationConstraint, hostname, storageClass, useSSL, disableChunkEncoding, options);
m_s3Client = new S3AwsClient(awsID, awsKey, locationConstraint, hostname, storageClass, useSSL, disableChunkEncoding, timeout, options);
}
else if (string.Equals(s3ClientOptionValue, "minio", StringComparison.OrdinalIgnoreCase))
{
s3Client = new S3MinioClient(awsID, awsKey, locationConstraint, hostname, storageClass, useSSL, options);
m_s3Client = new S3MinioClient(awsID, awsKey, locationConstraint, hostname, storageClass, useSSL, timeout, options);
}
else
{
@@ -310,8 +304,8 @@ namespace Duplicati.Library.Backend
{
get
{
StringBuilder hostnames = new StringBuilder();
StringBuilder locations = new StringBuilder();
var hostnames = new StringBuilder();
var locations = new StringBuilder();
foreach (var s in KNOWN_S3_PROVIDERS)
hostnames.AppendLine(string.Format("{0}: {1}", s.Key, s.Value));
@@ -320,9 +314,9 @@ namespace Duplicati.Library.Backend
var exts = S3AwsClient.GetAwsExtendedOptions();
var normal = new ICommandLineArgument[] {
new CommandLineArgument("aws-secret-access-key", CommandLineArgument.ArgumentType.Password, Strings.S3Backend.AMZKeyDescriptionShort, Strings.S3Backend.AMZKeyDescriptionLong,null, new string[] {"auth-password"}, null ),
new CommandLineArgument("aws-access-key-id", CommandLineArgument.ArgumentType.String, Strings.S3Backend.AMZUserIDDescriptionShort, Strings.S3Backend.AMZUserIDDescriptionLong, null, new string[] {"auth-username"}, null),
return [
new CommandLineArgument("aws-access-key-id", CommandLineArgument.ArgumentType.String, Strings.S3Backend.AMZUserIDDescriptionShort, Strings.S3Backend.AMZUserIDDescriptionLong, null, [AuthOptionsHelper.AuthUsernameOption], null),
new CommandLineArgument("aws-secret-access-key", CommandLineArgument.ArgumentType.Password, Strings.S3Backend.AMZKeyDescriptionShort, Strings.S3Backend.AMZKeyDescriptionLong,null, [AuthOptionsHelper.AuthPasswordOption], null ),
new CommandLineArgument(STORAGECLASS_OPTION, CommandLineArgument.ArgumentType.String, Strings.S3Backend.S3StorageclassDescriptionShort, Strings.S3Backend.S3StorageclassDescriptionLong),
new CommandLineArgument(SERVER_NAME, CommandLineArgument.ArgumentType.String, Strings.S3Backend.S3ServerNameDescriptionShort, Strings.S3Backend.S3ServerNameDescriptionLong(hostnames.ToString()), DEFAULT_S3_HOST),
new CommandLineArgument(LOCATION_OPTION, CommandLineArgument.ArgumentType.String, Strings.S3Backend.S3LocationDescriptionShort, Strings.S3Backend.S3LocationDescriptionLong(locations.ToString())),
@@ -331,12 +325,9 @@ namespace Duplicati.Library.Backend
new CommandLineArgument(S3_DISABLE_CHUNK_ENCODING_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.S3Backend.DescriptionDisableChunkEncodingShort, Strings.S3Backend.DescriptionDisableChunkEncodingLong, "false"),
new CommandLineArgument(S3_LIST_API_VERSION_OPTION, CommandLineArgument.ArgumentType.Enumeration, Strings.S3Backend.DescriptionListApiVersionShort, Strings.S3Backend.DescriptionListApiVersionLong, "v1", null, ["v1", "v2"]),
new CommandLineArgument(S3_RECURSIVE_LIST, CommandLineArgument.ArgumentType.Boolean, Strings.S3Backend.DescriptionRecursiveListShort, Strings.S3Backend.DescriptionRecursiveListLong, "false"),
new CommandLineArgument("auth-password", CommandLineArgument.ArgumentType.Password, Strings.S3Backend.AuthPasswordDescriptionShort, Strings.S3Backend.AuthPasswordDescriptionLong),
new CommandLineArgument("auth-username", CommandLineArgument.ArgumentType.String, Strings.S3Backend.AuthUsernameDescriptionShort, Strings.S3Backend.AuthUsernameDescriptionLong),
};
return normal.Union(exts).ToList();
.. TimeoutOptionsHelper.GetOptions(),
.. exts
];
}
}
@@ -372,37 +363,42 @@ namespace Duplicati.Library.Backend
public void Dispose()
{
s3Client?.Dispose();
s3Client = null;
m_s3Client?.Dispose();
}
#endregion
private IS3Client Connection
{
get { return s3Client; }
get { return m_s3Client; }
}
public Task<string[]> GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new[] { s3Client.GetDnsHost() });
private string GetFullKey(string name)
public Task<string[]> GetDNSNamesAsync(CancellationToken cancelToken)
{
//AWS SDK encodes the filenames correctly
return m_prefix + name;
var host = m_s3Client.GetDnsHost();
return Task.FromResult<string[]>(
string.IsNullOrWhiteSpace(host)
? []
: [host]
);
}
private string GetFullKey(string? name)
//AWS SDK encodes the filenames correctly
=> $"{m_prefix}{name}";
/// <inheritdoc/>
public IAsyncEnumerable<IFileEntry> ListAsync(string path, CancellationToken cancellationToken)
public IAsyncEnumerable<IFileEntry> ListAsync(string? path, CancellationToken cancellationToken)
{
var filterPath = GetFullKey(path);
if (!string.IsNullOrWhiteSpace(filterPath))
filterPath = Util.AppendDirSeparator(filterPath, "/");
return s3Client.ListBucketAsync(m_bucket, filterPath, m_recurseLists, cancellationToken);
return m_s3Client.ListBucketAsync(m_bucket, filterPath, m_recurseLists, cancellationToken);
}
/// <inheritdoc/>
public Task<IFileEntry> GetEntryAsync(string path, CancellationToken cancellationToken)
=> Task.FromResult<IFileEntry>(null);
public Task<IFileEntry?> GetEntryAsync(string path, CancellationToken cancellationToken)
=> Task.FromResult<IFileEntry?>(null);
}
}
+29 -40
View File
@@ -1,35 +1,32 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Collections.Generic;
using Duplicati.Library.Interface;
using Duplicati.Library.Localization.Short;
namespace Duplicati.Library.Backend
{
public class S3Config : IWebModule
{
private const ConfigType DEFAULT_CONFIG_TYPE = ConfigType.Providers;
private static readonly string DEFAULT_CONFIG_TYPE_STR = Enum.GetName(typeof(ConfigType), DEFAULT_CONFIG_TYPE);
private static readonly string DEFAULT_CONFIG_TYPE_STR = DEFAULT_CONFIG_TYPE.ToString();
private const string KEY_CONFIGTYPE = "s3-config";
public enum ConfigType
@@ -48,13 +45,11 @@ namespace Duplicati.Library.Backend
public IDictionary<string, string> Execute(IDictionary<string, string> options)
{
string k;
options.TryGetValue(KEY_CONFIGTYPE, out k);
options.TryGetValue(KEY_CONFIGTYPE, out var k);
if (string.IsNullOrWhiteSpace(k))
k = DEFAULT_CONFIG_TYPE_STR;
ConfigType ct;
if (!Enum.TryParse<ConfigType>(k, true, out ct))
if (!Enum.TryParse<ConfigType>(k, true, out var ct))
ct = DEFAULT_CONFIG_TYPE;
switch (ct)
@@ -70,22 +65,16 @@ namespace Duplicati.Library.Backend
}
}
public string Key { get { return "s3-getconfig"; } }
public string Key => "s3-getconfig";
public string DisplayName { get { return LC.L("S3 configuration module"); } }
public string DisplayName => Strings.S3Config.DisplayName;
public string Description { get { return LC.L("Expose S3 configuration as a web module"); } }
public string Description => Strings.S3Config.Description;
public IList<ICommandLineArgument> SupportedCommands
{
get
{
return new List<ICommandLineArgument>([
new CommandLineArgument(KEY_CONFIGTYPE, CommandLineArgument.ArgumentType.Enumeration, LC.L("The config to get"), LC.L("Provide different config values"), DEFAULT_CONFIG_TYPE_STR, Enum.GetNames(typeof(ConfigType)))
]);
}
}
public IList<ICommandLineArgument> SupportedCommands =>
[
new CommandLineArgument(KEY_CONFIGTYPE, CommandLineArgument.ArgumentType.Enumeration, Strings.S3Config.ConfigTypeShort, Strings.S3Config.ConfigTypeLong, DEFAULT_CONFIG_TYPE_STR, Enum.GetNames(typeof(ConfigType)))
];
#endregion
}
+18 -32
View File
@@ -19,13 +19,9 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Collections.Generic;
using System.Linq;
using Duplicati.Library.Interface;
using Amazon.IdentityManagement;
using Amazon.IdentityManagement.Model;
using Duplicati.Library.Localization.Short;
using Duplicati.Library.Utility;
namespace Duplicati.Library.Backend
@@ -67,71 +63,61 @@ namespace Duplicati.Library.Backend
}
";
public string Key { get { return "s3-iamconfig"; } }
public string Key => "s3-iamconfig";
public string DisplayName { get { return LC.L("S3 IAM support module"); } }
public string DisplayName => Strings.S3IAM.DisplayName;
public string Description { get { return LC.L("Expose S3 IAM manipulation as a web module"); } }
public string Description => Strings.S3IAM.Description;
public IList<ICommandLineArgument> SupportedCommands
{
get
{
return new List<ICommandLineArgument>([
new CommandLineArgument(KEY_OPERATION, CommandLineArgument.ArgumentType.Enumeration, LC.L("The operation to perform"), LC.L("Select the operation to perform"), null, Enum.GetNames(typeof(Operation))),
new CommandLineArgument(KEY_USERNAME, CommandLineArgument.ArgumentType.String, LC.L("The username"), LC.L("The Amazon Access Key ID")),
new CommandLineArgument(KEY_PASSWORD, CommandLineArgument.ArgumentType.String, LC.L("The password"), LC.L("The Amazon Secret Key")),
public IList<ICommandLineArgument> SupportedCommands => new List<ICommandLineArgument>([
new CommandLineArgument(KEY_OPERATION, CommandLineArgument.ArgumentType.Enumeration, Strings.S3IAM.OperationShort, Strings.S3IAM.OperationLong, null, Enum.GetNames(typeof(Operation))),
new CommandLineArgument(KEY_USERNAME, CommandLineArgument.ArgumentType.String, Strings.S3IAM.UsernameShort, Strings.S3IAM.UsernameLong),
new CommandLineArgument(KEY_PASSWORD, CommandLineArgument.ArgumentType.String, Strings.S3IAM.PasswordShort, Strings.S3IAM.PasswordLong)
]);
}
}
public IDictionary<string, string> Execute(IDictionary<string, string> options)
{
options.TryGetValue(KEY_OPERATION, out string operationstring);
options.TryGetValue(KEY_USERNAME, out string username);
options.TryGetValue(KEY_PASSWORD, out string password);
options.TryGetValue(KEY_PATH, out string path);
options.TryGetValue(KEY_OPERATION, out var operationstring);
options.TryGetValue(KEY_USERNAME, out var username);
options.TryGetValue(KEY_PASSWORD, out var password);
options.TryGetValue(KEY_PATH, out var path);
ValidateArgument(operationstring, KEY_OPERATION);
if (!Enum.TryParse(operationstring, true, out Operation operation))
if (!Enum.TryParse<Operation>(operationstring, true, out var operation))
throw new ArgumentException(string.Format("Unable to parse {0} as an operation", operationstring));
switch (operation)
{
case Operation.GetPolicyDoc:
ValidateArgument(path, KEY_PATH);
return GetPolicyDoc(path);
return GetPolicyDoc(path!);
case Operation.CreateIAMUser:
ValidateArgument(username, KEY_USERNAME);
ValidateArgument(password, KEY_PASSWORD);
ValidateArgument(path, KEY_PATH);
return CreateUnprivilegedUser(username, password, path);
return CreateUnprivilegedUser(username!, password!, path!);
default:
ValidateArgument(username, KEY_USERNAME);
ValidateArgument(password, KEY_PASSWORD);
return CanCreateUser(username, password);
return CanCreateUser(username!, password!);
}
}
private static void ValidateArgument(string arg, string type)
private static void ValidateArgument(string? arg, string type)
{
if (string.IsNullOrWhiteSpace(arg))
{
throw new ArgumentNullException(type);
}
}
private static IDictionary<string, string> GetPolicyDoc(string path)
{
return new Dictionary<string, string>
=> new Dictionary<string, string>
{
["doc"] = GeneratePolicyDoc(path)
};
}
private static string GeneratePolicyDoc(string path)
{
@@ -149,7 +135,7 @@ namespace Duplicati.Library.Backend
}
private static Boolean DetermineIfCreateUserIsAllowed(User user, AmazonIdentityManagementServiceClient cl)
private static bool DetermineIfCreateUserIsAllowed(User user, AmazonIdentityManagementServiceClient cl)
{
var simulatePrincipalPolicy = new SimulatePrincipalPolicyRequest
{
+40 -28
View File
@@ -18,17 +18,13 @@
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Collections.Generic;
using System.IO;
using System.Reactive.Linq;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Channels;
using System.Threading.Tasks;
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
using Duplicati.Library.Utility.Options;
using Minio;
using Minio.Exceptions;
@@ -38,13 +34,15 @@ namespace Duplicati.Library.Backend
{
private static readonly string Logtag = Logging.Log.LogTagFromType<S3MinioClient>();
private MinioClient m_client;
private readonly string m_locationConstraint;
private readonly MinioClient m_client;
private readonly string? m_locationConstraint;
private readonly string m_dnsHost;
private readonly TimeoutOptionsHelper.Timeouts m_timeouts;
public S3MinioClient(string awsID, string awsKey, string locationConstraint,
string servername, string storageClass, bool useSSL, Dictionary<string, string> options)
public S3MinioClient(string awsID, string awsKey, string? locationConstraint,
string servername, string? storageClass, bool useSSL, TimeoutOptionsHelper.Timeouts timeouts, Dictionary<string, string?> options)
{
m_timeouts = timeouts;
m_locationConstraint = locationConstraint;
m_client = new MinioClient(
servername,
@@ -84,12 +82,15 @@ namespace Duplicati.Library.Backend
public async IAsyncEnumerable<IFileEntry> ListBucketAsync(string bucketName, string prefix, bool recursive, [EnumeratorCancellation] CancellationToken cancellationToken)
{
ThrowExceptionIfBucketDoesNotExist(bucketName);
await ThrowExceptionIfBucketDoesNotExist(bucketName, cancellationToken).ConfigureAwait(false);
if (!string.IsNullOrWhiteSpace(prefix))
prefix = Util.AppendDirSeparator(prefix, "/");
var observable = m_client.ListObjectsAsync(bucketName, prefix, recursive, cancellationToken);
var observable = await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancellationToken, ct
=> m_client.ListObjectsAsync(bucketName, prefix, recursive, ct)
).ConfigureAwait(false);
await foreach (var obj in ToAsyncEnumerable(observable, cancellationToken).ConfigureAwait(false))
{
if (obj.Key == prefix || !obj.Key.StartsWith(prefix))
@@ -109,7 +110,9 @@ namespace Duplicati.Library.Backend
{
try
{
await m_client.MakeBucketAsync(bucketName, m_locationConstraint, cancelToken).ConfigureAwait(false);
await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct
=> m_client.MakeBucketAsync(bucketName, m_locationConstraint, cancelToken)
).ConfigureAwait(false);
}
catch (MinioException e)
{
@@ -123,7 +126,9 @@ namespace Duplicati.Library.Backend
{
try
{
await m_client.RemoveObjectAsync(bucketName, keyName, cancelToken).ConfigureAwait(false);
await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct
=> m_client.RemoveObjectAsync(bucketName, keyName, ct)
).ConfigureAwait(false);
}
catch (MinioException e)
{
@@ -140,8 +145,9 @@ namespace Duplicati.Library.Backend
{
try
{
await m_client.CopyObjectAsync(bucketName, source,
bucketName, target, cancellationToken: cancelToken).ConfigureAwait(false);
await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct
=> m_client.CopyObjectAsync(bucketName, source, bucketName, target, cancellationToken: ct)
).ConfigureAwait(false);
}
catch (MinioException e)
{
@@ -163,11 +169,17 @@ namespace Duplicati.Library.Backend
// If the object is not found, statObject() throws an exception,
// else it means that the object exists.
// Execution is successful.
await m_client.StatObjectAsync(bucketName, keyName, cancellationToken: cancelToken).ConfigureAwait(false);
await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct
=> m_client.StatObjectAsync(bucketName, keyName, cancellationToken: cancelToken)
).ConfigureAwait(false);
// Get input stream to have content of 'my-objectname' from 'my-bucketname'
await m_client.GetObjectAsync(bucketName, keyName,
(stream) => { Utility.Utility.CopyStream(stream, target); }).ConfigureAwait(false);
(stream) =>
{
using var t = stream.ObserveReadTimeout(m_timeouts.ReadWriteTimeout);
Utility.Utility.CopyStream(t, target);
}).ConfigureAwait(false);
}
catch (MinioException e)
{
@@ -180,7 +192,7 @@ namespace Duplicati.Library.Backend
}
}
public string GetDnsHost()
public string? GetDnsHost()
{
return m_dnsHost;
}
@@ -188,14 +200,15 @@ namespace Duplicati.Library.Backend
public virtual async Task AddFileStreamAsync(string bucketName, string keyName, Stream source,
CancellationToken cancelToken)
{
ThrowExceptionIfBucketDoesNotExist(bucketName);
await ThrowExceptionIfBucketDoesNotExist(bucketName, cancelToken).ConfigureAwait(false);
try
{
using var t = source.ObserveReadTimeout(m_timeouts.ReadWriteTimeout, false);
await m_client.PutObjectAsync(bucketName,
keyName,
source,
source.Length,
t,
t.Length,
"application/octet-stream", cancellationToken: cancelToken);
}
catch (MinioException e)
@@ -212,18 +225,17 @@ namespace Duplicati.Library.Backend
throw new FileMissingException($"File {keyName} not found in bucket {bucketName}");
}
private void ThrowExceptionIfBucketDoesNotExist(string bucketName)
{
if (!m_client.BucketExistsAsync(bucketName).Await())
throw new FolderMissingException($"Bucket {bucketName} does not exist.");
}
private Task ThrowExceptionIfBucketDoesNotExist(string bucketName, CancellationToken cancelToken)
=> Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, async ct =>
{
if (!await m_client.BucketExistsAsync(bucketName, ct))
throw new FolderMissingException($"Bucket {bucketName} does not exist.");
});
#region IDisposable Members
public void Dispose()
{
m_client = null;
}
#endregion
+21
View File
@@ -55,4 +55,25 @@ namespace Duplicati.Library.Backend.Strings
public static string DescriptionRecursiveListLong { get { return LC.L(@"To reduce the number of objects listed, the default is to only list the first level of objects. Use this option to list all objects in the bucket."); } }
public static string UnknownS3ClientError(string client) { return LC.L(@"Unknown S3 client: {0}", client); }
}
internal static class S3Config
{
public static string DisplayName => LC.L("S3 configuration module");
public static string Description => LC.L("Expose S3 configuration as a web module");
public static string ConfigTypeShort => LC.L("The config to get");
public static string ConfigTypeLong => LC.L("Provide different config values");
}
internal static class S3IAM
{
public static string DisplayName => LC.L("S3 IAM support module");
public static string Description => LC.L("Expose S3 IAM manipulation as a web module");
public static string OperationShort => LC.L("The operation to perform");
public static string OperationLong => LC.L("Select the operation to perform");
public static string UsernameShort => LC.L("The username to use");
public static string UsernameLong => LC.L("The Amazon Access Key ID");
public static string PasswordShort => LC.L("The password to use");
public static string PasswordLong => LC.L("The Amazon Secret Key");
}
}
@@ -0,0 +1,65 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
namespace Duplicati.Library.Backend;
/// <summary>
/// Native CIFS/SMB Backend implementation
/// </summary>
public class CIFSBackend : SMBBackend
{
/// <summary>
/// Log tag for the backend
/// </summary>
public static readonly string LOGTAG = Logging.Log.LogTagFromType<CIFSBackend>();
/// <summary>
/// Gets the protocol key for the backend
/// </summary>
public override string ProtocolKey => "cifs";
/// <summary>
/// Gets the display name for the backend
/// </summary>
public override string DisplayName => "CIFS (deprecated)";
/// <summary>
/// Gets the description for the backend
/// </summary>
public override string Description => "Same as SMB backend, but with a different name. Use SMB instead.";
/// <summary>
/// Empty constructor is required for the backend to be loaded by the backend factory
/// </summary>
public CIFSBackend() : base()
{
}
/// <summary>
/// Actual constructor for the backend that accepts the url and options
/// </summary>
/// <param name="url">URL in Duplicati Uri format</param>
/// <param name="options">options to be used in the backend</param>
public CIFSBackend(string url, Dictionary<string, string?> options)
: base(url, options)
{
Logging.Log.WriteWarningMessage(LOGTAG, "DeprecatedCIFSBackend", null, "The CIFS backend is deprecated, please use the SMB backend instead.");
}
}
@@ -4,6 +4,9 @@
<TargetFramework>net8.0</TargetFramework>
<OutputType>Library</OutputType>
<Copyright>Copyright © 2025 Team Duplicati, MIT license</Copyright>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<RootNamespace>Duplicati.Library.Backend.SMB</RootNamespace>
</PropertyGroup>
<ItemGroup>

Some files were not shown because too many files have changed in this diff Show More