// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
using System.Net;
using System.Net.Http.Headers;
using System.Reflection;
using Duplicati.Library.Backend.Backblaze.Model;
using FileEntry = Duplicati.Library.Common.IO.FileEntry;
using System.Runtime.CompilerServices;
namespace Duplicati.Library.Backend.Backblaze;
///
/// Backblaze B2 Backend
///
public class B2 : IStreamingBackend, ITimeoutExemptBackend
{
///
/// The default timeout in seconds for LIST/CreateFolder operations
///
private const int SHORT_OPERATION_TIMEOUT_SECONDS = 30;
///
/// The option key for specifying the Backblaze B2 account ID
///
private const string B2_ID_OPTION = "b2-accountid";
///
/// The option key for specifying the Backblaze B2 application key
///
private const string B2_KEY_OPTION = "b2-applicationkey";
///
/// The option key for specifying the number of files to retrieve per API request
///
private const string B2_PAGESIZE_OPTION = "b2-page-size";
///
/// The option key for specifying a custom download URL for the B2 service
///
private const string B2_DOWNLOAD_URL_OPTION = "b2-download-url";
///
/// The option key for specifying the bucket type when creating new buckets
///
private const string B2_CREATE_BUCKET_TYPE_OPTION = "b2-create-bucket-type";
///
/// The default bucket type for new buckets - set to private access
///
private const string DEFAULT_BUCKET_TYPE = "allPrivate";
///
/// The default number of files to retrieve per API request
///
private const int DEFAULT_PAGE_SIZE = 500;
///
/// Recommended chunk size as per Backblaze B2 documentation (100MB)
///
private const int B2_RECOMMENDED_CHUNK_SIZE = 100 * 1024 * 1024;
///
/// Default retry-after time in seconds for B2 API requests
///
private const int B2_RETRY_AFTER_SECONDS = 5;
///
/// The name of the B2 bucket being accessed
///
private readonly string _bucketName;
///
/// The path prefix for all operations within the bucket
///
private readonly string _prefix;
///
/// URL-encoded version of the path prefix for API requests
///
private readonly string _urlencodedPrefix;
///
/// The type of bucket (e.g., allPrivate, allPublic) being accessed or created
///
private readonly string _bucketType;
///
/// The number of files to retrieve per API request
///
private readonly int _pageSize;
///
/// Custom download URL for the B2 service, if specified
///
private readonly string _downloadUrl;
///
/// Helper class for handling B2 authentication and API requests
///
private readonly B2AuthHelper _b2AuthHelper;
///
/// Cached upload URL and authorization token for file uploads
///
private UploadUrlResponse _uploadUrl;
///
/// Cache of file listings, mapping filenames to their versions
///
private Dictionary> _filecache;
///
/// The current bucket entity containing bucket information and credentials
///
private BucketEntity _bucket;
///
/// A scoped instance of Http client to be used with the backend, this will use a
/// infinite timeout as a baseline, and requests will be controlled with timeout
/// cancellation tokens. It will be disposed when the backend is disposed
///
private readonly HttpClient _httpClient;
///
/// Cache lock for BucketEntity
///
private readonly SemaphoreSlim _cacheLock = new SemaphoreSlim(1, 1);
///
/// Empty constructor is required for the backend to be loaded by the backend factory
///
public B2()
{
}
///
/// Actual constructor for the backend that accepts the url and options
///
/// URL in Duplicati Uri format
/// options to be used in the backend
public B2(string url, Dictionary options)
{
var uri = new Utility.Uri(url);
_bucketName = uri.Host;
_prefix = Util.AppendDirSeparator("/" + uri.Path, "/");
// For B2 we do not use a leading slash
while (_prefix.StartsWith("/", StringComparison.Ordinal))
_prefix = _prefix.Substring(1);
_urlencodedPrefix = string.Join("/", _prefix.Split(new[] { '/' }).Select(x => Utility.Uri.UrlPathEncode(x)));
_bucketType = DEFAULT_BUCKET_TYPE;
if (options.TryGetValue(B2_CREATE_BUCKET_TYPE_OPTION, out var option1))
_bucketType = option1;
string accountId = null;
string accountKey = null;
// Takes the account ID and key from the options, or from the URL with the cascading precedence
if (options.TryGetValue("auth-username", out var authUsernameOption))
accountId = authUsernameOption;
if (options.TryGetValue("auth-password", out var authPasswordOption))
accountKey = authPasswordOption;
if (options.TryGetValue(B2_ID_OPTION, out var accountIdOption))
accountId = accountIdOption;
if (options.TryGetValue(B2_KEY_OPTION, out var accountKeyOption))
accountKey = accountKeyOption;
if (!string.IsNullOrEmpty(uri.Username))
accountId = uri.Username;
if (!string.IsNullOrEmpty(uri.Password))
accountKey = uri.Password;
if (string.IsNullOrEmpty(accountId))
throw new UserInformationException(Strings.B2.NoB2UserIDError, "B2MissingUserID");
if (string.IsNullOrEmpty(accountKey))
throw new UserInformationException(Strings.B2.NoB2KeyError, "B2MissingKey");
_pageSize = DEFAULT_PAGE_SIZE;
if (options.ContainsKey(B2_PAGESIZE_OPTION))
{
int.TryParse(options[B2_PAGESIZE_OPTION], out _pageSize);
if (_pageSize <= 0)
throw new UserInformationException(Strings.B2.InvalidPageSizeError(B2_PAGESIZE_OPTION, options[B2_PAGESIZE_OPTION]), "B2InvalidPageSize");
}
_downloadUrl = null;
if (options.TryGetValue(B2_DOWNLOAD_URL_OPTION, out var option)) _downloadUrl = option;
_httpClient = HttpClientHelper.CreateClient();
_httpClient.Timeout = Timeout.InfiniteTimeSpan;
_b2AuthHelper = new B2AuthHelper(accountId, accountKey, _httpClient);
}
///
/// Retrieves the bucket list from the B2 service using the account ID
/// and searchs for a match for the configured bucket name.
///
/// If not found, throws a FolderMissingException.
///
/// Caches the result in the _mBucket field.
///
/// CancellationToken that is combined with internal timeout token
///
private async Task GetBucketAsync(CancellationToken cancellationToken)
{
if (_bucket != null) return _bucket;
await _cacheLock.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
using var timeoutToken = new CancellationTokenSource();
timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS));
using var combinedCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken);
var buckets = await _b2AuthHelper.PostAndGetJsonDataAsync(
$"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_list_buckets",
new ListBucketsRequest(accountId: _b2AuthHelper.AccountId),
combinedCancellationToken.Token).ConfigureAwait(false);
return _bucket ??= buckets?.Buckets?.FirstOrDefault(x =>
string.Equals(x.BucketName, _bucketName, StringComparison.OrdinalIgnoreCase))
?? throw new FolderMissingException();
}
finally
{
_cacheLock.Release();
}
}
///
/// Retrieves the upload URL and authorization token for the current bucket.
///
/// CancellationToken that is combined with internal timeout token
private async Task GetUploadUrlDataAsync(CancellationToken cancellationToken)
{
if (_uploadUrl != null)
return _uploadUrl;
using var timeoutToken = new CancellationTokenSource();
timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS));
using var combinedCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken);
_uploadUrl = await _b2AuthHelper.PostAndGetJsonDataAsync(
$"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_get_upload_url",
new UploadUrlRequest { BucketID = GetBucketAsync(cancellationToken).Await().BucketID },
combinedCancellationToken.Token
).ConfigureAwait(false);
return _uploadUrl;
}
///
/// Retrieves the fileID searching by filename.
///
/// Filename
/// Cancellation Token
///
/// Exception thrown when file is not found
private async Task GetFileId(string filename, CancellationToken cancelToken)
{
if (_filecache != null && _filecache.TryGetValue(filename, out var value))
return value.OrderByDescending(x => x.UploadTimestamp).First().FileID;
await RebuildFileCache(cancelToken).ConfigureAwait(false);
if (_filecache != null && _filecache.TryGetValue(filename, out var value1))
return value1.OrderByDescending(x => x.UploadTimestamp).First().FileID;
throw new FileMissingException();
}
///
/// Returns the DownloadURL, either cached, or making a call to the server's /b2_authorize_account
///
private string DownloadUrl => string.IsNullOrEmpty(_downloadUrl) ? _b2AuthHelper.DownloadUrl : _downloadUrl;
///
public IList SupportedCommands =>
new List([
new CommandLineArgument(B2_ID_OPTION, CommandLineArgument.ArgumentType.String, Strings.B2.B2accountidDescriptionShort, Strings.B2.B2accountidDescriptionLong, null,
["auth-password"], null),
new CommandLineArgument(B2_KEY_OPTION, CommandLineArgument.ArgumentType.Password, Strings.B2.B2applicationkeyDescriptionShort, Strings.B2.B2applicationkeyDescriptionLong, null,
["auth-username"], null),
new CommandLineArgument("auth-password", CommandLineArgument.ArgumentType.Password, Strings.B2.AuthPasswordDescriptionShort, Strings.B2.AuthPasswordDescriptionLong),
new CommandLineArgument("auth-username", CommandLineArgument.ArgumentType.String, Strings.B2.AuthUsernameDescriptionShort, Strings.B2.AuthUsernameDescriptionLong),
new CommandLineArgument(B2_CREATE_BUCKET_TYPE_OPTION, CommandLineArgument.ArgumentType.String, Strings.B2.B2createbuckettypeDescriptionShort, Strings.B2.B2createbuckettypeDescriptionLong, DEFAULT_BUCKET_TYPE),
new CommandLineArgument(B2_PAGESIZE_OPTION, CommandLineArgument.ArgumentType.Integer, Strings.B2.B2pagesizeDescriptionShort, Strings.B2.B2pagesizeDescriptionLong, DEFAULT_PAGE_SIZE.ToString()),
new CommandLineArgument(B2_DOWNLOAD_URL_OPTION, CommandLineArgument.ArgumentType.String, Strings.B2.B2downloadurlDescriptionShort, Strings.B2.B2downloadurlDescriptionLong)
]);
public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken)
{
TempFile tmp = null;
var measure = stream;
while (measure is OverrideableStream os &&
os.GetType().GetField("m_basestream", BindingFlags.NonPublic | BindingFlags.Instance)?.GetValue(os) is Stream baseStream)
{
measure = baseStream;
}
if (measure == null)
throw new Exception($"Unable to unwrap stream from: {stream.GetType()}");
string sha1;
if (measure.CanSeek)
{
var p = measure.Position;
// Compute the hash
using (var hashalg = HashFactory.CreateHasher("SHA1"))
sha1 = Library.Utility.Utility.ByteArrayAsHexString(hashalg.ComputeHash(measure));
measure.Position = p;
}
else
{
// No seeking possible, use a temp file
tmp = new TempFile();
await using (var sr = File.OpenWrite(tmp))
using (var hasher = HashFactory.CreateHasher("SHA1"))
await using (var hc = new HashCalculatingStream(measure, hasher))
{
await Utility.Utility.CopyStreamAsync(hc, sr, cancelToken).ConfigureAwait(false);
sha1 = hc.GetFinalHashString();
}
stream = File.OpenRead(tmp);
}
if (_filecache == null)
await RebuildFileCache(cancelToken).ConfigureAwait(false);
var uploadUrlData = await GetUploadUrlDataAsync(cancelToken).ConfigureAwait(false);
try
{
// For PutAsync, no timeout is specified. The only thing that can stop it is the cancellation token
using var request = new HttpRequestMessage(HttpMethod.Post, uploadUrlData.UploadUrl);
request.Headers.TryAddWithoutValidation("Authorization", uploadUrlData.AuthorizationToken);
request.Headers.Add("X-Bz-Content-Sha1", sha1);
request.Headers.Add("X-Bz-File-Name", _urlencodedPrefix + Utility.Uri.UrlPathEncode(remotename));
request.Content = new StreamContent(stream, B2_RECOMMENDED_CHUNK_SIZE);
request.Content.Headers.Add("Content-Type", "application/octet-stream");
request.Content.Headers.Add("Content-Length", stream.Length.ToString());
var response = await _httpClient.UploadStream(request, cancelToken).ConfigureAwait(false);
var rdata = await response.Content.ReadAsStreamAsync(cancelToken).ConfigureAwait(false);
UploadFileResponse fileinfo;
using (var tr = new StreamReader(rdata))
await using (var jr = new Newtonsoft.Json.JsonTextReader(tr))
fileinfo = new Newtonsoft.Json.JsonSerializer().Deserialize(jr);
// Delete old versions
if (_filecache!.ContainsKey(remotename))
await DeleteAsync(remotename, cancelToken).ConfigureAwait(false);
_filecache[remotename] =
[
new FileEntity
{
FileID = fileinfo.FileID,
FileName = fileinfo.FileName,
Action = "upload",
Size = fileinfo.ContentLength,
UploadTimestamp = (long)(DateTime.UtcNow - Utility.Utility.EPOCH).TotalMilliseconds
}
];
}
catch (Exception ex)
{
_filecache = null;
var code = (int)B2AuthHelper.GetExceptionStatusCode(ex);
if (code is >= 500 and <= 599)
_uploadUrl = null;
throw;
}
finally
{
tmp?.Dispose();
}
}
///
/// Download files from remote
///
/// Filename at remote location
/// Destination stream to write to
/// CancellationToken that is combined with internal timeout token
/// FileMissingException when file is not found
/// Exceptions arising from either code execution or FileMissingException
public async Task GetAsync(string remotename, Stream stream, CancellationToken cancellationToken)
{
if (_filecache == null || !_filecache.ContainsKey(remotename))
await RebuildFileCache(cancellationToken).ConfigureAwait(false);
using var request = _filecache != null && _filecache.ContainsKey(remotename)
? _b2AuthHelper.CreateRequest(
$"{DownloadUrl}/b2api/v1/b2_download_file_by_id?fileId={Utility.Uri.UrlEncode(await GetFileId(remotename, cancellationToken))}")
: _b2AuthHelper.CreateRequest(
$"{DownloadUrl}/{_urlencodedPrefix}{Utility.Uri.UrlPathEncode(remotename)}");
HttpResponseMessage response = null;
try
{
// For GetAsync, no timeout is specified. The only thing that can stop it is the cancellation token
response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken)
.ConfigureAwait(false);
response.EnsureSuccessStatusCode();
await using var responseStream = await response.Content.ReadAsStreamAsync(cancellationToken);
await responseStream.CopyToAsync(stream, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
if (B2AuthHelper.GetExceptionStatusCode(ex) == HttpStatusCode.NotFound)
throw new FileMissingException();
_b2AuthHelper.AttemptParseAndThrowException(ex, response);
throw;
}
finally
{
response?.Dispose();
}
}
///
/// Implementation of interface method for listing remote folder contents
///
/// List of IFileEntry with directory listing result
private async Task RebuildFileCache(CancellationToken cancellationToken)
{
_filecache = null;
var cache = new Dictionary>();
string nextFileId = null;
string nextFileName = null;
string listVersionUrl = $"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_list_file_versions";
var listRetryHelper = RetryAfterHelper.CreateOrGetRetryAfterHelper(listVersionUrl);
do
{
try
{
using var timeoutToken = new CancellationTokenSource();
timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS));
using var combinedCancellationToken =
CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken);
var resp = await _b2AuthHelper.PostAndGetJsonDataAsync(
$"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_list_file_versions",
new ListFilesRequest
{
BucketID = GetBucketAsync(cancellationToken).Await().BucketID,
MaxFileCount = _pageSize,
Prefix = _prefix,
StartFileID = nextFileId,
StartFileName = nextFileName
},
combinedCancellationToken.Token
).ConfigureAwait(false);
nextFileId = resp.NextFileID;
nextFileName = resp.NextFileName;
if (resp.Files == null || resp.Files.Length == 0)
break;
foreach (var file in resp.Files)
{
if (!file.FileName.StartsWith(_prefix, StringComparison.Ordinal))
continue;
var name = file.FileName[_prefix.Length..];
if (name.Contains('/'))
continue;
cache.TryGetValue(name, out var lst);
if (lst == null)
cache[name] = lst = new System.Collections.Generic.List(1);
lst.Add(file);
}
}
catch (TooManyRequestException tex)
{
// Backblaze's B2 API rate limit reached. Presently they don't add Retry-After headers, we will default to a delay, but respect it if present.
if (tex.RetryAfter == null || (tex.RetryAfter.Date == null && tex.RetryAfter.Delta == null))
listRetryHelper.SetRetryAfter(
new RetryConditionHeaderValue(TimeSpan.FromSeconds(B2_RETRY_AFTER_SECONDS)));
else
listRetryHelper.SetRetryAfter(tex.RetryAfter);
await listRetryHelper.WaitForRetryAfterAsync(cancellationToken);
}
} while (nextFileId != null);
_filecache = cache;
}
///
/// Implementation of interface method for listing remote folder contents
///
/// List of IFileEntry with directory listing result
public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancellationToken)
{
await RebuildFileCache(cancellationToken).ConfigureAwait(false);
foreach (var x in _filecache)
{
var newest = x.Value.OrderByDescending(y => y.UploadTimestamp).First();
var ts = Utility.Utility.EPOCH.AddMilliseconds(newest.UploadTimestamp);
yield return new FileEntry(x.Key, newest.Size, ts, ts);
}
}
//
public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken)
{
await using FileStream fs = File.OpenRead(filename);
await PutAsync(remotename, fs, cancelToken).ConfigureAwait(false);
}
///
/// Download files from remote
///
/// Filename at remote location
/// Destination file to write to
/// CancellationToken that is combined with internal timeout token
/// FileMissingException when file is not found
/// Exceptions arising from either code execution or FileMissingException
public async Task GetAsync(string remotename, string filename, CancellationToken cancellationToken)
{
await using var fs = File.Create(filename);
await GetAsync(remotename, fs, cancellationToken).ConfigureAwait(false);
}
///
/// Delete remote file if it exists, if now, throws FileMissingException
///
/// filename to be deleted on the remote
/// CancellationToken that is combined with internal timeout token
///
/// FileMissingException when file is not found
/// Exceptions arising from either code execution
public async Task DeleteAsync(string remotename, CancellationToken cancellationToken)
{
try
{
using var timeoutToken = new CancellationTokenSource();
timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS));
using var combinedCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken);
if (_filecache == null || !_filecache.ContainsKey(remotename))
await RebuildFileCache(cancellationToken).ConfigureAwait(false);
if (!_filecache.TryGetValue(remotename, out var value))
throw new FileMissingException();
foreach (var n in value.OrderBy(x => x.UploadTimestamp))
await _b2AuthHelper.PostAndGetJsonDataAsync(
$"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_delete_file_version",
new DeleteRequest
{
FileName = _prefix + remotename,
FileId = n.FileID
},
combinedCancellationToken.Token
).ConfigureAwait(false);
_filecache[remotename].Clear();
}
catch
{
_filecache = null;
throw;
}
}
///
/// Performs test with backend (internally it uses the List() command, which by definition means the backend is working)
///
/// Cancellation Token
public Task TestAsync(CancellationToken cancelToken)
=> this.TestListAsync(cancelToken);
///
/// Create remote folder
///
/// CancellationToken that will be combined with internal timeout token
///
///
public async Task CreateFolderAsync(CancellationToken cancellationToken)
{
using var timeoutToken = new CancellationTokenSource();
timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS));
using var combinedCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken);
_bucket = await _b2AuthHelper.PostAndGetJsonDataAsync(
$"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_create_bucket",
new BucketEntity
{
AccountID = _b2AuthHelper.AccountId,
BucketName = _bucketName,
BucketType = _bucketType
},
combinedCancellationToken.Token
).ConfigureAwait(false);
}
///
public string DisplayName => Strings.B2.DisplayName;
///
public string ProtocolKey => "b2";
///
public string Description => Strings.B2.Description;
///
public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new string[] {
new System.Uri(B2AuthHelper.AUTH_URL).Host,
_b2AuthHelper?.ApiDnsName,
_b2AuthHelper?.DownloadDnsName
}.Where(x => !string.IsNullOrEmpty(x))
.ToArray());
///
/// Handles disposal of the backend's resources.
///
public void Dispose()
{
try
{
_httpClient?.Dispose();
}
catch
{
// ignored
}
}
}