// Copyright (C) 2026, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal in the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System.Net; using System.Net.Http.Headers; using System.Runtime.CompilerServices; using Duplicati.Library.Backend.Backblaze.Model; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using Duplicati.Library.Utility; using Duplicati.Library.Utility.Options; using FileEntry = Duplicati.Library.Common.IO.FileEntry; namespace Duplicati.Library.Backend.Backblaze; /// /// Lock mode options for B2 object lock /// public enum B2LockMode { /// /// Governance mode - allows privileged users to bypass retention /// Governance, /// /// Compliance mode - strict retention that cannot be bypassed /// Compliance } /// /// Backblaze B2 Backend /// public class B2 : IStreamingBackend, ILockingBackend, IRenameEnabledBackend { /// /// The option key for specifying the Backblaze B2 account ID /// private const string B2_ID_OPTION = "b2-accountid"; /// /// The option key for specifying the Backblaze B2 application key /// private const string B2_KEY_OPTION = "b2-applicationkey"; /// /// The option key for specifying the number of files to retrieve per API request /// private const string B2_PAGESIZE_OPTION = "b2-page-size"; /// /// The option key for specifying a custom download URL for the B2 service /// private const string B2_DOWNLOAD_URL_OPTION = "b2-download-url"; /// /// The option key for specifying the bucket type when creating new buckets /// private const string B2_CREATE_BUCKET_TYPE_OPTION = "b2-create-bucket-type"; /// /// The option key for specifying the lock mode for the backend /// private const string B2_LOCK_MODE_OPTION = "b2-lock-mode"; /// /// The default bucket type for new buckets - set to private access /// private const string DEFAULT_BUCKET_TYPE = "allPrivate"; /// /// The default number of files to retrieve per API request /// private const int DEFAULT_PAGE_SIZE = 500; /// /// Default retry-after time in seconds for B2 API requests /// private const int B2_RETRY_AFTER_SECONDS = 5; /// /// The name of the B2 bucket being accessed /// private readonly string _bucketName; /// /// The path prefix for all operations within the bucket /// private readonly string _prefix; /// /// URL-encoded version of the path prefix for API requests /// private readonly string _urlencodedPrefix; /// /// The type of bucket (e.g., allPrivate, allPublic) being accessed or created /// private readonly string? _bucketType; /// /// The number of files to retrieve per API request /// private readonly int _pageSize; /// /// Custom download URL for the B2 service, if specified /// private readonly string? _downloadUrl; /// /// The lock mode to use for the backend /// private readonly B2LockMode _lockMode; /// /// Helper class for handling B2 authentication and API requests /// private readonly B2AuthHelper _b2AuthHelper; /// /// The timeout options for the backend /// private readonly TimeoutOptionsHelper.Timeouts _timeouts; /// /// Cached upload URL and authorization token for file uploads /// private UploadUrlResponse? _uploadUrl; /// /// Cache of file listings, mapping filenames to their versions /// private Dictionary>? _filecache; /// /// The current bucket entity containing bucket information and credentials /// private BucketEntity? _bucket; /// /// A scoped instance of Http client to be used with the backend, this will use a /// infinite timeout as a baseline, and requests will be controlled with timeout /// cancellation tokens. It will be disposed when the backend is disposed /// private readonly HttpClient _httpClient; /// /// Cache lock for BucketEntity /// private readonly SemaphoreSlim _cacheLock = new SemaphoreSlim(1, 1); /// /// Empty constructor is required for the backend to be loaded by the backend factory /// public B2() { _bucketName = null!; _prefix = null!; _urlencodedPrefix = null!; _b2AuthHelper = null!; _timeouts = null!; _httpClient = null!; } /// /// Actual constructor for the backend that accepts the url and options /// /// URL in Duplicati Uri format /// options to be used in the backend public B2(string url, Dictionary options) { var uri = new Utility.Uri(url); _bucketName = uri.Host ?? ""; _prefix = Util.AppendDirSeparator("/" + uri.Path, "/"); // For B2 we do not use a leading slash _prefix = _prefix.TrimStart('/'); _urlencodedPrefix = string.Join("/", _prefix.Split(new[] { '/' }).Select(x => Utility.Uri.UrlPathEncode(x))); _bucketType = DEFAULT_BUCKET_TYPE; if (options.TryGetValue(B2_CREATE_BUCKET_TYPE_OPTION, out var option1)) _bucketType = option1; var auth = AuthOptionsHelper.ParseWithAlias(options, uri, B2_ID_OPTION, B2_KEY_OPTION); if (!auth.HasUsername) throw new UserInformationException(Strings.B2.NoB2UserIDError, "B2MissingUserID"); if (!auth.HasPassword) throw new UserInformationException(Strings.B2.NoB2KeyError, "B2MissingKey"); _pageSize = DEFAULT_PAGE_SIZE; if (options.ContainsKey(B2_PAGESIZE_OPTION)) { int.TryParse(options[B2_PAGESIZE_OPTION], out _pageSize); if (_pageSize <= 0) throw new UserInformationException(Strings.B2.InvalidPageSizeError(B2_PAGESIZE_OPTION, options[B2_PAGESIZE_OPTION]), "B2InvalidPageSize"); } _downloadUrl = null; if (options.TryGetValue(B2_DOWNLOAD_URL_OPTION, out var option)) _downloadUrl = option; // Parse lock mode option, default to Governance _lockMode = B2LockMode.Governance; // Default to governance if (options.TryGetValue(B2_LOCK_MODE_OPTION, out var lockModeOption)) { if (Enum.TryParse(lockModeOption, true, out var parsedMode)) { _lockMode = parsedMode; } else { throw new UserInformationException(Strings.B2.InvalidLockModeError(B2_LOCK_MODE_OPTION, lockModeOption), "B2InvalidLockMode"); } } _httpClient = HttpClientHelper.CreateClient(); _httpClient.Timeout = Timeout.InfiniteTimeSpan; _timeouts = TimeoutOptionsHelper.Parse(options); _b2AuthHelper = new B2AuthHelper(auth.Username!, auth.Password!, _httpClient, _timeouts); } /// /// Retrieves the bucket list from the B2 service using the account ID /// and searchs for a match for the configured bucket name. /// /// If not found, throws a FolderMissingException. /// /// Caches the result in the _mBucket field. /// /// CancellationToken that is combined with internal timeout token /// private async Task GetBucketAsync(CancellationToken cancellationToken) { if (_bucket != null) return _bucket; var config = await _b2AuthHelper.GetConfigAsync(cancellationToken).ConfigureAwait(false); await _cacheLock.WaitAsync(cancellationToken).ConfigureAwait(false); try { var buckets = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct => await _b2AuthHelper.PostAndGetJsonDataAsync( $"{config.APIUrl}/b2api/v1/b2_list_buckets", new ListBucketsRequest(accountId: config.AccountID), ct).ConfigureAwait(false) ).ConfigureAwait(false); return _bucket ??= buckets?.Buckets?.FirstOrDefault(x => string.Equals(x.BucketName, _bucketName, StringComparison.OrdinalIgnoreCase)) ?? throw new FolderMissingException(); } finally { _cacheLock.Release(); } } /// /// Retrieves the upload URL and authorization token for the current bucket. /// /// CancellationToken that is combined with internal timeout token private async Task GetUploadUrlDataAsync(CancellationToken cancellationToken) { if (_uploadUrl != null) return _uploadUrl; var config = await _b2AuthHelper.GetConfigAsync(cancellationToken).ConfigureAwait(false); var bucket = await GetBucketAsync(cancellationToken).ConfigureAwait(false); _uploadUrl = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct => await _b2AuthHelper.PostAndGetJsonDataAsync( $"{config.APIUrl}/b2api/v1/b2_get_upload_url", new UploadUrlRequest { BucketID = bucket.BucketID ?? throw new Exception("BucketID is null") }, ct).ConfigureAwait(false) ).ConfigureAwait(false); return _uploadUrl; } /// /// Retrieves the fileID searching by filename. /// /// Filename /// Cancellation Token /// /// Exception thrown when file is not found private async Task GetFileId(string filename, CancellationToken cancelToken) { if (_filecache != null && _filecache.TryGetValue(filename, out var value)) return value.OrderByDescending(x => x.UploadTimestamp).First().FileID ?? throw new InvalidDataException("Missing file ID"); await RebuildFileCache(cancelToken).ConfigureAwait(false); if (_filecache != null && _filecache.TryGetValue(filename, out var value1)) return value1.OrderByDescending(x => x.UploadTimestamp).First().FileID ?? throw new InvalidDataException("Missing file ID"); throw new FileMissingException(); } /// public IList SupportedCommands => new List([ new CommandLineArgument(B2_ID_OPTION, CommandLineArgument.ArgumentType.String, Strings.B2.B2accountidDescriptionShort, Strings.B2.B2accountidDescriptionLong, null, [AuthOptionsHelper.AuthUsernameOption], null), new CommandLineArgument(B2_KEY_OPTION, CommandLineArgument.ArgumentType.Password, Strings.B2.B2applicationkeyDescriptionShort, Strings.B2.B2applicationkeyDescriptionLong, null, [AuthOptionsHelper.AuthPasswordOption], null), new CommandLineArgument(B2_CREATE_BUCKET_TYPE_OPTION, CommandLineArgument.ArgumentType.String, Strings.B2.B2createbuckettypeDescriptionShort, Strings.B2.B2createbuckettypeDescriptionLong, DEFAULT_BUCKET_TYPE), new CommandLineArgument(B2_PAGESIZE_OPTION, CommandLineArgument.ArgumentType.Integer, Strings.B2.B2pagesizeDescriptionShort, Strings.B2.B2pagesizeDescriptionLong, DEFAULT_PAGE_SIZE.ToString()), new CommandLineArgument(B2_DOWNLOAD_URL_OPTION, CommandLineArgument.ArgumentType.String, Strings.B2.B2downloadurlDescriptionShort, Strings.B2.B2downloadurlDescriptionLong), new CommandLineArgument(B2_LOCK_MODE_OPTION, CommandLineArgument.ArgumentType.Enumeration, Strings.B2.B2lockmodeDescriptionShort, Strings.B2.B2lockmodeDescriptionLong, B2LockMode.Governance.ToString()), ..TimeoutOptionsHelper.GetOptions() ]); public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken) { (stream, var sha1, var tmp) = await Utility.Utility.CalculateThrottledStreamHash(stream, "SHA1", cancelToken).ConfigureAwait(false); using var _ = tmp; if (_filecache == null) await RebuildFileCache(cancelToken).ConfigureAwait(false); var uploadUrlData = await GetUploadUrlDataAsync(cancelToken).ConfigureAwait(false); try { using var request = new HttpRequestMessage(HttpMethod.Post, uploadUrlData.UploadUrl); using var timeoutStream = stream.ObserveReadTimeout(_timeouts.ReadWriteTimeout, false); request.Headers.TryAddWithoutValidation("Authorization", uploadUrlData.AuthorizationToken); request.Headers.Add("X-Bz-Content-Sha1", sha1); request.Headers.Add("X-Bz-File-Name", _urlencodedPrefix + Utility.Uri.UrlPathEncode(remotename)); request.Content = new StreamContent(timeoutStream); request.Content.Headers.Add("Content-Type", "application/octet-stream"); request.Content.Headers.Add("Content-Length", timeoutStream.Length.ToString()); var response = await _httpClient.UploadStream(request, cancelToken).ConfigureAwait(false); response.EnsureSuccessStatusCode(); var rdata = await response.Content.ReadAsStreamAsync(cancelToken).ConfigureAwait(false); UploadFileResponse fileinfo; using (var tr = new StreamReader(rdata)) await using (var jr = new Newtonsoft.Json.JsonTextReader(tr)) fileinfo = new Newtonsoft.Json.JsonSerializer().Deserialize(jr) ?? throw new Exception("Failed to parse response"); // Delete old versions if (_filecache!.ContainsKey(remotename)) await DeleteAsync(remotename, cancelToken).ConfigureAwait(false); _filecache[remotename] = [ new FileEntity { FileID = fileinfo.FileID, FileName = fileinfo.FileName, Action = "upload", Size = fileinfo.ContentLength, UploadTimestamp = (long)(DateTime.UtcNow - Utility.Utility.EPOCH).TotalMilliseconds } ]; } catch (Exception ex) { _filecache = null; var code = (int)B2AuthHelper.GetExceptionStatusCode(ex); if (code is >= 500 and <= 599) _uploadUrl = null; throw; } } /// /// Download files from remote /// /// Filename at remote location /// Destination stream to write to /// CancellationToken that is combined with internal timeout token /// FileMissingException when file is not found /// Exceptions arising from either code execution or FileMissingException public async Task GetAsync(string remotename, Stream stream, CancellationToken cancellationToken) { if (_filecache == null || !_filecache.ContainsKey(remotename)) await RebuildFileCache(cancellationToken).ConfigureAwait(false); var config = await _b2AuthHelper.GetConfigAsync(cancellationToken).ConfigureAwait(false); using var request = _filecache != null && _filecache.ContainsKey(remotename) ? await _b2AuthHelper.CreateRequestAsync( $"{config.DownloadUrl}/b2api/v1/b2_download_file_by_id?fileId={Utility.Uri.UrlEncode(await GetFileId(remotename, cancellationToken))}", HttpMethod.Get, cancellationToken).ConfigureAwait(false) : await _b2AuthHelper.CreateRequestAsync( $"{config.DownloadUrl}/{_urlencodedPrefix}{Utility.Uri.UrlPathEncode(remotename)}", HttpMethod.Get, cancellationToken).ConfigureAwait(false); HttpResponseMessage? response = null; try { response = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, ct => _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, ct)) .ConfigureAwait(false); response.EnsureSuccessStatusCode(); await using var responseStream = await response.Content.ReadAsStreamAsync(cancellationToken); using var timeoutStream = responseStream.ObserveReadTimeout(_timeouts.ReadWriteTimeout, false); await timeoutStream.CopyToAsync(stream, cancellationToken).ConfigureAwait(false); } catch (Exception ex) { if (B2AuthHelper.GetExceptionStatusCode(ex) == HttpStatusCode.NotFound) throw new FileMissingException(); await _b2AuthHelper.AttemptParseAndThrowExceptionAsync(ex, response, cancellationToken).ConfigureAwait(false); throw; } finally { response?.Dispose(); } } /// public async Task GetObjectLockUntilAsync(string remotename, CancellationToken cancellationToken) { var fileId = await GetFileId(remotename, cancellationToken).ConfigureAwait(false); var config = await _b2AuthHelper.GetConfigAsync(cancellationToken).ConfigureAwait(false); var response = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct => await _b2AuthHelper.PostAndGetJsonDataAsync( $"{config.APIUrl}/b2api/v2/b2_get_file_info", new GetFileInfoRequest { FileId = fileId }, ct ).ConfigureAwait(false) ).ConfigureAwait(false); if (response?.FileRetention?.Value?.RetainUntilTimestamp is long timestamp && timestamp > 0) return Utility.Utility.EPOCH.AddMilliseconds(timestamp); return null; } /// /// Implementation of interface method for listing remote folder contents /// /// List of IFileEntry with directory listing result private async Task>> RebuildFileCache(CancellationToken cancellationToken) { _filecache = null; var cache = new Dictionary>(); string? nextFileId = null; string? nextFileName = null; var config = await _b2AuthHelper.GetConfigAsync(cancellationToken).ConfigureAwait(false); var listVersionUrl = $"{config.APIUrl}/b2api/v1/b2_list_file_versions"; var listRetryHelper = RetryAfterHelper.CreateOrGetRetryAfterHelper(listVersionUrl); do { try { var resp = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct => await _b2AuthHelper.PostAndGetJsonDataAsync( $"{config.APIUrl}/b2api/v1/b2_list_file_versions", new ListFilesRequest { BucketID = (await GetBucketAsync(cancellationToken)).BucketID ?? throw new Exception("BucketID is null"), MaxFileCount = _pageSize, Prefix = _prefix, StartFileID = nextFileId, StartFileName = nextFileName }, ct ).ConfigureAwait(false) ).ConfigureAwait(false); nextFileId = resp.NextFileID; nextFileName = resp.NextFileName; if (resp.Files == null || resp.Files.Length == 0) break; foreach (var file in resp.Files) { if (string.IsNullOrWhiteSpace(file.FileName)) continue; if (!file.FileName.StartsWith(_prefix, StringComparison.Ordinal)) continue; var name = file.FileName[_prefix.Length..]; if (name.Contains('/')) continue; cache.TryGetValue(name, out var lst); if (lst == null) cache[name] = lst = new System.Collections.Generic.List(1); lst.Add(file); } } catch (TooManyRequestException tex) { // Backblaze's B2 API rate limit reached. Presently they don't add Retry-After headers, we will default to a delay, but respect it if present. if (tex.RetryAfter == null || (tex.RetryAfter.Date == null && tex.RetryAfter.Delta == null)) listRetryHelper.SetRetryAfter( new RetryConditionHeaderValue(TimeSpan.FromSeconds(B2_RETRY_AFTER_SECONDS))); else listRetryHelper.SetRetryAfter(tex.RetryAfter); await listRetryHelper.WaitForRetryAfterAsync(cancellationToken); } } while (nextFileId != null); return _filecache = cache; } /// /// Implementation of interface method for listing remote folder contents /// /// List of IFileEntry with directory listing result public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancellationToken) { var filecache = await RebuildFileCache(cancellationToken).ConfigureAwait(false); foreach (var x in filecache) { var newest = x.Value.OrderByDescending(y => y.UploadTimestamp).First(); var ts = Utility.Utility.EPOCH.AddMilliseconds(newest.UploadTimestamp); yield return new FileEntry(x.Key, newest.Size, ts, ts); } } // public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) { await using FileStream fs = File.OpenRead(filename); await PutAsync(remotename, fs, cancelToken).ConfigureAwait(false); } /// /// Download files from remote /// /// Filename at remote location /// Destination file to write to /// CancellationToken that is combined with internal timeout token /// FileMissingException when file is not found /// Exceptions arising from either code execution or FileMissingException public async Task GetAsync(string remotename, string filename, CancellationToken cancellationToken) { await using var fs = File.Create(filename); await GetAsync(remotename, fs, cancellationToken).ConfigureAwait(false); } /// /// Delete remote file if it exists, if now, throws FileMissingException /// /// filename to be deleted on the remote /// CancellationToken that is combined with internal timeout token /// /// FileMissingException when file is not found /// Exceptions arising from either code execution public async Task DeleteAsync(string remotename, CancellationToken cancellationToken) { try { var filecache = _filecache; if (filecache == null || !filecache.ContainsKey(remotename)) filecache = await RebuildFileCache(cancellationToken).ConfigureAwait(false); if (!filecache.TryGetValue(remotename, out var value)) throw new FileMissingException(); var config = await _b2AuthHelper.GetConfigAsync(cancellationToken).ConfigureAwait(false); foreach (var n in value.OrderBy(x => x.UploadTimestamp)) await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct => await _b2AuthHelper.PostAndGetJsonDataAsync( $"{config.APIUrl}/b2api/v1/b2_delete_file_version", new DeleteRequest { FileName = _prefix + remotename, FileId = n.FileID }, ct ).ConfigureAwait(false) ).ConfigureAwait(false); filecache[remotename].Clear(); } catch { _filecache = null; throw; } } /// public async Task SetObjectLockUntilAsync(string remotename, DateTime lockUntilUtc, CancellationToken cancellationToken) { var fileId = await GetFileId(remotename, cancellationToken).ConfigureAwait(false); var config = await _b2AuthHelper.GetConfigAsync(cancellationToken).ConfigureAwait(false); var res = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct => await _b2AuthHelper.PostAndGetJsonDataAsync( $"{config.APIUrl}/b2api/v2/b2_update_file_retention", new UpdateFileRetentionRequest { FileId = fileId, FileName = _prefix + remotename, FileRetention = new FileRetention { Mode = _lockMode.ToString().ToLower(), RetainUntilTimestamp = (long)(lockUntilUtc.ToUniversalTime() - Utility.Utility.EPOCH).TotalMilliseconds }, BypassGovernance = false }, ct ).ConfigureAwait(false) ).ConfigureAwait(false); if (res.FileRetention?.RetainUntilTimestamp == null) throw new Exception("Failed to set object lock, call succeeded but no retention info returned"); } /// /// Performs test with backend (internally it uses the List() command, which by definition means the backend is working) /// /// Cancellation Token public Task TestAsync(CancellationToken cancelToken) => this.TestReadWritePermissionsAsync(cancelToken); /// /// Create remote folder /// /// CancellationToken that will be combined with internal timeout token /// /// public async Task CreateFolderAsync(CancellationToken cancellationToken) { var config = await _b2AuthHelper.GetConfigAsync(cancellationToken); _bucket = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct => await _b2AuthHelper.PostAndGetJsonDataAsync( $"{config.APIUrl}/b2api/v1/b2_create_bucket", new BucketEntity { AccountID = config.AccountID, BucketName = _bucketName, BucketType = _bucketType }, ct ).ConfigureAwait(false) ).ConfigureAwait(false); } /// public string DisplayName => Strings.B2.DisplayName; /// public string ProtocolKey => "b2"; /// public string Description => Strings.B2.Description; /// public bool SupportsStreaming => true; /// public async Task GetDNSNamesAsync(CancellationToken cancelToken) { var config = await _b2AuthHelper.GetConfigAsync(cancelToken); return new string?[] { B2AuthHelper.AUTH_URL, config.APIUrl, config.DownloadUrl }.WhereNotNullOrWhiteSpace() .Select(x => new System.Uri(x).Host) .Distinct() .ToArray(); } public async Task RenameAsync(string oldname, string newname, CancellationToken cancellationToken) { var sourceFileId = await GetFileId(oldname, cancellationToken).ConfigureAwait(false); var config = await _b2AuthHelper.GetConfigAsync(cancellationToken).ConfigureAwait(false); // Copy file await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct => await _b2AuthHelper.PostAndGetJsonDataAsync( $"{config.APIUrl}/b2api/v1/b2_copy_file", new CopyFileRequest(sourceFileId, _urlencodedPrefix + Utility.Uri.UrlPathEncode(newname)), ct).ConfigureAwait(false) ).ConfigureAwait(false); // Delete old file await DeleteAsync(oldname, cancellationToken).ConfigureAwait(false); } /// /// Handles disposal of the backend's resources. /// public void Dispose() { try { _httpClient?.Dispose(); } catch { // ignored } } }