diff --git a/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs b/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs index b01c483ed..2a55637f9 100644 --- a/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs +++ b/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs @@ -56,7 +56,7 @@ namespace Duplicati.Library.Backend.AliyunOSS { _timeouts = TimeoutOptionsHelper.Parse(options); - var uri = new Utility.Uri(url?.Trim()); + var uri = new Utility.Uri(url?.Trim() ?? ""); var prefix = uri.HostAndPath?.TrimPath(); var auth = AuthOptionsHelper.ParseWithAlias(options, uri, OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY_SECRET) diff --git a/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs b/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs index cd32134a4..f107d9143 100644 --- a/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs +++ b/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs @@ -88,7 +88,7 @@ namespace Duplicati.Library.Backend.AzureBlob var uri = new Utility.Uri(url); uri.RequireHost(); - var containerName = uri.Host.ToLowerInvariant(); + var containerName = (uri.Host ?? "").ToLowerInvariant(); var auth = AuthOptionsHelper.ParseWithAlias(options, uri, AZURE_ACCOUNT_NAME_OPTION, AZURE_ACCESS_KEY_OPTION); var timeouts = TimeoutOptionsHelper.Parse(options); diff --git a/Duplicati/Library/Backend/Backblaze/B2.cs b/Duplicati/Library/Backend/Backblaze/B2.cs index 07fe993c6..a138b7376 100644 --- a/Duplicati/Library/Backend/Backblaze/B2.cs +++ b/Duplicati/Library/Backend/Backblaze/B2.cs @@ -1,629 +1,629 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - -using System.Net; -using System.Net.Http.Headers; -using System.Runtime.CompilerServices; -using Duplicati.Library.Backend.Backblaze.Model; -using Duplicati.Library.Common.IO; -using Duplicati.Library.Interface; -using Duplicati.Library.Utility; -using Duplicati.Library.Utility.Options; -using FileEntry = Duplicati.Library.Common.IO.FileEntry; - -namespace Duplicati.Library.Backend.Backblaze; - -/// -/// Backblaze B2 Backend -/// -public class B2 : IStreamingBackend -{ - /// - /// The option key for specifying the Backblaze B2 account ID - /// - private const string B2_ID_OPTION = "b2-accountid"; - - /// - /// The option key for specifying the Backblaze B2 application key - /// - private const string B2_KEY_OPTION = "b2-applicationkey"; - - /// - /// The option key for specifying the number of files to retrieve per API request - /// - private const string B2_PAGESIZE_OPTION = "b2-page-size"; - - /// - /// The option key for specifying a custom download URL for the B2 service - /// - private const string B2_DOWNLOAD_URL_OPTION = "b2-download-url"; - - /// - /// The option key for specifying the bucket type when creating new buckets - /// - private const string B2_CREATE_BUCKET_TYPE_OPTION = "b2-create-bucket-type"; - - /// - /// The default bucket type for new buckets - set to private access - /// - private const string DEFAULT_BUCKET_TYPE = "allPrivate"; - - /// - /// The default number of files to retrieve per API request - /// - private const int DEFAULT_PAGE_SIZE = 500; - - /// - /// Default retry-after time in seconds for B2 API requests - /// - private const int B2_RETRY_AFTER_SECONDS = 5; - - /// - /// The name of the B2 bucket being accessed - /// - private readonly string _bucketName; - - /// - /// The path prefix for all operations within the bucket - /// - private readonly string _prefix; - - /// - /// URL-encoded version of the path prefix for API requests - /// - private readonly string _urlencodedPrefix; - - /// - /// The type of bucket (e.g., allPrivate, allPublic) being accessed or created - /// - private readonly string? _bucketType; - - /// - /// The number of files to retrieve per API request - /// - private readonly int _pageSize; - - /// - /// Custom download URL for the B2 service, if specified - /// - private readonly string? _downloadUrl; - - /// - /// Helper class for handling B2 authentication and API requests - /// - private readonly B2AuthHelper _b2AuthHelper; - - /// - /// The timeout options for the backend - /// - private readonly TimeoutOptionsHelper.Timeouts _timeouts; - - /// - /// Cached upload URL and authorization token for file uploads - /// - private UploadUrlResponse? _uploadUrl; - - /// - /// Cache of file listings, mapping filenames to their versions - /// - private Dictionary>? _filecache; - - /// - /// The current bucket entity containing bucket information and credentials - /// - private BucketEntity? _bucket; - - /// - /// A scoped instance of Http client to be used with the backend, this will use a - /// infinite timeout as a baseline, and requests will be controlled with timeout - /// cancellation tokens. It will be disposed when the backend is disposed - /// - private readonly HttpClient _httpClient; - - /// - /// Cache lock for BucketEntity - /// - private readonly SemaphoreSlim _cacheLock = new SemaphoreSlim(1, 1); - - /// - /// Empty constructor is required for the backend to be loaded by the backend factory - /// - public B2() - { - _bucketName = null!; - _prefix = null!; - _urlencodedPrefix = null!; - _b2AuthHelper = null!; - _timeouts = null!; - _httpClient = null!; - } - - /// - /// Actual constructor for the backend that accepts the url and options - /// - /// URL in Duplicati Uri format - /// options to be used in the backend - public B2(string url, Dictionary options) - { - var uri = new Utility.Uri(url); - - _bucketName = uri.Host; - _prefix = Util.AppendDirSeparator("/" + uri.Path, "/"); - - // For B2 we do not use a leading slash - _prefix = _prefix.TrimStart('/'); - - _urlencodedPrefix = string.Join("/", _prefix.Split(new[] { '/' }).Select(x => Utility.Uri.UrlPathEncode(x))); - - _bucketType = DEFAULT_BUCKET_TYPE; - if (options.TryGetValue(B2_CREATE_BUCKET_TYPE_OPTION, out var option1)) - _bucketType = option1; - - var auth = AuthOptionsHelper.ParseWithAlias(options, uri, B2_ID_OPTION, B2_KEY_OPTION); - - if (!auth.HasUsername) - throw new UserInformationException(Strings.B2.NoB2UserIDError, "B2MissingUserID"); - - if (!auth.HasPassword) - throw new UserInformationException(Strings.B2.NoB2KeyError, "B2MissingKey"); - - _pageSize = DEFAULT_PAGE_SIZE; - if (options.ContainsKey(B2_PAGESIZE_OPTION)) - { - int.TryParse(options[B2_PAGESIZE_OPTION], out _pageSize); - - if (_pageSize <= 0) - throw new UserInformationException(Strings.B2.InvalidPageSizeError(B2_PAGESIZE_OPTION, options[B2_PAGESIZE_OPTION]), "B2InvalidPageSize"); - } - - _downloadUrl = null; - if (options.TryGetValue(B2_DOWNLOAD_URL_OPTION, out var option)) _downloadUrl = option; - - _httpClient = HttpClientHelper.CreateClient(); - _httpClient.Timeout = Timeout.InfiniteTimeSpan; - - _timeouts = TimeoutOptionsHelper.Parse(options); - _b2AuthHelper = new B2AuthHelper(auth.Username!, auth.Password!, _httpClient, _timeouts); - - } - - /// - /// Retrieves the bucket list from the B2 service using the account ID - /// and searchs for a match for the configured bucket name. - /// - /// If not found, throws a FolderMissingException. - /// - /// Caches the result in the _mBucket field. - /// - /// CancellationToken that is combined with internal timeout token - /// - private async Task GetBucketAsync(CancellationToken cancellationToken) - { - if (_bucket != null) - return _bucket; - - var config = await _b2AuthHelper.GetConfigAsync(cancellationToken).ConfigureAwait(false); - await _cacheLock.WaitAsync(cancellationToken).ConfigureAwait(false); - - try - { - var buckets = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct - => await _b2AuthHelper.PostAndGetJsonDataAsync( - $"{config.APIUrl}/b2api/v1/b2_list_buckets", - new ListBucketsRequest(accountId: config.AccountID), - ct).ConfigureAwait(false) - ).ConfigureAwait(false); - - return _bucket ??= buckets?.Buckets?.FirstOrDefault(x => - string.Equals(x.BucketName, _bucketName, StringComparison.OrdinalIgnoreCase)) - ?? throw new FolderMissingException(); - } - finally - { - _cacheLock.Release(); - } - } - - /// - /// Retrieves the upload URL and authorization token for the current bucket. - /// - /// CancellationToken that is combined with internal timeout token - private async Task GetUploadUrlDataAsync(CancellationToken cancellationToken) - { - if (_uploadUrl != null) - return _uploadUrl; - - var config = await _b2AuthHelper.GetConfigAsync(cancellationToken).ConfigureAwait(false); - var bucket = await GetBucketAsync(cancellationToken).ConfigureAwait(false); - - _uploadUrl = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct - => await _b2AuthHelper.PostAndGetJsonDataAsync( - $"{config.APIUrl}/b2api/v1/b2_get_upload_url", - new UploadUrlRequest - { - BucketID = bucket.BucketID - ?? throw new Exception("BucketID is null") - }, - ct).ConfigureAwait(false) - ).ConfigureAwait(false); - - return _uploadUrl; - } - - /// - /// Retrieves the fileID searching by filename. - /// - /// Filename - /// Cancellation Token - /// - /// Exception thrown when file is not found - private async Task GetFileId(string filename, CancellationToken cancelToken) - { - if (_filecache != null && _filecache.TryGetValue(filename, out var value)) - return value.OrderByDescending(x => x.UploadTimestamp).First().FileID - ?? throw new InvalidDataException("Missing file ID"); - - await RebuildFileCache(cancelToken).ConfigureAwait(false); - - if (_filecache != null && _filecache.TryGetValue(filename, out var value1)) - return value1.OrderByDescending(x => x.UploadTimestamp).First().FileID - ?? throw new InvalidDataException("Missing file ID"); - - throw new FileMissingException(); - } - - /// - public IList SupportedCommands => - new List([ - new CommandLineArgument(B2_ID_OPTION, CommandLineArgument.ArgumentType.String, Strings.B2.B2accountidDescriptionShort, Strings.B2.B2accountidDescriptionLong, null, [AuthOptionsHelper.AuthUsernameOption], null), - new CommandLineArgument(B2_KEY_OPTION, CommandLineArgument.ArgumentType.Password, Strings.B2.B2applicationkeyDescriptionShort, Strings.B2.B2applicationkeyDescriptionLong, null, [AuthOptionsHelper.AuthPasswordOption], null), - new CommandLineArgument(B2_CREATE_BUCKET_TYPE_OPTION, CommandLineArgument.ArgumentType.String, Strings.B2.B2createbuckettypeDescriptionShort, Strings.B2.B2createbuckettypeDescriptionLong, DEFAULT_BUCKET_TYPE), - new CommandLineArgument(B2_PAGESIZE_OPTION, CommandLineArgument.ArgumentType.Integer, Strings.B2.B2pagesizeDescriptionShort, Strings.B2.B2pagesizeDescriptionLong, DEFAULT_PAGE_SIZE.ToString()), - new CommandLineArgument(B2_DOWNLOAD_URL_OPTION, CommandLineArgument.ArgumentType.String, Strings.B2.B2downloadurlDescriptionShort, Strings.B2.B2downloadurlDescriptionLong), - ..TimeoutOptionsHelper.GetOptions() - ]); - - public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken) - { - (stream, var sha1, var tmp) = await Utility.Utility.CalculateThrottledStreamHash(stream, "SHA1", cancelToken).ConfigureAwait(false); - using var _ = tmp; - - if (_filecache == null) - await RebuildFileCache(cancelToken).ConfigureAwait(false); - - var uploadUrlData = await GetUploadUrlDataAsync(cancelToken).ConfigureAwait(false); - try - { - using var request = new HttpRequestMessage(HttpMethod.Post, uploadUrlData.UploadUrl); - using var timeoutStream = stream.ObserveReadTimeout(_timeouts.ReadWriteTimeout, false); - - request.Headers.TryAddWithoutValidation("Authorization", uploadUrlData.AuthorizationToken); - request.Headers.Add("X-Bz-Content-Sha1", sha1); - request.Headers.Add("X-Bz-File-Name", _urlencodedPrefix + Utility.Uri.UrlPathEncode(remotename)); - request.Content = new StreamContent(timeoutStream); - - request.Content.Headers.Add("Content-Type", "application/octet-stream"); - request.Content.Headers.Add("Content-Length", timeoutStream.Length.ToString()); - - var response = await _httpClient.UploadStream(request, cancelToken).ConfigureAwait(false); - var rdata = await response.Content.ReadAsStreamAsync(cancelToken).ConfigureAwait(false); - - UploadFileResponse fileinfo; - using (var tr = new StreamReader(rdata)) - await using (var jr = new Newtonsoft.Json.JsonTextReader(tr)) - fileinfo = new Newtonsoft.Json.JsonSerializer().Deserialize(jr) - ?? throw new Exception("Failed to parse response"); - - // Delete old versions - if (_filecache!.ContainsKey(remotename)) - await DeleteAsync(remotename, cancelToken).ConfigureAwait(false); - - _filecache[remotename] = - [ - new FileEntity - { - FileID = fileinfo.FileID, - FileName = fileinfo.FileName, - Action = "upload", - Size = fileinfo.ContentLength, - UploadTimestamp = (long)(DateTime.UtcNow - Utility.Utility.EPOCH).TotalMilliseconds - } - ]; - } - catch (Exception ex) - { - _filecache = null; - - var code = (int)B2AuthHelper.GetExceptionStatusCode(ex); - if (code is >= 500 and <= 599) - _uploadUrl = null; - - throw; - } - } - - /// - /// Download files from remote - /// - /// Filename at remote location - /// Destination stream to write to - /// CancellationToken that is combined with internal timeout token - /// FileMissingException when file is not found - /// Exceptions arising from either code execution or FileMissingException - public async Task GetAsync(string remotename, Stream stream, CancellationToken cancellationToken) - { - if (_filecache == null || !_filecache.ContainsKey(remotename)) - await RebuildFileCache(cancellationToken).ConfigureAwait(false); - - var config = await _b2AuthHelper.GetConfigAsync(cancellationToken).ConfigureAwait(false); - - using var request = _filecache != null && _filecache.ContainsKey(remotename) - ? await _b2AuthHelper.CreateRequestAsync( - $"{config.DownloadUrl}/b2api/v1/b2_download_file_by_id?fileId={Utility.Uri.UrlEncode(await GetFileId(remotename, cancellationToken))}", HttpMethod.Get, cancellationToken).ConfigureAwait(false) - : await _b2AuthHelper.CreateRequestAsync( - $"{config.DownloadUrl}/{_urlencodedPrefix}{Utility.Uri.UrlPathEncode(remotename)}", HttpMethod.Get, cancellationToken).ConfigureAwait(false); - - HttpResponseMessage? response = null; - try - { - response = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, ct => _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, ct)) - .ConfigureAwait(false); - response.EnsureSuccessStatusCode(); - await using var responseStream = await response.Content.ReadAsStreamAsync(cancellationToken); - using var timeoutStream = responseStream.ObserveReadTimeout(_timeouts.ReadWriteTimeout, false); - await timeoutStream.CopyToAsync(stream, cancellationToken).ConfigureAwait(false); - } - catch (Exception ex) - { - if (B2AuthHelper.GetExceptionStatusCode(ex) == HttpStatusCode.NotFound) - throw new FileMissingException(); - - await _b2AuthHelper.AttemptParseAndThrowExceptionAsync(ex, response, cancellationToken).ConfigureAwait(false); - throw; - } - finally - { - response?.Dispose(); - } - } - - /// - /// Implementation of interface method for listing remote folder contents - /// - /// List of IFileEntry with directory listing result - private async Task>> RebuildFileCache(CancellationToken cancellationToken) - { - _filecache = null; - var cache = new Dictionary>(); - string? nextFileId = null; - string? nextFileName = null; - var config = await _b2AuthHelper.GetConfigAsync(cancellationToken).ConfigureAwait(false); - var listVersionUrl = $"{config.APIUrl}/b2api/v1/b2_list_file_versions"; - - var listRetryHelper = RetryAfterHelper.CreateOrGetRetryAfterHelper(listVersionUrl); - - do - { - try - { - var resp = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct - => await _b2AuthHelper.PostAndGetJsonDataAsync( - $"{config.APIUrl}/b2api/v1/b2_list_file_versions", - new ListFilesRequest - { - BucketID = (await GetBucketAsync(cancellationToken)).BucketID - ?? throw new Exception("BucketID is null"), - MaxFileCount = _pageSize, - Prefix = _prefix, - StartFileID = nextFileId, - StartFileName = nextFileName - }, - ct - ).ConfigureAwait(false) - ).ConfigureAwait(false); - - nextFileId = resp.NextFileID; - nextFileName = resp.NextFileName; - - if (resp.Files == null || resp.Files.Length == 0) - break; - - foreach (var file in resp.Files) - { - if (string.IsNullOrWhiteSpace(file.FileName)) - continue; - - if (!file.FileName.StartsWith(_prefix, StringComparison.Ordinal)) - continue; - - var name = file.FileName[_prefix.Length..]; - if (name.Contains('/')) - continue; - - cache.TryGetValue(name, out var lst); - if (lst == null) - cache[name] = lst = new System.Collections.Generic.List(1); - lst.Add(file); - } - } - catch (TooManyRequestException tex) - { - // Backblaze's B2 API rate limit reached. Presently they don't add Retry-After headers, we will default to a delay, but respect it if present. - if (tex.RetryAfter == null || (tex.RetryAfter.Date == null && tex.RetryAfter.Delta == null)) - listRetryHelper.SetRetryAfter( - new RetryConditionHeaderValue(TimeSpan.FromSeconds(B2_RETRY_AFTER_SECONDS))); - else - listRetryHelper.SetRetryAfter(tex.RetryAfter); - - await listRetryHelper.WaitForRetryAfterAsync(cancellationToken); - } - } while (nextFileId != null); - - return _filecache = cache; - } - - /// - /// Implementation of interface method for listing remote folder contents - /// - /// List of IFileEntry with directory listing result - public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancellationToken) - { - var filecache = await RebuildFileCache(cancellationToken).ConfigureAwait(false); - foreach (var x in filecache) - { - var newest = x.Value.OrderByDescending(y => y.UploadTimestamp).First(); - var ts = Utility.Utility.EPOCH.AddMilliseconds(newest.UploadTimestamp); - yield return new FileEntry(x.Key, newest.Size, ts, ts); - } - } - - // - public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) - { - await using FileStream fs = File.OpenRead(filename); - await PutAsync(remotename, fs, cancelToken).ConfigureAwait(false); - } - - /// - /// Download files from remote - /// - /// Filename at remote location - /// Destination file to write to - /// CancellationToken that is combined with internal timeout token - /// FileMissingException when file is not found - /// Exceptions arising from either code execution or FileMissingException - public async Task GetAsync(string remotename, string filename, CancellationToken cancellationToken) - { - await using var fs = File.Create(filename); - await GetAsync(remotename, fs, cancellationToken).ConfigureAwait(false); - } - - /// - /// Delete remote file if it exists, if now, throws FileMissingException - /// - /// filename to be deleted on the remote - /// CancellationToken that is combined with internal timeout token - /// - /// FileMissingException when file is not found - /// Exceptions arising from either code execution - public async Task DeleteAsync(string remotename, CancellationToken cancellationToken) - { - try - { - var filecache = _filecache; - if (filecache == null || !filecache.ContainsKey(remotename)) - filecache = await RebuildFileCache(cancellationToken).ConfigureAwait(false); - - if (!filecache.TryGetValue(remotename, out var value)) - throw new FileMissingException(); - - var config = await _b2AuthHelper.GetConfigAsync(cancellationToken).ConfigureAwait(false); - - foreach (var n in value.OrderBy(x => x.UploadTimestamp)) - await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct - => await _b2AuthHelper.PostAndGetJsonDataAsync( - $"{config.APIUrl}/b2api/v1/b2_delete_file_version", - new DeleteRequest - { - FileName = _prefix + remotename, - FileId = n.FileID - }, - ct - ).ConfigureAwait(false) - ).ConfigureAwait(false); - - filecache[remotename].Clear(); - } - catch - { - _filecache = null; - throw; - } - } - - /// - /// Performs test with backend (internally it uses the List() command, which by definition means the backend is working) - /// - /// Cancellation Token - public Task TestAsync(CancellationToken cancelToken) - => this.TestReadWritePermissionsAsync(cancelToken); - - /// - /// Create remote folder - /// - /// CancellationToken that will be combined with internal timeout token - /// - /// - public async Task CreateFolderAsync(CancellationToken cancellationToken) - { - var config = await _b2AuthHelper.GetConfigAsync(cancellationToken); - _bucket = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct - => await _b2AuthHelper.PostAndGetJsonDataAsync( - $"{config.APIUrl}/b2api/v1/b2_create_bucket", - new BucketEntity - { - AccountID = config.AccountID, - BucketName = _bucketName, - BucketType = _bucketType - }, - ct - ).ConfigureAwait(false) - ).ConfigureAwait(false); - } - - /// - public string DisplayName => Strings.B2.DisplayName; - - /// - public string ProtocolKey => "b2"; - - /// - public string Description => Strings.B2.Description; - - /// - public async Task GetDNSNamesAsync(CancellationToken cancelToken) - { - var config = await _b2AuthHelper.GetConfigAsync(cancelToken); - return new string?[] { - B2AuthHelper.AUTH_URL, - config.APIUrl, - config.DownloadUrl - }.WhereNotNullOrWhiteSpace() - .Select(x => new System.Uri(x).Host) - .Distinct() - .ToArray(); - } - - /// - /// Handles disposal of the backend's resources. - /// - public void Dispose() - { - try - { - _httpClient?.Dispose(); - } - catch - { - // ignored - } - } -} +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using System.Net; +using System.Net.Http.Headers; +using System.Runtime.CompilerServices; +using Duplicati.Library.Backend.Backblaze.Model; +using Duplicati.Library.Common.IO; +using Duplicati.Library.Interface; +using Duplicati.Library.Utility; +using Duplicati.Library.Utility.Options; +using FileEntry = Duplicati.Library.Common.IO.FileEntry; + +namespace Duplicati.Library.Backend.Backblaze; + +/// +/// Backblaze B2 Backend +/// +public class B2 : IStreamingBackend +{ + /// + /// The option key for specifying the Backblaze B2 account ID + /// + private const string B2_ID_OPTION = "b2-accountid"; + + /// + /// The option key for specifying the Backblaze B2 application key + /// + private const string B2_KEY_OPTION = "b2-applicationkey"; + + /// + /// The option key for specifying the number of files to retrieve per API request + /// + private const string B2_PAGESIZE_OPTION = "b2-page-size"; + + /// + /// The option key for specifying a custom download URL for the B2 service + /// + private const string B2_DOWNLOAD_URL_OPTION = "b2-download-url"; + + /// + /// The option key for specifying the bucket type when creating new buckets + /// + private const string B2_CREATE_BUCKET_TYPE_OPTION = "b2-create-bucket-type"; + + /// + /// The default bucket type for new buckets - set to private access + /// + private const string DEFAULT_BUCKET_TYPE = "allPrivate"; + + /// + /// The default number of files to retrieve per API request + /// + private const int DEFAULT_PAGE_SIZE = 500; + + /// + /// Default retry-after time in seconds for B2 API requests + /// + private const int B2_RETRY_AFTER_SECONDS = 5; + + /// + /// The name of the B2 bucket being accessed + /// + private readonly string _bucketName; + + /// + /// The path prefix for all operations within the bucket + /// + private readonly string _prefix; + + /// + /// URL-encoded version of the path prefix for API requests + /// + private readonly string _urlencodedPrefix; + + /// + /// The type of bucket (e.g., allPrivate, allPublic) being accessed or created + /// + private readonly string? _bucketType; + + /// + /// The number of files to retrieve per API request + /// + private readonly int _pageSize; + + /// + /// Custom download URL for the B2 service, if specified + /// + private readonly string? _downloadUrl; + + /// + /// Helper class for handling B2 authentication and API requests + /// + private readonly B2AuthHelper _b2AuthHelper; + + /// + /// The timeout options for the backend + /// + private readonly TimeoutOptionsHelper.Timeouts _timeouts; + + /// + /// Cached upload URL and authorization token for file uploads + /// + private UploadUrlResponse? _uploadUrl; + + /// + /// Cache of file listings, mapping filenames to their versions + /// + private Dictionary>? _filecache; + + /// + /// The current bucket entity containing bucket information and credentials + /// + private BucketEntity? _bucket; + + /// + /// A scoped instance of Http client to be used with the backend, this will use a + /// infinite timeout as a baseline, and requests will be controlled with timeout + /// cancellation tokens. It will be disposed when the backend is disposed + /// + private readonly HttpClient _httpClient; + + /// + /// Cache lock for BucketEntity + /// + private readonly SemaphoreSlim _cacheLock = new SemaphoreSlim(1, 1); + + /// + /// Empty constructor is required for the backend to be loaded by the backend factory + /// + public B2() + { + _bucketName = null!; + _prefix = null!; + _urlencodedPrefix = null!; + _b2AuthHelper = null!; + _timeouts = null!; + _httpClient = null!; + } + + /// + /// Actual constructor for the backend that accepts the url and options + /// + /// URL in Duplicati Uri format + /// options to be used in the backend + public B2(string url, Dictionary options) + { + var uri = new Utility.Uri(url); + + _bucketName = uri.Host ?? ""; + _prefix = Util.AppendDirSeparator("/" + uri.Path, "/"); + + // For B2 we do not use a leading slash + _prefix = _prefix.TrimStart('/'); + + _urlencodedPrefix = string.Join("/", _prefix.Split(new[] { '/' }).Select(x => Utility.Uri.UrlPathEncode(x))); + + _bucketType = DEFAULT_BUCKET_TYPE; + if (options.TryGetValue(B2_CREATE_BUCKET_TYPE_OPTION, out var option1)) + _bucketType = option1; + + var auth = AuthOptionsHelper.ParseWithAlias(options, uri, B2_ID_OPTION, B2_KEY_OPTION); + + if (!auth.HasUsername) + throw new UserInformationException(Strings.B2.NoB2UserIDError, "B2MissingUserID"); + + if (!auth.HasPassword) + throw new UserInformationException(Strings.B2.NoB2KeyError, "B2MissingKey"); + + _pageSize = DEFAULT_PAGE_SIZE; + if (options.ContainsKey(B2_PAGESIZE_OPTION)) + { + int.TryParse(options[B2_PAGESIZE_OPTION], out _pageSize); + + if (_pageSize <= 0) + throw new UserInformationException(Strings.B2.InvalidPageSizeError(B2_PAGESIZE_OPTION, options[B2_PAGESIZE_OPTION]), "B2InvalidPageSize"); + } + + _downloadUrl = null; + if (options.TryGetValue(B2_DOWNLOAD_URL_OPTION, out var option)) _downloadUrl = option; + + _httpClient = HttpClientHelper.CreateClient(); + _httpClient.Timeout = Timeout.InfiniteTimeSpan; + + _timeouts = TimeoutOptionsHelper.Parse(options); + _b2AuthHelper = new B2AuthHelper(auth.Username!, auth.Password!, _httpClient, _timeouts); + + } + + /// + /// Retrieves the bucket list from the B2 service using the account ID + /// and searchs for a match for the configured bucket name. + /// + /// If not found, throws a FolderMissingException. + /// + /// Caches the result in the _mBucket field. + /// + /// CancellationToken that is combined with internal timeout token + /// + private async Task GetBucketAsync(CancellationToken cancellationToken) + { + if (_bucket != null) + return _bucket; + + var config = await _b2AuthHelper.GetConfigAsync(cancellationToken).ConfigureAwait(false); + await _cacheLock.WaitAsync(cancellationToken).ConfigureAwait(false); + + try + { + var buckets = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct + => await _b2AuthHelper.PostAndGetJsonDataAsync( + $"{config.APIUrl}/b2api/v1/b2_list_buckets", + new ListBucketsRequest(accountId: config.AccountID), + ct).ConfigureAwait(false) + ).ConfigureAwait(false); + + return _bucket ??= buckets?.Buckets?.FirstOrDefault(x => + string.Equals(x.BucketName, _bucketName, StringComparison.OrdinalIgnoreCase)) + ?? throw new FolderMissingException(); + } + finally + { + _cacheLock.Release(); + } + } + + /// + /// Retrieves the upload URL and authorization token for the current bucket. + /// + /// CancellationToken that is combined with internal timeout token + private async Task GetUploadUrlDataAsync(CancellationToken cancellationToken) + { + if (_uploadUrl != null) + return _uploadUrl; + + var config = await _b2AuthHelper.GetConfigAsync(cancellationToken).ConfigureAwait(false); + var bucket = await GetBucketAsync(cancellationToken).ConfigureAwait(false); + + _uploadUrl = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct + => await _b2AuthHelper.PostAndGetJsonDataAsync( + $"{config.APIUrl}/b2api/v1/b2_get_upload_url", + new UploadUrlRequest + { + BucketID = bucket.BucketID + ?? throw new Exception("BucketID is null") + }, + ct).ConfigureAwait(false) + ).ConfigureAwait(false); + + return _uploadUrl; + } + + /// + /// Retrieves the fileID searching by filename. + /// + /// Filename + /// Cancellation Token + /// + /// Exception thrown when file is not found + private async Task GetFileId(string filename, CancellationToken cancelToken) + { + if (_filecache != null && _filecache.TryGetValue(filename, out var value)) + return value.OrderByDescending(x => x.UploadTimestamp).First().FileID + ?? throw new InvalidDataException("Missing file ID"); + + await RebuildFileCache(cancelToken).ConfigureAwait(false); + + if (_filecache != null && _filecache.TryGetValue(filename, out var value1)) + return value1.OrderByDescending(x => x.UploadTimestamp).First().FileID + ?? throw new InvalidDataException("Missing file ID"); + + throw new FileMissingException(); + } + + /// + public IList SupportedCommands => + new List([ + new CommandLineArgument(B2_ID_OPTION, CommandLineArgument.ArgumentType.String, Strings.B2.B2accountidDescriptionShort, Strings.B2.B2accountidDescriptionLong, null, [AuthOptionsHelper.AuthUsernameOption], null), + new CommandLineArgument(B2_KEY_OPTION, CommandLineArgument.ArgumentType.Password, Strings.B2.B2applicationkeyDescriptionShort, Strings.B2.B2applicationkeyDescriptionLong, null, [AuthOptionsHelper.AuthPasswordOption], null), + new CommandLineArgument(B2_CREATE_BUCKET_TYPE_OPTION, CommandLineArgument.ArgumentType.String, Strings.B2.B2createbuckettypeDescriptionShort, Strings.B2.B2createbuckettypeDescriptionLong, DEFAULT_BUCKET_TYPE), + new CommandLineArgument(B2_PAGESIZE_OPTION, CommandLineArgument.ArgumentType.Integer, Strings.B2.B2pagesizeDescriptionShort, Strings.B2.B2pagesizeDescriptionLong, DEFAULT_PAGE_SIZE.ToString()), + new CommandLineArgument(B2_DOWNLOAD_URL_OPTION, CommandLineArgument.ArgumentType.String, Strings.B2.B2downloadurlDescriptionShort, Strings.B2.B2downloadurlDescriptionLong), + ..TimeoutOptionsHelper.GetOptions() + ]); + + public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken) + { + (stream, var sha1, var tmp) = await Utility.Utility.CalculateThrottledStreamHash(stream, "SHA1", cancelToken).ConfigureAwait(false); + using var _ = tmp; + + if (_filecache == null) + await RebuildFileCache(cancelToken).ConfigureAwait(false); + + var uploadUrlData = await GetUploadUrlDataAsync(cancelToken).ConfigureAwait(false); + try + { + using var request = new HttpRequestMessage(HttpMethod.Post, uploadUrlData.UploadUrl); + using var timeoutStream = stream.ObserveReadTimeout(_timeouts.ReadWriteTimeout, false); + + request.Headers.TryAddWithoutValidation("Authorization", uploadUrlData.AuthorizationToken); + request.Headers.Add("X-Bz-Content-Sha1", sha1); + request.Headers.Add("X-Bz-File-Name", _urlencodedPrefix + Utility.Uri.UrlPathEncode(remotename)); + request.Content = new StreamContent(timeoutStream); + + request.Content.Headers.Add("Content-Type", "application/octet-stream"); + request.Content.Headers.Add("Content-Length", timeoutStream.Length.ToString()); + + var response = await _httpClient.UploadStream(request, cancelToken).ConfigureAwait(false); + var rdata = await response.Content.ReadAsStreamAsync(cancelToken).ConfigureAwait(false); + + UploadFileResponse fileinfo; + using (var tr = new StreamReader(rdata)) + await using (var jr = new Newtonsoft.Json.JsonTextReader(tr)) + fileinfo = new Newtonsoft.Json.JsonSerializer().Deserialize(jr) + ?? throw new Exception("Failed to parse response"); + + // Delete old versions + if (_filecache!.ContainsKey(remotename)) + await DeleteAsync(remotename, cancelToken).ConfigureAwait(false); + + _filecache[remotename] = + [ + new FileEntity + { + FileID = fileinfo.FileID, + FileName = fileinfo.FileName, + Action = "upload", + Size = fileinfo.ContentLength, + UploadTimestamp = (long)(DateTime.UtcNow - Utility.Utility.EPOCH).TotalMilliseconds + } + ]; + } + catch (Exception ex) + { + _filecache = null; + + var code = (int)B2AuthHelper.GetExceptionStatusCode(ex); + if (code is >= 500 and <= 599) + _uploadUrl = null; + + throw; + } + } + + /// + /// Download files from remote + /// + /// Filename at remote location + /// Destination stream to write to + /// CancellationToken that is combined with internal timeout token + /// FileMissingException when file is not found + /// Exceptions arising from either code execution or FileMissingException + public async Task GetAsync(string remotename, Stream stream, CancellationToken cancellationToken) + { + if (_filecache == null || !_filecache.ContainsKey(remotename)) + await RebuildFileCache(cancellationToken).ConfigureAwait(false); + + var config = await _b2AuthHelper.GetConfigAsync(cancellationToken).ConfigureAwait(false); + + using var request = _filecache != null && _filecache.ContainsKey(remotename) + ? await _b2AuthHelper.CreateRequestAsync( + $"{config.DownloadUrl}/b2api/v1/b2_download_file_by_id?fileId={Utility.Uri.UrlEncode(await GetFileId(remotename, cancellationToken))}", HttpMethod.Get, cancellationToken).ConfigureAwait(false) + : await _b2AuthHelper.CreateRequestAsync( + $"{config.DownloadUrl}/{_urlencodedPrefix}{Utility.Uri.UrlPathEncode(remotename)}", HttpMethod.Get, cancellationToken).ConfigureAwait(false); + + HttpResponseMessage? response = null; + try + { + response = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, ct => _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, ct)) + .ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + await using var responseStream = await response.Content.ReadAsStreamAsync(cancellationToken); + using var timeoutStream = responseStream.ObserveReadTimeout(_timeouts.ReadWriteTimeout, false); + await timeoutStream.CopyToAsync(stream, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + if (B2AuthHelper.GetExceptionStatusCode(ex) == HttpStatusCode.NotFound) + throw new FileMissingException(); + + await _b2AuthHelper.AttemptParseAndThrowExceptionAsync(ex, response, cancellationToken).ConfigureAwait(false); + throw; + } + finally + { + response?.Dispose(); + } + } + + /// + /// Implementation of interface method for listing remote folder contents + /// + /// List of IFileEntry with directory listing result + private async Task>> RebuildFileCache(CancellationToken cancellationToken) + { + _filecache = null; + var cache = new Dictionary>(); + string? nextFileId = null; + string? nextFileName = null; + var config = await _b2AuthHelper.GetConfigAsync(cancellationToken).ConfigureAwait(false); + var listVersionUrl = $"{config.APIUrl}/b2api/v1/b2_list_file_versions"; + + var listRetryHelper = RetryAfterHelper.CreateOrGetRetryAfterHelper(listVersionUrl); + + do + { + try + { + var resp = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct + => await _b2AuthHelper.PostAndGetJsonDataAsync( + $"{config.APIUrl}/b2api/v1/b2_list_file_versions", + new ListFilesRequest + { + BucketID = (await GetBucketAsync(cancellationToken)).BucketID + ?? throw new Exception("BucketID is null"), + MaxFileCount = _pageSize, + Prefix = _prefix, + StartFileID = nextFileId, + StartFileName = nextFileName + }, + ct + ).ConfigureAwait(false) + ).ConfigureAwait(false); + + nextFileId = resp.NextFileID; + nextFileName = resp.NextFileName; + + if (resp.Files == null || resp.Files.Length == 0) + break; + + foreach (var file in resp.Files) + { + if (string.IsNullOrWhiteSpace(file.FileName)) + continue; + + if (!file.FileName.StartsWith(_prefix, StringComparison.Ordinal)) + continue; + + var name = file.FileName[_prefix.Length..]; + if (name.Contains('/')) + continue; + + cache.TryGetValue(name, out var lst); + if (lst == null) + cache[name] = lst = new System.Collections.Generic.List(1); + lst.Add(file); + } + } + catch (TooManyRequestException tex) + { + // Backblaze's B2 API rate limit reached. Presently they don't add Retry-After headers, we will default to a delay, but respect it if present. + if (tex.RetryAfter == null || (tex.RetryAfter.Date == null && tex.RetryAfter.Delta == null)) + listRetryHelper.SetRetryAfter( + new RetryConditionHeaderValue(TimeSpan.FromSeconds(B2_RETRY_AFTER_SECONDS))); + else + listRetryHelper.SetRetryAfter(tex.RetryAfter); + + await listRetryHelper.WaitForRetryAfterAsync(cancellationToken); + } + } while (nextFileId != null); + + return _filecache = cache; + } + + /// + /// Implementation of interface method for listing remote folder contents + /// + /// List of IFileEntry with directory listing result + public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancellationToken) + { + var filecache = await RebuildFileCache(cancellationToken).ConfigureAwait(false); + foreach (var x in filecache) + { + var newest = x.Value.OrderByDescending(y => y.UploadTimestamp).First(); + var ts = Utility.Utility.EPOCH.AddMilliseconds(newest.UploadTimestamp); + yield return new FileEntry(x.Key, newest.Size, ts, ts); + } + } + + // + public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) + { + await using FileStream fs = File.OpenRead(filename); + await PutAsync(remotename, fs, cancelToken).ConfigureAwait(false); + } + + /// + /// Download files from remote + /// + /// Filename at remote location + /// Destination file to write to + /// CancellationToken that is combined with internal timeout token + /// FileMissingException when file is not found + /// Exceptions arising from either code execution or FileMissingException + public async Task GetAsync(string remotename, string filename, CancellationToken cancellationToken) + { + await using var fs = File.Create(filename); + await GetAsync(remotename, fs, cancellationToken).ConfigureAwait(false); + } + + /// + /// Delete remote file if it exists, if now, throws FileMissingException + /// + /// filename to be deleted on the remote + /// CancellationToken that is combined with internal timeout token + /// + /// FileMissingException when file is not found + /// Exceptions arising from either code execution + public async Task DeleteAsync(string remotename, CancellationToken cancellationToken) + { + try + { + var filecache = _filecache; + if (filecache == null || !filecache.ContainsKey(remotename)) + filecache = await RebuildFileCache(cancellationToken).ConfigureAwait(false); + + if (!filecache.TryGetValue(remotename, out var value)) + throw new FileMissingException(); + + var config = await _b2AuthHelper.GetConfigAsync(cancellationToken).ConfigureAwait(false); + + foreach (var n in value.OrderBy(x => x.UploadTimestamp)) + await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct + => await _b2AuthHelper.PostAndGetJsonDataAsync( + $"{config.APIUrl}/b2api/v1/b2_delete_file_version", + new DeleteRequest + { + FileName = _prefix + remotename, + FileId = n.FileID + }, + ct + ).ConfigureAwait(false) + ).ConfigureAwait(false); + + filecache[remotename].Clear(); + } + catch + { + _filecache = null; + throw; + } + } + + /// + /// Performs test with backend (internally it uses the List() command, which by definition means the backend is working) + /// + /// Cancellation Token + public Task TestAsync(CancellationToken cancelToken) + => this.TestReadWritePermissionsAsync(cancelToken); + + /// + /// Create remote folder + /// + /// CancellationToken that will be combined with internal timeout token + /// + /// + public async Task CreateFolderAsync(CancellationToken cancellationToken) + { + var config = await _b2AuthHelper.GetConfigAsync(cancellationToken); + _bucket = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct + => await _b2AuthHelper.PostAndGetJsonDataAsync( + $"{config.APIUrl}/b2api/v1/b2_create_bucket", + new BucketEntity + { + AccountID = config.AccountID, + BucketName = _bucketName, + BucketType = _bucketType + }, + ct + ).ConfigureAwait(false) + ).ConfigureAwait(false); + } + + /// + public string DisplayName => Strings.B2.DisplayName; + + /// + public string ProtocolKey => "b2"; + + /// + public string Description => Strings.B2.Description; + + /// + public async Task GetDNSNamesAsync(CancellationToken cancelToken) + { + var config = await _b2AuthHelper.GetConfigAsync(cancelToken); + return new string?[] { + B2AuthHelper.AUTH_URL, + config.APIUrl, + config.DownloadUrl + }.WhereNotNullOrWhiteSpace() + .Select(x => new System.Uri(x).Host) + .Distinct() + .ToArray(); + } + + /// + /// Handles disposal of the backend's resources. + /// + public void Dispose() + { + try + { + _httpClient?.Dispose(); + } + catch + { + // ignored + } + } +} diff --git a/Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs b/Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs index 150092986..27667e9c8 100644 --- a/Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs +++ b/Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs @@ -62,7 +62,7 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage { var uri = new Utility.Uri(url); - m_bucket = uri.Host; + m_bucket = uri.Host ?? ""; m_prefix = Util.AppendDirSeparator("/" + uri.Path, "/"); // For GCS we do not use a leading slash diff --git a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs index fc6972e2a..cf4585b3d 100644 --- a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs +++ b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs @@ -58,7 +58,7 @@ namespace Duplicati.Library.Backend client.Timeout = Timeout.InfiniteTimeSpan; return client; }); - + /// /// The timeout options /// @@ -86,7 +86,7 @@ namespace Duplicati.Library.Backend public Idrivee2Backend(string url, Dictionary options) { var uri = new Utility.Uri(url); - _bucket = uri.Host; + _bucket = uri.Host ?? ""; _prefix = uri.Path; _prefix = _prefix.Trim(); if (_prefix.Length != 0) diff --git a/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs b/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs index 134058b10..c14caefaf 100644 --- a/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs +++ b/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs @@ -110,7 +110,7 @@ public class OpenStackStorage : IStreamingBackend { var uri = new Uri(url); - m_container = uri.Host; + m_container = uri.Host ?? ""; m_prefix = Util.AppendDirSeparator("/" + uri.Path, "/"); _timeouts = TimeoutOptionsHelper.Parse(options); diff --git a/Duplicati/Library/Backend/Rclone/Rclone.cs b/Duplicati/Library/Backend/Rclone/Rclone.cs index 7bbbcfd60..71debac14 100644 --- a/Duplicati/Library/Backend/Rclone/Rclone.cs +++ b/Duplicati/Library/Backend/Rclone/Rclone.cs @@ -74,7 +74,7 @@ namespace Duplicati.Library.Backend if (string.IsNullOrWhiteSpace(local_repo)) local_repo = "local"; if (string.IsNullOrWhiteSpace(remote_repo)) - remote_repo = uri.Host; + remote_repo = uri.Host ?? ""; if (string.IsNullOrWhiteSpace(remote_path)) remote_path = uri.Path; if (string.IsNullOrWhiteSpace(rclone_executable)) diff --git a/Duplicati/Library/Backend/S3/S3Backend.cs b/Duplicati/Library/Backend/S3/S3Backend.cs index d13952901..262a8433d 100644 --- a/Duplicati/Library/Backend/S3/S3Backend.cs +++ b/Duplicati/Library/Backend/S3/S3Backend.cs @@ -209,7 +209,7 @@ namespace Duplicati.Library.Backend var uri = new Utility.Uri(url); uri.RequireHost(); - m_bucket = uri.Host; + m_bucket = uri.Host ?? ""; m_prefix = uri.Path; var timeout = TimeoutOptionsHelper.Parse(options); diff --git a/Duplicati/Library/Backend/SMB/SMBBackend.cs b/Duplicati/Library/Backend/SMB/SMBBackend.cs index c9bafe770..22cef79af 100644 --- a/Duplicati/Library/Backend/SMB/SMBBackend.cs +++ b/Duplicati/Library/Backend/SMB/SMBBackend.cs @@ -134,7 +134,7 @@ public class SMBBackend : IStreamingBackend, IFolderEnabledBackend var uri = new Utility.Uri(url); uri.RequireHost(); - _DnsName = uri.Host; + _DnsName = uri.Host ?? ""; var input = uri.Path.TrimEnd('/'); var slashIndex = input.IndexOf('/'); // Find first slash to separate server and share if present. @@ -165,7 +165,7 @@ public class SMBBackend : IStreamingBackend, IFolderEnabledBackend _timeouts = TimeoutOptionsHelper.Parse(options); _connectionParameters = new SMBConnectionParameters( - uri.Host, + _DnsName, transportType, slashIndex >= 0 ? input[..slashIndex] : input, slashIndex >= 0 ? input[(slashIndex + 1)..] : "", diff --git a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs index 183600a9a..51e50c37f 100644 --- a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs +++ b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs @@ -112,7 +112,7 @@ namespace Duplicati.Library.Backend m_path = "/" + m_path; } - m_server = uri.Host; + m_server = uri.Host ?? ""; if (uri.Port > 0) m_port = uri.Port; diff --git a/Duplicati/Library/Backend/SharePoint/SharePointBackend.cs b/Duplicati/Library/Backend/SharePoint/SharePointBackend.cs index ca5ce82dc..28b24ce77 100644 --- a/Duplicati/Library/Backend/SharePoint/SharePointBackend.cs +++ b/Duplicati/Library/Backend/SharePoint/SharePointBackend.cs @@ -655,6 +655,8 @@ namespace Duplicati.Library.Backend var ctx = await GetSpClientContextAsync(useNewContext, cancelToken).ConfigureAwait(false); try { + if (string.IsNullOrWhiteSpace(m_spWebUrl)) + throw new HttpRequestException(Strings.SharePoint.NoSharePointWebFoundError(m_orgUrl.ToString())); var pathLengthToWeb = new Utility.Uri(m_spWebUrl).Path.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries).Length; var folderNames = m_serverRelPath.Substring(0, m_serverRelPath.Length - 1).Split('/'); diff --git a/Duplicati/Library/Backend/TencentCOS/COSBackend.cs b/Duplicati/Library/Backend/TencentCOS/COSBackend.cs index 014323994..0c8a8f828 100644 --- a/Duplicati/Library/Backend/TencentCOS/COSBackend.cs +++ b/Duplicati/Library/Backend/TencentCOS/COSBackend.cs @@ -123,7 +123,7 @@ namespace Duplicati.Library.Backend.TencentCOS public COS(string url, Dictionary options) { - var uri = new Utility.Uri(url?.Trim()); + var uri = new Utility.Uri(url?.Trim() ?? ""); var prefix = uri.HostAndPath?.Trim()?.Trim('/')?.Trim('\\'); var auth = AuthOptionsHelper.ParseWithAlias(options, uri, COS_SECRET_ID, COS_SECRET_KEY) .RequireCredentials(); diff --git a/Duplicati/Library/Backend/WEBDAV/WEBDAV.cs b/Duplicati/Library/Backend/WEBDAV/WEBDAV.cs index 99dc8294a..1bf5e8373 100644 --- a/Duplicati/Library/Backend/WEBDAV/WEBDAV.cs +++ b/Duplicati/Library/Backend/WEBDAV/WEBDAV.cs @@ -142,7 +142,7 @@ namespace Duplicati.Library.Backend { var u = new Utility.Uri(url); u.RequireHost(); - m_dnsName = u.Host; + m_dnsName = u.Host ?? ""; var auth = AuthOptionsHelper.Parse(options, u); if (auth.HasUsername) { diff --git a/Duplicati/Library/Backend/pCloud/pCloudBackend.cs b/Duplicati/Library/Backend/pCloud/pCloudBackend.cs index 7f6893483..d06b78a62 100644 --- a/Duplicati/Library/Backend/pCloud/pCloudBackend.cs +++ b/Duplicati/Library/Backend/pCloud/pCloudBackend.cs @@ -128,22 +128,22 @@ public class pCloudBackend : IStreamingBackend { var uri = new Utility.Uri(url); uri.RequireHost(); - _DnsName = uri.Host; + _DnsName = uri.Host ?? ""; _Token = AuthIdOptionsHelper.Parse(options) .RequireCredentials(TOKEN_URL) .AuthId!; - if (!PCLOUD_SERVERS.ContainsValue(uri.Host)) + if (!PCLOUD_SERVERS.ContainsValue(_DnsName)) throw new UserInformationException(Strings.pCloudBackend.InvalidServerSpecified, "InvalidpCloudServerSpecified"); - if (string.IsNullOrWhiteSpace(uri.Host)) + if (string.IsNullOrWhiteSpace(_DnsName)) throw new UserInformationException(Strings.pCloudBackend.NoServerSpecified, "NopCloudServerSpecified"); // Ensure that the path is in the correct format, without starting or tailing slashes _Path = uri.Path.TrimStart(PATH_SEPARATORS).TrimEnd(PATH_SEPARATORS).Trim(); - _ServerUrl = uri.Host; + _ServerUrl = _DnsName; _Timeouts = TimeoutOptionsHelper.Parse(options); var httpHandler = new HttpClientHandler(); diff --git a/Duplicati/Library/Main/CLIDatabaseLocator.cs b/Duplicati/Library/Main/CLIDatabaseLocator.cs index f24736418..f9030520c 100644 --- a/Duplicati/Library/Main/CLIDatabaseLocator.cs +++ b/Duplicati/Library/Main/CLIDatabaseLocator.cs @@ -129,7 +129,7 @@ namespace Duplicati.Library.Main ?? new List(); var uri = new Library.Utility.Uri(backend); - var server = uri.Host; + var server = uri.Host ?? ""; var path = uri.Path; var type = uri.Scheme; var port = uri.Port; diff --git a/WebserverCore.Client.UsageExample/Program.cs b/WebserverCore.Client.UsageExample/Program.cs index feb749c53..dfcf64b05 100644 --- a/WebserverCore.Client.UsageExample/Program.cs +++ b/WebserverCore.Client.UsageExample/Program.cs @@ -71,7 +71,7 @@ class Program { // Issue a signin token var signinTokenResult = await client.IssueSigninTokenV1Async( - new IssueSigninTokenInputDto( Environment.GetEnvironmentVariable("TEST_PASSWORD") ) { }, CancellationToken.None); + new IssueSigninTokenInputDto(Environment.GetEnvironmentVariable("TEST_PASSWORD")!) { }, CancellationToken.None); Console.WriteLine($"✓ Signin token issued: {signinTokenResult.Token[..10]}..."); // Issue a single operation token @@ -85,7 +85,7 @@ class Program var foreverToken = await client.IssueForeverTokenV1Async(CancellationToken.None); Console.WriteLine($"✓ Forever token issued: {foreverToken.Token[..10]}..."); } - catch (Exception e) + catch (Exception) { // expected here if foreever tokens are disabled } @@ -137,7 +137,7 @@ class Program // Export as command line var cmdlineExport = await client.ExportCommandlineV1Async(backup.Backup.ID, CancellationToken.None); - Console.WriteLine($" ✓ Command line exported: {cmdlineExport.Command[..50]}..."); + Console.WriteLine($" ✓ Command line exported: {cmdlineExport!.Command![..50]}..."); // Export arguments only var argsExport = await client.ExportArgsOnlyV1Async(backup.Backup.ID, CancellationToken.None); @@ -351,12 +351,12 @@ class Program // Export backup configuration var exportToken = await client.IssueTokenV1Async("export", CancellationToken.None); - var exportedConfig = await client.ExportBackupV1Async(firstBackup.Backup.ID,true, "random", exportToken.Token, CancellationToken.None); + var exportedConfig = await client.ExportBackupV1Async(firstBackup.Backup.ID, true, "random", exportToken.Token, CancellationToken.None); Console.WriteLine($"✓ Exported backup configuration: {exportedConfig}"); // Export as command line var cmdlineExport = await client.ExportCommandlineV1Async(firstBackup.Backup.ID, CancellationToken.None); - Console.WriteLine($"✓ Exported command line: {cmdlineExport.Command.Length} characters"); + Console.WriteLine($"✓ Exported command line: {cmdlineExport!.Command![..50]}..."); // Export arguments only var argsExport = await client.ExportArgsOnlyV1Async(firstBackup.Backup.ID, CancellationToken.None); @@ -581,7 +581,7 @@ class Program BackupId = firstBackup.Backup.ID }; var filesetsResponse = await client.ListFilesetsV2Async(filesetsRequest, CancellationToken.None); - Console.WriteLine($"✓ V2 Filesets: {filesetsResponse.Data.ToArray().Length} items"); + Console.WriteLine($"✓ V2 Filesets: {filesetsResponse!.Data!.ToArray().Length} items"); // List folder content with pagination var folderRequest = new ListFolderContentRequestDto @@ -593,7 +593,7 @@ class Program Page = null }; var folderResponse = await client.ListFolderContentV2Async(folderRequest, CancellationToken.None); - Console.WriteLine($"✓ V2 Folder Content: {folderResponse.Data.ToArray().Length} items,"); + Console.WriteLine($"✓ V2 Folder Content: {folderResponse!.Data!.ToArray().Length} items,"); // Search entries var searchRequest = new SearchEntriesRequestDto @@ -606,7 +606,7 @@ class Program Page = 0 }; var searchResponse = await client.SearchEntriesV2Async(searchRequest, CancellationToken.None); - Console.WriteLine($"✓ V2 Search Results: {searchResponse.Data.ToArray().Length} items"); + Console.WriteLine($"✓ V2 Search Results: {searchResponse!.Data!.ToArray().Length} items"); // Test destination (commented out as it requires valid destination configuration) /*