From 356d0c1bfdaf530fefd145046c5f4f485ee4d670 Mon Sep 17 00:00:00 2001 From: "Marcelo C." Date: Tue, 3 Dec 2024 12:16:17 -0300 Subject: [PATCH 01/10] Implement nested folder support and improve file operations on pCloud Backend - Add support for subfolders - Implement robust file selection using fileID for Download/Delete operations instead of concatenation of path + filename - Improved validation for folder existence during backend connectivity tests supporting subfolders. --- .../pCloud/Model/pCloudFolderContent.cs | 2 +- .../Library/Backend/pCloud/pCloudBackend.cs | 185 ++++++++++++------ 2 files changed, 124 insertions(+), 63 deletions(-) diff --git a/Duplicati/Library/Backend/pCloud/Model/pCloudFolderContent.cs b/Duplicati/Library/Backend/pCloud/Model/pCloudFolderContent.cs index 24c718553..66589fc46 100644 --- a/Duplicati/Library/Backend/pCloud/Model/pCloudFolderContent.cs +++ b/Duplicati/Library/Backend/pCloud/Model/pCloudFolderContent.cs @@ -85,7 +85,7 @@ internal record pCloudFolderContent /// /// Folder ID if the item is a folder (null otherwise) /// - public long? folderid { get; init; } + public ulong folderid { get; init; } /// /// File ID if the item is a file (null otherwise) diff --git a/Duplicati/Library/Backend/pCloud/pCloudBackend.cs b/Duplicati/Library/Backend/pCloud/pCloudBackend.cs index 454c7046d..8447d8e16 100644 --- a/Duplicati/Library/Backend/pCloud/pCloudBackend.cs +++ b/Duplicati/Library/Backend/pCloud/pCloudBackend.cs @@ -92,9 +92,20 @@ public class pCloudBackend : IStreamingBackend /// requests /// private ulong? _CachedFolderID; - + + /// + /// Name of the authentication parameter/option + /// private const string AUTHENTICATION_OPTION = "authid"; + /// + /// Path separators (both Windows \ and unix /) to be used in path manipulation + /// + private static readonly char[] PATH_SEPARATORS = ['/', '\\']; + + /// + /// List of pcloud Servers and their respective hostnames + /// private static readonly Dictionary PCLOUD_SERVERS = new(StringComparer.OrdinalIgnoreCase) { { "pCloud Global", "api.pcloud.com" }, @@ -128,8 +139,9 @@ public class pCloudBackend : IStreamingBackend if (string.IsNullOrWhiteSpace(uri.Host)) throw new UserInformationException(Strings.pCloudBackend.NoServerSpecified, "NopCloudServerSpecified"); - - _Path = uri.Path; + + // Ensure that the path is in the correct format, without starting or tailing slashes + _Path = uri.Path.TrimStart(PATH_SEPARATORS).TrimEnd(PATH_SEPARATORS).Trim(); _ServerUrl = uri.Host; } @@ -155,19 +167,31 @@ public class pCloudBackend : IStreamingBackend /// The root parameter is used to list the root folder, as the pCloud API /// when using oauth tokens creates an isolated folder Applications/ApplicationName /// - /// When root = true, this is the directory it will be listed. - /// When root = false, it will list the _Path directory if it exists, which is - /// where the Duplicati backup files are stored. /// - /// True to list absolute root, false to list _Path directory + /// The folder ID to consider as root /// - private IEnumerable List(bool root) + private IEnumerable List(ulong folderId) + { + return ListWithMetadata(folderId) + .Select(item => new pCloudFileEntry + { + IsFolder = item.isfolder, + Name = item.name, + Size = item.size ?? 0, + LastAccess = DateTime.Parse(item.created), + LastModification = DateTime.Parse(item.modified) + }) + .ToList(); + } + + /// + /// Lists folders with pCloud metadata, necessary to obtain the folder IDs + /// + /// FolderId to be used as root. + /// + /// + private IEnumerable ListWithMetadata(ulong folderId) { - if (_CachedFolderID == null && string.IsNullOrWhiteSpace(_Path) == false && root == false) - CreateFolderAsync(CancellationToken.None).Await(); - - ulong folderId = root ? 0 : _CachedFolderID ?? 0; - var timeoutToken = new CancellationTokenSource(); timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS)); @@ -187,19 +211,7 @@ public class pCloudBackend : IStreamingBackend if (listFolderResponse.result != 0) throw new Exception(Strings.pCloudBackend.FailedWithUnexpectedErrorCode("list", listFolderResponse.result)); - if (listFolderResponse?.metadata?.contents == null) - return new List(); - - return listFolderResponse.metadata.contents - .Select(item => new pCloudFileEntry - { - IsFolder = item.isfolder, - Name = item.name, - Size = item.size ?? 0, - LastAccess = DateTime.Parse(item.created), - LastModification = DateTime.Parse(item.modified) - }) - .ToList(); + return listFolderResponse.metadata?.contents ?? []; } /// @@ -208,7 +220,9 @@ public class pCloudBackend : IStreamingBackend /// List of IFileEntry with directory listing result public IEnumerable List() { - return List(false); + _CachedFolderID ??= GetFolderId().Await(); + + return List(_CachedFolderID.Value); } /// @@ -236,10 +250,8 @@ public class pCloudBackend : IStreamingBackend /// Exceptions arising from either code execution public async Task PutAsync(string remotename, Stream input, CancellationToken cancellationToken) { - // If we don't have the folder ID cached, lets fetch it with the create folder api - if (_CachedFolderID == null) - await CreateFolderAsync(cancellationToken).ConfigureAwait(false); - + _CachedFolderID ??= GetFolderId().Await(); + using var timeoutToken = new CancellationTokenSource(); timeoutToken.CancelAfter(TimeSpan.FromSeconds(LONG_OPERATION_TIMEOUT_SECONDS)); using var combinedTokens = @@ -250,6 +262,7 @@ public class pCloudBackend : IStreamingBackend using var requestResources = CreateRequest($"/uploadfile?folderid={_CachedFolderID}&filename={encodedPath}&nopartial=1", HttpMethod.Post); + requestResources.RequestMessage.Content = new StreamContent(input); requestResources.RequestMessage.Content.Headers.ContentLength = input.Length; requestResources.RequestMessage.Content.Headers.ContentType = @@ -293,12 +306,9 @@ public class pCloudBackend : IStreamingBackend /// /// FileMissingException when file is not found /// Exceptions arising from either code execution - private async Task GetFileId(string filename) + private async Task GetFileLink(string filename) { - var path = string.IsNullOrWhiteSpace(_Path) ? "" : $"/{_Path}"; - var encodedPath = Uri.EscapeDataString($"{path}/{filename}"); - - using var requestResources = CreateRequest($"/getfilelink?path={encodedPath}", HttpMethod.Get); + using var requestResources = CreateRequest($"/getfilelink?fileid={GetFileId(filename).Await()}", HttpMethod.Get); using var response = await requestResources.HttpClient.SendAsync( requestResources.RequestMessage, @@ -317,7 +327,7 @@ public class pCloudBackend : IStreamingBackend if (getFileIdResponse.result != 0) throw new Exception( - Strings.pCloudBackend.FailedWithUnexpectedErrorCode("getfileid", getFileIdResponse.result)); + Strings.pCloudBackend.FailedWithUnexpectedErrorCode("getfilelink", getFileIdResponse.result)); return $"https://{getFileIdResponse.hosts[0]}{getFileIdResponse.path}"; } @@ -341,7 +351,7 @@ public class pCloudBackend : IStreamingBackend using var requestResources = CreateRequest(string.Empty, HttpMethod.Get); - requestResources.RequestMessage.RequestUri = new Uri(await GetFileId(remotename)); + requestResources.RequestMessage.RequestUri = new Uri(GetFileLink(remotename).Await()); await requestResources.HttpClient.DownloadFile(requestResources.RequestMessage, output, null, timeoutToken.Token).ConfigureAwait(false); @@ -377,15 +387,12 @@ public class pCloudBackend : IStreamingBackend /// Exceptions arising from either code execution or business logic when return code from pcloud indicates an error. public Task DeleteAsync(string remotename, CancellationToken cancellationToken) { - if (!String.IsNullOrWhiteSpace(_Path)) - remotename = $"/{_Path}/{remotename}"; - using var timeoutToken = new CancellationTokenSource(); timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS)); using var combinedTokens = CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken); - using var requestResources = CreateRequest($"/deletefile?path={remotename}", HttpMethod.Get); + using var requestResources = CreateRequest($"/deletefile?fileid={GetFileId(remotename).Await()}", HttpMethod.Get); using var response = requestResources.HttpClient.SendAsync( requestResources.RequestMessage, @@ -418,17 +425,23 @@ public class pCloudBackend : IStreamingBackend public Task GetDNSNamesAsync(CancellationToken cancellationToken) => Task.FromResult(new[] { _DnsName }); /// - /// Performs basic backend conectivity tests, in this case, checks - /// that the configured path, if configured, exists + /// Tests backend connectivity by verifying the configured path exists /// - /// CancellationToken, in this call not used. - /// + /// The cancellation token (not used) /// Thrown when configured path does not exist public Task TestAsync(CancellationToken cancellationToken) { - // List the scoped root directory and check if _Path folder exists - if (!List(true).Any(x => x.IsFolder && x.Name == _Path) && !string.IsNullOrWhiteSpace(_Path)) - throw new FolderMissingException(); + if (string.IsNullOrWhiteSpace(_Path)) + return Task.CompletedTask; + + var segments = _Path.Split(PATH_SEPARATORS, StringSplitOptions.RemoveEmptyEntries); + if (segments.Length == 0) + return Task.CompletedTask; + + _ = segments.Aggregate(0UL, (parentId, folder) => + ListWithMetadata(parentId) + .FirstOrDefault(x => x.isfolder && x.name == folder) + ?.folderid ?? throw new FolderMissingException()); return Task.CompletedTask; } @@ -440,13 +453,34 @@ public class pCloudBackend : IStreamingBackend /// /// public Task CreateFolderAsync(CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(_Path)) + _CachedFolderID = 0; + else + _CachedFolderID = _Path.IndexOfAny(PATH_SEPARATORS) == -1 + ? CreateFolder(cancellationToken, 0, _Path).Await() + : _Path.Split(PATH_SEPARATORS, StringSplitOptions.RemoveEmptyEntries) + .Aggregate(0UL, (currentId, folder) => + CreateFolder(cancellationToken, currentId, folder).Await()); + + return Task.CompletedTask; + } + + /// + /// Create remote folder in relation to the parent folder + /// + /// Cancellation Token + /// Parent Folder ID + /// Folder name + /// The folderID of the newly created folder + private Task CreateFolder(CancellationToken cancellationToken, ulong parentFolderId, string folderName) { using var timeoutToken = new CancellationTokenSource(); timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS)); using var combinedTokens = CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken); - using var requestResources = CreateRequest($"/createfolderifnotexists?folderid=0&name={_Path}", HttpMethod.Get); + using var requestResources = CreateRequest($"/createfolderifnotexists?folderid={parentFolderId}&name={folderName}", HttpMethod.Get); using var response = requestResources.HttpClient.SendAsync( requestResources.RequestMessage, @@ -459,21 +493,48 @@ public class pCloudBackend : IStreamingBackend if (pCloudErrorList.ErrorMessages.TryGetValue(createFolderResponse.result, out var message)) throw new Exception(message); - /* - * If no error code is matched, result was == 0 so it successfully created the folder - * We must cache the folderID, as that will be used to upload files, reason being that - * for oauth clients, pCloud creates a scoped folder Applications/ApplicationName - * and the folders created with /createfolderifnotexists api are created under that folder. - * - * However, when uploading files, the folderid is required (and surprisingly if not provided - * it will create the file outside the scoped folder, at the root of the drive) - */ - if (createFolderResponse.result == 0) _CachedFolderID = createFolderResponse.metadata?.folderid; - - return Task.CompletedTask; + + return createFolderResponse is { result: 0, metadata.folderid: var id } + ? Task.FromResult(id) + : throw new Exception(Strings.pCloudBackend.FailedWithUnexpectedErrorCode("createfolder", createFolderResponse.result)); } + /// + /// Returns the fileID by listing the folder and searching for the filename & metadata. + /// + /// For operations such as delete/getfilelink using the fileID is more reliable than using direct path/filename concatenation. + /// + /// The filename + /// + /// + private Task GetFileId(string name) + { + _CachedFolderID ??= GetFolderId().Await(); + + return Task.FromResult(ListWithMetadata((ulong)_CachedFolderID) + .FirstOrDefault(x => !x.isfolder && x.name == name) + ?.fileid ?? throw new FileNotFoundException(name)); + } + + /// + /// Returns the folder ID for the configured path, regardless of the depth + /// + /// + private Task GetFolderId() + { + if (string.IsNullOrWhiteSpace(_Path)) + return Task.FromResult(0UL); + var segments = _Path.Split(PATH_SEPARATORS, StringSplitOptions.RemoveEmptyEntries); + if (segments.Length == 0) + return Task.FromResult(0UL); + + return Task.FromResult(segments.Aggregate(0UL, (parentId, folder) => + ListWithMetadata(parentId) + .FirstOrDefault(x => x.isfolder && x.name == folder) + ?.folderid ?? throw new FolderMissingException())); + } + /// /// Wrapper for the tupple of HttpClient and HttpRequestMessage used in web requests. /// From ce73e0f1f60e5bbe4e09853ea3d1b783c8412bec Mon Sep 17 00:00:00 2001 From: "Marcelo C." Date: Wed, 4 Dec 2024 09:29:14 -0300 Subject: [PATCH 02/10] Implementation of fixes suggested on code review Post changes tests: - Folder creation and discovery checked. - Backup and restore operations checked. - Tests with backend tests performed. --- .../Library/Backend/pCloud/pCloudBackend.cs | 192 ++++++++++-------- 1 file changed, 103 insertions(+), 89 deletions(-) diff --git a/Duplicati/Library/Backend/pCloud/pCloudBackend.cs b/Duplicati/Library/Backend/pCloud/pCloudBackend.cs index 8447d8e16..8c6c2261a 100644 --- a/Duplicati/Library/Backend/pCloud/pCloudBackend.cs +++ b/Duplicati/Library/Backend/pCloud/pCloudBackend.cs @@ -92,7 +92,7 @@ public class pCloudBackend : IStreamingBackend /// requests /// private ulong? _CachedFolderID; - + /// /// Name of the authentication parameter/option /// @@ -139,7 +139,7 @@ public class pCloudBackend : IStreamingBackend if (string.IsNullOrWhiteSpace(uri.Host)) throw new UserInformationException(Strings.pCloudBackend.NoServerSpecified, "NopCloudServerSpecified"); - + // Ensure that the path is in the correct format, without starting or tailing slashes _Path = uri.Path.TrimStart(PATH_SEPARATORS).TrimEnd(PATH_SEPARATORS).Trim(); _ServerUrl = uri.Host; @@ -166,43 +166,47 @@ public class pCloudBackend : IStreamingBackend /// Implementation of interface method for listing remote folder contents. /// The root parameter is used to list the root folder, as the pCloud API /// when using oauth tokens creates an isolated folder Applications/ApplicationName - /// + /// /// /// The folder ID to consider as root + /// Cancellation Token /// - private IEnumerable List(ulong folderId) + private async Task> List(ulong folderId, CancellationToken cancellationToken) { - return ListWithMetadata(folderId) - .Select(item => new pCloudFileEntry - { - IsFolder = item.isfolder, - Name = item.name, - Size = item.size ?? 0, - LastAccess = DateTime.Parse(item.created), - LastModification = DateTime.Parse(item.modified) - }) + var result = await ListWithMetadata(folderId, cancellationToken).ConfigureAwait(false); + return result.Select(item => new pCloudFileEntry + { + IsFolder = item.isfolder, + Name = item.name, + Size = item.size ?? 0, + LastAccess = DateTime.Parse(item.created), + LastModification = DateTime.Parse(item.modified) + }) .ToList(); } - + /// /// Lists folders with pCloud metadata, necessary to obtain the folder IDs /// /// FolderId to be used as root. + /// Cancellation Token /// /// - private IEnumerable ListWithMetadata(ulong folderId) + private async Task> ListWithMetadata(ulong folderId, CancellationToken cancellationToken) { - var timeoutToken = new CancellationTokenSource(); + using var timeoutToken = new CancellationTokenSource(); timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS)); + using var combinedTokens = + CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken); using var requestResources = CreateRequest($"/listfolder?folderid={folderId}", HttpMethod.Get); - using var response = requestResources.HttpClient + using var response = await requestResources.HttpClient .SendAsync(requestResources.RequestMessage, HttpCompletionOption.ResponseContentRead, - timeoutToken.Token).Await(); + timeoutToken.Token).ConfigureAwait(false); response.EnsureSuccessStatusCode(); - var content = response.Content.ReadAsStringAsync(timeoutToken.Token).Result; + var content = await response.Content.ReadAsStringAsync(timeoutToken.Token).ConfigureAwait(false); var listFolderResponse = JsonSerializer.Deserialize(content); if (pCloudErrorList.ErrorMessages.TryGetValue(listFolderResponse.result, out var message)) @@ -218,13 +222,19 @@ public class pCloudBackend : IStreamingBackend /// Implementation of interface method for listing remote folder contents /// /// List of IFileEntry with directory listing result - public IEnumerable List() + public async Task> ListAsync(CancellationToken cancellationToken) { - _CachedFolderID ??= GetFolderId().Await(); + _CachedFolderID ??= await GetFolderId(cancellationToken).ConfigureAwait(false); - return List(_CachedFolderID.Value); + return await List(_CachedFolderID.Value, cancellationToken).ConfigureAwait(false); } + /// + /// Wrapper method of legacy non async call to list files in the remote folder + /// + /// + public IEnumerable List() => ListAsync(CancellationToken.None).Await(); + /// /// Upload files to remote location /// @@ -250,8 +260,8 @@ public class pCloudBackend : IStreamingBackend /// Exceptions arising from either code execution public async Task PutAsync(string remotename, Stream input, CancellationToken cancellationToken) { - _CachedFolderID ??= GetFolderId().Await(); - + _CachedFolderID ??= await GetFolderId(cancellationToken).ConfigureAwait(false); + using var timeoutToken = new CancellationTokenSource(); timeoutToken.CancelAfter(TimeSpan.FromSeconds(LONG_OPERATION_TIMEOUT_SECONDS)); using var combinedTokens = @@ -262,7 +272,7 @@ public class pCloudBackend : IStreamingBackend using var requestResources = CreateRequest($"/uploadfile?folderid={_CachedFolderID}&filename={encodedPath}&nopartial=1", HttpMethod.Post); - + requestResources.RequestMessage.Content = new StreamContent(input); requestResources.RequestMessage.Content.Headers.ContentLength = input.Length; requestResources.RequestMessage.Content.Headers.ContentType = @@ -289,7 +299,7 @@ public class pCloudBackend : IStreamingBackend /// Destination stream to write to /// CancellationToken that is combined with internal timeout token /// FileMissingException when file is not found - /// Exceptions arising from either code execution + /// Exceptions arising from either code execution or FileMissingException public async Task GetAsync(string remotename, string localname, CancellationToken cancellationToken) { await using var fs = File.Open(localname, @@ -303,23 +313,27 @@ public class pCloudBackend : IStreamingBackend /// /// /// Filename at remote, path is automatically concatenated if needed + /// Cancellation Token /// /// FileMissingException when file is not found /// Exceptions arising from either code execution - private async Task GetFileLink(string filename) + private async Task GetFileLink(string filename, CancellationToken cancellationToken) { - using var requestResources = CreateRequest($"/getfilelink?fileid={GetFileId(filename).Await()}", HttpMethod.Get); + using var timeoutToken = new CancellationTokenSource(); + timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS)); + using var combinedTokens = + CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken); + + using var requestResources = CreateRequest($"/getfilelink?fileid={await GetFileId(filename, cancellationToken).ConfigureAwait(false)}", HttpMethod.Get); using var response = await requestResources.HttpClient.SendAsync( requestResources.RequestMessage, - HttpCompletionOption.ResponseContentRead).ConfigureAwait(false); + HttpCompletionOption.ResponseContentRead, combinedTokens.Token).ConfigureAwait(false); if (!response.IsSuccessStatusCode) - { throw new Exception($"Failed to get download link. Status: {response.StatusCode}"); - } - var content = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + var content = await response.Content.ReadAsStringAsync(combinedTokens.Token).ConfigureAwait(false); var getFileIdResponse = JsonSerializer.Deserialize(content); if (pCloudErrorList.ErrorMessages.TryGetValue(getFileIdResponse.result, out var message)) @@ -339,7 +353,7 @@ public class pCloudBackend : IStreamingBackend /// Destination stream to write to /// CancellationToken that is combined with internal timeout token /// FileMissingException when file is not found - /// Exceptions arising from either code execution + /// Exceptions arising from either code execution or FileMissingException public async Task GetAsync(string remotename, Stream output, CancellationToken cancellationToken) { try @@ -351,10 +365,10 @@ public class pCloudBackend : IStreamingBackend using var requestResources = CreateRequest(string.Empty, HttpMethod.Get); - requestResources.RequestMessage.RequestUri = new Uri(GetFileLink(remotename).Await()); + requestResources.RequestMessage.RequestUri = new Uri(await GetFileLink(remotename, cancellationToken).ConfigureAwait(false)); await requestResources.HttpClient.DownloadFile(requestResources.RequestMessage, output, null, - timeoutToken.Token).ConfigureAwait(false); + combinedTokens.Token).ConfigureAwait(false); } catch (HttpRequestException wex) { @@ -385,22 +399,22 @@ public class pCloudBackend : IStreamingBackend /// /// FileMissingException when file is not found /// Exceptions arising from either code execution or business logic when return code from pcloud indicates an error. - public Task DeleteAsync(string remotename, CancellationToken cancellationToken) + public async Task DeleteAsync(string remotename, CancellationToken cancellationToken) { using var timeoutToken = new CancellationTokenSource(); timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS)); using var combinedTokens = CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken); - using var requestResources = CreateRequest($"/deletefile?fileid={GetFileId(remotename).Await()}", HttpMethod.Get); + using var requestResources = CreateRequest($"/deletefile?fileid={await GetFileId(remotename, cancellationToken).ConfigureAwait(false)}", HttpMethod.Get); - using var response = requestResources.HttpClient.SendAsync( + using var response = await requestResources.HttpClient.SendAsync( requestResources.RequestMessage, - HttpCompletionOption.ResponseContentRead, combinedTokens.Token).Await(); + HttpCompletionOption.ResponseContentRead, combinedTokens.Token).ConfigureAwait(false); response.EnsureSuccessStatusCode(); - var content = response.Content.ReadAsStringAsync(combinedTokens.Token).Await(); + var content = await response.Content.ReadAsStringAsync(combinedTokens.Token).ConfigureAwait(false); var deleteFileResponse = JsonSerializer.Deserialize(content); // If no error code is matched, result was == 0 so it successfully created the folder @@ -414,7 +428,6 @@ public class pCloudBackend : IStreamingBackend throw new Exception( Strings.pCloudBackend.FailedWithUnexpectedErrorCode("delete", deleteFileResponse.result)); - return Task.CompletedTask; } /// @@ -429,21 +442,14 @@ public class pCloudBackend : IStreamingBackend /// /// The cancellation token (not used) /// Thrown when configured path does not exist - public Task TestAsync(CancellationToken cancellationToken) + public async Task TestAsync(CancellationToken cancellationToken) { - if (string.IsNullOrWhiteSpace(_Path)) - return Task.CompletedTask; + if (string.IsNullOrWhiteSpace(_Path) || _Path.Split(PATH_SEPARATORS, StringSplitOptions.RemoveEmptyEntries).Length == 0) + return; - var segments = _Path.Split(PATH_SEPARATORS, StringSplitOptions.RemoveEmptyEntries); - if (segments.Length == 0) - return Task.CompletedTask; + // This method will search for the folderId recursively and throw an exception if the folder is not found + await GetFolderId(cancellationToken).ConfigureAwait(false); - _ = segments.Aggregate(0UL, (parentId, folder) => - ListWithMetadata(parentId) - .FirstOrDefault(x => x.isfolder && x.name == folder) - ?.folderid ?? throw new FolderMissingException()); - - return Task.CompletedTask; } /// @@ -452,20 +458,21 @@ public class pCloudBackend : IStreamingBackend /// CancellationToken that will be combined with internal timeout token /// /// - public Task CreateFolderAsync(CancellationToken cancellationToken) + public async Task CreateFolderAsync(CancellationToken cancellationToken) { - if (string.IsNullOrWhiteSpace(_Path)) + if (string.IsNullOrWhiteSpace(_Path) || _Path.Split(PATH_SEPARATORS, StringSplitOptions.RemoveEmptyEntries).Length == 0) + { _CachedFolderID = 0; - else - _CachedFolderID = _Path.IndexOfAny(PATH_SEPARATORS) == -1 - ? CreateFolder(cancellationToken, 0, _Path).Await() - : _Path.Split(PATH_SEPARATORS, StringSplitOptions.RemoveEmptyEntries) - .Aggregate(0UL, (currentId, folder) => - CreateFolder(cancellationToken, currentId, folder).Await()); - - return Task.CompletedTask; + return; + } + + var currentId = 0UL; + foreach (var folder in _Path.Split(PATH_SEPARATORS, StringSplitOptions.RemoveEmptyEntries)) + currentId = await CreateFolder(cancellationToken, currentId, folder).ConfigureAwait(false); + + _CachedFolderID = currentId; } - + /// /// Create remote folder in relation to the parent folder /// @@ -473,7 +480,7 @@ public class pCloudBackend : IStreamingBackend /// Parent Folder ID /// Folder name /// The folderID of the newly created folder - private Task CreateFolder(CancellationToken cancellationToken, ulong parentFolderId, string folderName) + private async Task CreateFolder(CancellationToken cancellationToken, ulong parentFolderId, string folderName) { using var timeoutToken = new CancellationTokenSource(); timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS)); @@ -482,59 +489,66 @@ public class pCloudBackend : IStreamingBackend using var requestResources = CreateRequest($"/createfolderifnotexists?folderid={parentFolderId}&name={folderName}", HttpMethod.Get); - using var response = requestResources.HttpClient.SendAsync( + using var response = await requestResources.HttpClient.SendAsync( requestResources.RequestMessage, - HttpCompletionOption.ResponseContentRead, combinedTokens.Token).Await(); + HttpCompletionOption.ResponseContentRead, combinedTokens.Token).ConfigureAwait(false); response.EnsureSuccessStatusCode(); - var content = response.Content.ReadAsStringAsync(combinedTokens.Token).Await(); + var content = await response.Content.ReadAsStringAsync(combinedTokens.Token).ConfigureAwait(false); var createFolderResponse = JsonSerializer.Deserialize(content); if (pCloudErrorList.ErrorMessages.TryGetValue(createFolderResponse.result, out var message)) throw new Exception(message); - + return createFolderResponse is { result: 0, metadata.folderid: var id } - ? Task.FromResult(id) + ? id : throw new Exception(Strings.pCloudBackend.FailedWithUnexpectedErrorCode("createfolder", createFolderResponse.result)); } /// /// Returns the fileID by listing the folder and searching for the filename & metadata. - /// + /// /// For operations such as delete/getfilelink using the fileID is more reliable than using direct path/filename concatenation. /// /// The filename + /// Cancellation Token /// /// - private Task GetFileId(string name) + private async Task GetFileId(string name, CancellationToken cancellationToken) { - _CachedFolderID ??= GetFolderId().Await(); - - return Task.FromResult(ListWithMetadata((ulong)_CachedFolderID) - .FirstOrDefault(x => !x.isfolder && x.name == name) - ?.fileid ?? throw new FileNotFoundException(name)); + _CachedFolderID ??= await GetFolderId(cancellationToken).ConfigureAwait(false); + + var result = await ListWithMetadata((ulong)_CachedFolderID, cancellationToken).ConfigureAwait(false); + + return result.FirstOrDefault(x => !x.isfolder && x.name == name)?.fileid ?? throw new FileNotFoundException(name); + } - + /// /// Returns the folder ID for the configured path, regardless of the depth /// /// - private Task GetFolderId() + private async Task GetFolderId(CancellationToken cancellationToken) { - if (string.IsNullOrWhiteSpace(_Path)) - return Task.FromResult(0UL); + if (string.IsNullOrWhiteSpace(_Path) || _Path.Split(PATH_SEPARATORS, StringSplitOptions.RemoveEmptyEntries).Length == 0) + return 0UL; - var segments = _Path.Split(PATH_SEPARATORS, StringSplitOptions.RemoveEmptyEntries); - if (segments.Length == 0) - return Task.FromResult(0UL); + var currentFolderId = 0UL; + foreach (var folder in _Path.Split(PATH_SEPARATORS, StringSplitOptions.RemoveEmptyEntries)) + { + var folderContent = await ListWithMetadata(currentFolderId, cancellationToken).ConfigureAwait(false); + var matchingFolder = folderContent.FirstOrDefault(x => x.isfolder && x.name == folder); - return Task.FromResult(segments.Aggregate(0UL, (parentId, folder) => - ListWithMetadata(parentId) - .FirstOrDefault(x => x.isfolder && x.name == folder) - ?.folderid ?? throw new FolderMissingException())); + if (matchingFolder?.folderid == null) + throw new FolderMissingException(); + + currentFolderId = matchingFolder.folderid; + } + + return currentFolderId; } - + /// /// Wrapper for the tupple of HttpClient and HttpRequestMessage used in web requests. /// From aa22766939d71bfe2589ea4e3b264eb12c46f235 Mon Sep 17 00:00:00 2001 From: "Marcelo C." Date: Wed, 4 Dec 2024 13:25:55 -0300 Subject: [PATCH 03/10] Adding support for self-signed or specific by hash certificates This will add support for the parameters accept-any-ssl-certificate and accept-specified-ssl-hash to the Http sender used in reporting. --- .../Modules/Builtin/SendHttpMessage.cs | 34 +++++++++++++++++-- Duplicati/Library/Modules/Builtin/Strings.cs | 4 +++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Modules/Builtin/SendHttpMessage.cs b/Duplicati/Library/Modules/Builtin/SendHttpMessage.cs index 13b92a1dc..138279207 100644 --- a/Duplicati/Library/Modules/Builtin/SendHttpMessage.cs +++ b/Duplicati/Library/Modules/Builtin/SendHttpMessage.cs @@ -26,6 +26,8 @@ using System.Net.Http; using System.Net.Http.Headers; using System.Text; using System.Threading.Tasks; +using Duplicati.Library.Utility; +using Uri = System.Uri; namespace Duplicati.Library.Modules.Builtin { @@ -101,7 +103,16 @@ namespace Duplicati.Library.Modules.Builtin /// Option used to set the maximum number of log lines /// private const string OPTION_MAX_LOG_LINES = "send-http-max-log-lines"; - + + /// + /// The option used to accept a specific SSL certificate hash + /// + private const string OPTION_ACCEPT_SPECIFIED_CERTIFICATE = "accept-specified-ssl-hash"; + /// + /// The option used to accept any SSL certificate + /// + private const string OPTION_ACCEPT_ANY_CERTIFICATE = "accept-any-ssl-certificate"; + #endregion #region Option defaults @@ -132,6 +143,16 @@ namespace Duplicati.Library.Modules.Builtin /// The message parameter name /// private string m_extraParameters; + + /// + /// Option to accept any SSL certificate + /// + private bool m_acceptAnyCertificate; + + /// + /// Specific hashes to be accepted by the certificate validator + /// + private string[] m_acceptSpecificCertificates; #endregion @@ -184,6 +205,9 @@ namespace Duplicati.Library.Modules.Builtin new CommandLineArgument(OPTION_URL_FORM, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.SendhttpurlsformShort, Strings.SendHttpMessage.SendhttpurlsformLong), new CommandLineArgument(OPTION_URL_JSON, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.SendhttpurlsjsonShort, Strings.SendHttpMessage.SendhttpurlsjsonLong), + + new CommandLineArgument(OPTION_ACCEPT_ANY_CERTIFICATE, CommandLineArgument.ArgumentType.Boolean, Strings.SendHttpMessage.AcceptAnyCertificateShort, Strings.SendHttpMessage.AcceptAnyCertificateLong), + new CommandLineArgument(OPTION_ACCEPT_SPECIFIED_CERTIFICATE, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.AcceptSpecifiedCertificateShort, Strings.SendHttpMessage.AcceptSpecifiedCertificateLong), }); } } @@ -242,6 +266,9 @@ namespace Duplicati.Library.Modules.Builtin m_messageParameterName = DEFAULT_MESSAGE_PARAMETER_NAME; commandlineOptions.TryGetValue(OPTION_EXTRA_PARAMETERS, out m_extraParameters); + m_acceptAnyCertificate = commandlineOptions.ContainsKey(OPTION_ACCEPT_ANY_CERTIFICATE) && Utility.Utility.ParseBoolOption(commandlineOptions.AsReadOnly(), OPTION_ACCEPT_ANY_CERTIFICATE); + m_acceptSpecificCertificates = commandlineOptions.ContainsKey(OPTION_ACCEPT_SPECIFIED_CERTIFICATE) ? commandlineOptions[OPTION_ACCEPT_SPECIFIED_CERTIFICATE].Split([",", ";"], StringSplitOptions.RemoveEmptyEntries): null; + return true; } @@ -327,7 +354,10 @@ namespace Duplicati.Library.Modules.Builtin if (m_report_targets == null || m_cachedBodyResults == null) return; - using var client = new HttpClient(); + using HttpClientHandler httpHandler = new HttpClientHandler(); + HttpClientHelper.ConfigureHandlerCertificateValidator(httpHandler, m_acceptAnyCertificate, m_acceptSpecificCertificates); + + using var client = new HttpClient(httpHandler); Exception ex = null; diff --git a/Duplicati/Library/Modules/Builtin/Strings.cs b/Duplicati/Library/Modules/Builtin/Strings.cs index 538c02c0c..4ec00d57e 100644 --- a/Duplicati/Library/Modules/Builtin/Strings.cs +++ b/Duplicati/Library/Modules/Builtin/Strings.cs @@ -225,6 +225,10 @@ You can supply multiple options with a comma separator, e.g. ""{0},{1}"". The sp public static string SendhttpurlsformShort { get { return LC.L(@"HTTP report URLs for sending form data"); } } public static string SendhttpurlsjsonLong { get { return LC.L(@"Use this option to set HTTP report URLs for sending JSON data. This option accepts multiple URLs, seperated by a semi-colon. All URLs will receive the same data. Note that this option ignores the format and verb settings."); } } public static string SendhttpurlsjsonShort { get { return LC.L(@"HTTP report URLs for sending JSON data"); } } + public static string AcceptAnyCertificateLong { get { return LC.L(@"Use this option to accept any server certificate, regardless of what errors it may have. Please use --{0} instead, whenever possible.", "accept-specified-ssl-hash"); } } + public static string AcceptAnyCertificateShort { get { return LC.L(@"Accept any server certificate"); } } + public static string AcceptSpecifiedCertificateLong { get { return LC.L(@"If your server certificate is reported as invalid (e.g. with self-signed certificates), you can supply the certificate hash (SHA1) to approve it anyway. The hash value must be entered in hex format without spaces or colons. You can enter multiple hashes separated by commas."); } } + public static string AcceptSpecifiedCertificateShort { get { return LC.L(@"Optionally accept a known SSL certificate"); } } } internal static class ReportHelper From f8043332eb3b9311d8786b01fc9766a6ae575fe4 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 5 Dec 2024 12:20:11 +0100 Subject: [PATCH 04/10] Allow agent keeping password --- Duplicati/Agent/Program.cs | 50 +++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/Duplicati/Agent/Program.cs b/Duplicati/Agent/Program.cs index 041ec13c5..90910fbac 100644 --- a/Duplicati/Agent/Program.cs +++ b/Duplicati/Agent/Program.cs @@ -69,6 +69,7 @@ public static class Program /// Disable signin tokens for the webserver /// Enable ping-pong keepalive /// Disable the pre-shared key + /// Keep the webserver password /// The secret provider to use /// The secret provider cache level /// The secret provider pattern @@ -89,6 +90,7 @@ public static class Program bool WebserviceDisableSigninTokens, bool PingPongKeepalive, bool DisablePreSharedKey, + bool KeepWebservicePassword, string? SecretProvider, SecretProviderHelper.CachingLevel SecretProviderCache, string SecretProviderPattern @@ -107,7 +109,7 @@ public static class Program new Option("--agent-register-only", description: "Only register the agent, then exit", getDefaultValue: () => false), new Option("--webservice-listen-interface", description: "The interface to listen on for the webserver", getDefaultValue: () => "loopback"), new Option("--webservice-port", description: "The port to listen on for the webserver", getDefaultValue: () => "8210"), - new Option("--webservice-password", description: "The password for the webserver, if set to \"random\" a random passphrase is used", getDefaultValue: () => "random"), + new Option("--webservice-password", description: "The password for the webserver, not set, or set to \"random\", a random value is used.", getDefaultValue: () => null), new Option("--settings-encryption-key", description: "The encryption key for the database settings", getDefaultValue: () => null), new Option("--windows-eventlog", description: "The Windows event log to write to", getDefaultValue: () => "Duplicati"), new Option("--disable-db-encryption", description: "Disable database encryption", getDefaultValue: () => false), @@ -117,6 +119,7 @@ public static class Program new Option("--webservice-disable-signin-tokens", description: "Disable signin tokens for the webserver", getDefaultValue: () => true), new Option("--ping-pong-keepalive", description: "Enable ping-pong keepalive", getDefaultValue: () => false), new Option("--disable-pre-shared-key", description: "Disable the pre-shared key that prevents outside access to the webserver", getDefaultValue: () => false), + new Option("--keep-webservice-password", description: "Disables the random password assigned to the webserver on startup if no password is provided", getDefaultValue: () => false), new Option("--secret-provider", description: "The secret provider to use", getDefaultValue: () => null), new Option("--secret-provider-cache", description: "The secret provider cache level", getDefaultValue: () => SecretProviderHelper.CachingLevel.None), new Option("--secret-provider-pattern", description: "The secret provider pattern", getDefaultValue: () => SecretProviderHelper.DEFAULT_PATTERN), @@ -208,6 +211,7 @@ public static class Program WebserviceDisableSigninTokens: true, PingPongKeepalive: false, DisablePreSharedKey: false, + KeepWebservicePassword: false, SecretProvider: secretProvider, SecretProviderCache: SecretProviderHelper.CachingLevel.None, SecretProviderPattern: secretProviderPattern @@ -252,9 +256,18 @@ public static class Program prop.Value.SetValue(agentConfig, value); } - // Prevent access to the webserver interface from anything but the agent - if (string.Equals("random", agentConfig.WebservicePassword, StringComparison.OrdinalIgnoreCase)) - agentConfig = agentConfig with { WebservicePassword = System.Security.Cryptography.RandomNumberGenerator.GetHexString(128) }; + if (agentConfig.KeepWebservicePassword) + { + // Check for conflicting options, require no password + if (!string.IsNullOrWhiteSpace(agentConfig.WebservicePassword)) + throw new UserInformationException("Cannot use --keep-webservice-password with a provided password", "KeepWebservicePasswordWithPassword"); + } + else + { + // Prevent access to the webserver interface from anything but the agent + if (string.Equals("random", agentConfig.WebservicePassword, StringComparison.OrdinalIgnoreCase) || string.IsNullOrWhiteSpace(agentConfig.WebservicePassword)) + agentConfig = agentConfig with { WebservicePassword = System.Security.Cryptography.RandomNumberGenerator.GetHexString(128) }; + } // Set the pre-shared key for the agent if (!agentConfig.DisablePreSharedKey) @@ -454,19 +467,28 @@ public static class Program else if (!string.IsNullOrWhiteSpace(settings.SettingsEncryptionKey)) settingsEncryptionKey = settings.SettingsEncryptionKey; + // Helper method to support empty arguments + static string EncodeOption(string option, string? value) + { + if (string.IsNullOrWhiteSpace(value)) + return ""; + + return $"{option}={value}"; + } + // Lock down the instance, reset tokens and password, // and forward relevant settings from agent to the webserver var args = new[] { - $"--webservice-listen-interface={agentConfig.WebserviceListenInterface}", - $"--webservice-password={agentConfig.WebservicePassword}", - OperatingSystem.IsWindows() ? $"--windows-eventlog={agentConfig.WindowsEventLog}" : "", - $"--webservice-port={agentConfig.WebservicePort}", - $"--webservice-reset-jwt-config={agentConfig.WebserviceResetJwtConfig}", - $"--webservice-allowed-hostnames={agentConfig.WebserviceAllowedHostnames}", - $"--webservice-api-only={agentConfig.WebserviceApiOnly}", - $"--webservice-disable-signin-tokens={agentConfig.WebserviceDisableSigninTokens}", - $"--disable-db-encryption={agentConfig.DisableDbEncryption}", - string.IsNullOrWhiteSpace(settingsEncryptionKey) ? "" : $"--settings-encryption-key={settingsEncryptionKey}", + EncodeOption("--webservice-listen-interface", agentConfig.WebserviceListenInterface), + EncodeOption("--webservice-password", agentConfig.WebservicePassword), + EncodeOption("--windows-eventlog", OperatingSystem.IsWindows() ? agentConfig.WindowsEventLog : ""), + EncodeOption("--webservice-port", agentConfig.WebservicePort), + EncodeOption("--webservice-reset-jwt-config", agentConfig.WebserviceResetJwtConfig.ToString()), + EncodeOption("--webservice-allowed-hostnames", agentConfig.WebserviceAllowedHostnames), + EncodeOption("--webservice-api-only", agentConfig.WebserviceApiOnly.ToString()), + EncodeOption("--webservice-disable-signin-tokens", agentConfig.WebserviceDisableSigninTokens.ToString()), + EncodeOption("--disable-db-encryption", agentConfig.DisableDbEncryption.ToString()), + EncodeOption("--settings-encryption-key", settingsEncryptionKey), } .Where(x => !string.IsNullOrWhiteSpace(x)) .ToArray(); From a3967527e88e7e53f840f446722627497bad4341 Mon Sep 17 00:00:00 2001 From: "Marcelo C." Date: Thu, 5 Dec 2024 10:26:37 -0300 Subject: [PATCH 05/10] Renaming the options to scope them down to the send-http module --- Duplicati/Library/Modules/Builtin/SendHttpMessage.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Modules/Builtin/SendHttpMessage.cs b/Duplicati/Library/Modules/Builtin/SendHttpMessage.cs index 138279207..169174846 100644 --- a/Duplicati/Library/Modules/Builtin/SendHttpMessage.cs +++ b/Duplicati/Library/Modules/Builtin/SendHttpMessage.cs @@ -107,11 +107,11 @@ namespace Duplicati.Library.Modules.Builtin /// /// The option used to accept a specific SSL certificate hash /// - private const string OPTION_ACCEPT_SPECIFIED_CERTIFICATE = "accept-specified-ssl-hash"; + private const string OPTION_ACCEPT_SPECIFIED_CERTIFICATE = "send-http-accept-specified-ssl-hash"; /// /// The option used to accept any SSL certificate /// - private const string OPTION_ACCEPT_ANY_CERTIFICATE = "accept-any-ssl-certificate"; + private const string OPTION_ACCEPT_ANY_CERTIFICATE = "send-http-accept-any-ssl-certificate"; #endregion From 6156024e8f864e486daa4777ee312bd4113cf0b5 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 5 Dec 2024 14:33:44 +0100 Subject: [PATCH 06/10] Bumped SSH.NET to latest This fixes #4674 --- .../Backend/SSHv2/Duplicati.Library.Backend.SSHv2.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Backend/SSHv2/Duplicati.Library.Backend.SSHv2.csproj b/Duplicati/Library/Backend/SSHv2/Duplicati.Library.Backend.SSHv2.csproj index 5c7100b7a..f5f32f772 100644 --- a/Duplicati/Library/Backend/SSHv2/Duplicati.Library.Backend.SSHv2.csproj +++ b/Duplicati/Library/Backend/SSHv2/Duplicati.Library.Backend.SSHv2.csproj @@ -8,7 +8,7 @@ - + From e738221bb20df2fac2b2cc55fecf285cdb9d8a91 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 5 Dec 2024 14:42:53 +0100 Subject: [PATCH 07/10] Removed unused dependency --- .../Library/Backend/SSHv2/Duplicati.Library.Backend.SSHv2.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/Duplicati/Library/Backend/SSHv2/Duplicati.Library.Backend.SSHv2.csproj b/Duplicati/Library/Backend/SSHv2/Duplicati.Library.Backend.SSHv2.csproj index f5f32f772..ac3182bf0 100644 --- a/Duplicati/Library/Backend/SSHv2/Duplicati.Library.Backend.SSHv2.csproj +++ b/Duplicati/Library/Backend/SSHv2/Duplicati.Library.Backend.SSHv2.csproj @@ -9,7 +9,6 @@ - From 4a523efd24e3be6061a3b3e4ee2a89a698a3f8f1 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 5 Dec 2024 16:41:28 +0100 Subject: [PATCH 08/10] Sometimes, the temporary empty volumes aren't removed from the database --- .../Main/Operation/Backup/DataBlockProcessor.cs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs b/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs index 2ec79b592..49545a46f 100644 --- a/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs +++ b/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs @@ -22,7 +22,6 @@ using CoCoL; using Duplicati.Library.Main.Operation.Common; using Duplicati.Library.Main.Volumes; -using Duplicati.Library.Utility; using System; using System.Threading.Tasks; using static Duplicati.Library.Main.Operation.Common.BackendHandler; @@ -106,8 +105,8 @@ namespace Duplicati.Library.Main.Operation.Backup // even if they are already added as non-blocklist blocks, but filter out duplicates if (indexvolume != null && isMandatoryBlocklistHash) { - // This can cause a race between workers, - // but the side-effect is that the index files are slightly larger + // This can cause a race between workers, + // but the side-effect is that the index files are slightly larger if (newBlock || !await database.IsBlocklistHashKnownAsync(b.HashKey)) { blocklistHashesAdded++; @@ -163,9 +162,16 @@ namespace Duplicati.Library.Main.Operation.Backup if (ex.IsRetiredException()) { // If we have collected data, merge all pending volumes into a single volume - if (blockvolume != null && (blockvolume.SourceSize > 0 || blocklistHashesAdded > 0)) + if (blockvolume != null) { - await self.SpillPickup.WriteAsync(new SpillVolumeRequest(blockvolume, indexvolume)); + if (blockvolume.SourceSize > 0 || blocklistHashesAdded > 0) + { + await self.SpillPickup.WriteAsync(new SpillVolumeRequest(blockvolume, indexvolume)); + } + else + { + await database.RemoveRemoteVolumeAsync(blockvolume.RemoteFilename); + } } } From 0c58de82b58112dceed03a31b8b6d6ea16fb4074 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Thu, 5 Dec 2024 16:42:09 +0100 Subject: [PATCH 09/10] Added a check for whether the file exists, rather than throwing exceptions. --- .../Operation/Backup/CountFilesHandler.cs | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/Duplicati/Library/Main/Operation/Backup/CountFilesHandler.cs b/Duplicati/Library/Main/Operation/Backup/CountFilesHandler.cs index 490f49a07..479bc0674 100644 --- a/Duplicati/Library/Main/Operation/Backup/CountFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/Backup/CountFilesHandler.cs @@ -1,22 +1,22 @@ // Copyright (C) 2024, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System.Collections.Generic; @@ -24,6 +24,7 @@ using System.Threading.Tasks; using Duplicati.Library.Utility; using Duplicati.Library.Snapshots; using CoCoL; +using Duplicati.Library.Common.IO; namespace Duplicati.Library.Main.Operation.Backup { @@ -57,7 +58,8 @@ namespace Duplicati.Library.Main.Operation.Backup try { - size += snapshot.GetFileSize(path); + if (SystemIO.IO_OS.FileExists(path)) + size += snapshot.GetFileSize(path); } catch { From 414776586692ad94c4eaf1f00baa45aa0603d422 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Fri, 6 Dec 2024 07:50:17 +0100 Subject: [PATCH 10/10] The extra repo for FreeBSD libs during restore is not responding. Removing the entry. --- nuget.config | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 nuget.config diff --git a/nuget.config b/nuget.config deleted file mode 100644 index 1b587ea8a..000000000 --- a/nuget.config +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - - - - - \ No newline at end of file