Merge branch 'master' into performance/restoreopt

Duplicati/Agent/Program.cs

@@ -69,6 +69,7 @@ public static class Program
     /// <param name="WebserviceDisableSigninTokens">Disable signin tokens for the webserver</param>
     /// <param name="PingPongKeepalive">Enable ping-pong keepalive</param>
     /// <param name="DisablePreSharedKey">Disable the pre-shared key</param>
+    /// <param name="KeepWebservicePassword">Keep the webserver password</param>
     /// <param name="SecretProvider">The secret provider to use</param>
     /// <param name="SecretProviderCache">The secret provider cache level</param>
     /// <param name="SecretProviderPattern">The secret provider pattern</param>
@@ -89,6 +90,7 @@ public static class Program
         bool WebserviceDisableSigninTokens,
         bool PingPongKeepalive,
         bool DisablePreSharedKey,
+        bool KeepWebservicePassword,
         string? SecretProvider,
         SecretProviderHelper.CachingLevel SecretProviderCache,
         string SecretProviderPattern
@@ -107,7 +109,7 @@ public static class Program
             new Option<bool>("--agent-register-only", description: "Only register the agent, then exit", getDefaultValue: () => false),
             new Option<string>("--webservice-listen-interface", description: "The interface to listen on for the webserver", getDefaultValue: () => "loopback"),
             new Option<string>("--webservice-port", description: "The port to listen on for the webserver", getDefaultValue: () => "8210"),
-            new Option<string?>("--webservice-password", description: "The password for the webserver, if set to \"random\" a random passphrase is used", getDefaultValue: () => "random"),
+            new Option<string?>("--webservice-password", description: "The password for the webserver, not set, or set to \"random\", a random value is used.", getDefaultValue: () => null),
             new Option<string?>("--settings-encryption-key", description: "The encryption key for the database settings", getDefaultValue: () => null),
             new Option<string>("--windows-eventlog", description: "The Windows event log to write to", getDefaultValue: () => "Duplicati"),
             new Option<bool>("--disable-db-encryption", description: "Disable database encryption", getDefaultValue: () => false),
@@ -117,6 +119,7 @@ public static class Program
             new Option<bool>("--webservice-disable-signin-tokens", description: "Disable signin tokens for the webserver", getDefaultValue: () => true),
             new Option<bool>("--ping-pong-keepalive", description: "Enable ping-pong keepalive", getDefaultValue: () => false),
             new Option<bool>("--disable-pre-shared-key", description: "Disable the pre-shared key that prevents outside access to the webserver", getDefaultValue: () => false),
+            new Option<bool>("--keep-webservice-password", description: "Disables the random password assigned to the webserver on startup if no password is provided", getDefaultValue: () => false),
             new Option<string?>("--secret-provider", description: "The secret provider to use", getDefaultValue: () => null),
             new Option<SecretProviderHelper.CachingLevel>("--secret-provider-cache", description: "The secret provider cache level", getDefaultValue: () => SecretProviderHelper.CachingLevel.None),
             new Option<string>("--secret-provider-pattern", description: "The secret provider pattern", getDefaultValue: () => SecretProviderHelper.DEFAULT_PATTERN),
@@ -208,6 +211,7 @@ public static class Program
             WebserviceDisableSigninTokens: true,
             PingPongKeepalive: false,
             DisablePreSharedKey: false,
+            KeepWebservicePassword: false,
             SecretProvider: secretProvider,
             SecretProviderCache: SecretProviderHelper.CachingLevel.None,
             SecretProviderPattern: secretProviderPattern
@@ -252,9 +256,18 @@ public static class Program
                     prop.Value.SetValue(agentConfig, value);
         }
 
-        // Prevent access to the webserver interface from anything but the agent
-        if (string.Equals("random", agentConfig.WebservicePassword, StringComparison.OrdinalIgnoreCase))
-            agentConfig = agentConfig with { WebservicePassword = System.Security.Cryptography.RandomNumberGenerator.GetHexString(128) };
+        if (agentConfig.KeepWebservicePassword)
+        {
+            // Check for conflicting options, require no password
+            if (!string.IsNullOrWhiteSpace(agentConfig.WebservicePassword))
+                throw new UserInformationException("Cannot use --keep-webservice-password with a provided password", "KeepWebservicePasswordWithPassword");
+        }
+        else
+        {
+            // Prevent access to the webserver interface from anything but the agent
+            if (string.Equals("random", agentConfig.WebservicePassword, StringComparison.OrdinalIgnoreCase) || string.IsNullOrWhiteSpace(agentConfig.WebservicePassword))
+                agentConfig = agentConfig with { WebservicePassword = System.Security.Cryptography.RandomNumberGenerator.GetHexString(128) };
+        }
 
         // Set the pre-shared key for the agent
         if (!agentConfig.DisablePreSharedKey)
@@ -454,19 +467,28 @@ public static class Program
         else if (!string.IsNullOrWhiteSpace(settings.SettingsEncryptionKey))
             settingsEncryptionKey = settings.SettingsEncryptionKey;
 
+        // Helper method to support empty arguments
+        static string EncodeOption(string option, string? value)
+        {
+            if (string.IsNullOrWhiteSpace(value))
+                return "";
+
+            return $"{option}={value}";
+        }
+
         // Lock down the instance, reset tokens and password,
         // and forward relevant settings from agent to the webserver
         var args = new[] {
-                $"--webservice-listen-interface={agentConfig.WebserviceListenInterface}",
-                $"--webservice-password={agentConfig.WebservicePassword}",
-                OperatingSystem.IsWindows() ? $"--windows-eventlog={agentConfig.WindowsEventLog}" : "",
-                $"--webservice-port={agentConfig.WebservicePort}",
-                $"--webservice-reset-jwt-config={agentConfig.WebserviceResetJwtConfig}",
-                $"--webservice-allowed-hostnames={agentConfig.WebserviceAllowedHostnames}",
-                $"--webservice-api-only={agentConfig.WebserviceApiOnly}",
-                $"--webservice-disable-signin-tokens={agentConfig.WebserviceDisableSigninTokens}",
-                $"--disable-db-encryption={agentConfig.DisableDbEncryption}",
-                string.IsNullOrWhiteSpace(settingsEncryptionKey) ? "" : $"--settings-encryption-key={settingsEncryptionKey}",
+            EncodeOption("--webservice-listen-interface", agentConfig.WebserviceListenInterface),
+            EncodeOption("--webservice-password", agentConfig.WebservicePassword),
+            EncodeOption("--windows-eventlog", OperatingSystem.IsWindows() ? agentConfig.WindowsEventLog : ""),
+            EncodeOption("--webservice-port", agentConfig.WebservicePort),
+            EncodeOption("--webservice-reset-jwt-config", agentConfig.WebserviceResetJwtConfig.ToString()),
+            EncodeOption("--webservice-allowed-hostnames", agentConfig.WebserviceAllowedHostnames),
+            EncodeOption("--webservice-api-only", agentConfig.WebserviceApiOnly.ToString()),
+            EncodeOption("--webservice-disable-signin-tokens", agentConfig.WebserviceDisableSigninTokens.ToString()),
+            EncodeOption("--disable-db-encryption", agentConfig.DisableDbEncryption.ToString()),
+            EncodeOption("--settings-encryption-key", settingsEncryptionKey),
             }
             .Where(x => !string.IsNullOrWhiteSpace(x))
             .ToArray();

Duplicati/Library/Backend/SSHv2/Duplicati.Library.Backend.SSHv2.csproj

@@ -8,8 +8,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="SSH.NET" Version="2023.0.1" />
-    <PackageReference Include="System.Security.Cryptography.Cng" Version="5.0.0" />
+    <PackageReference Include="SSH.NET" Version="2024.2.0" />
   </ItemGroup>
   
   <ItemGroup>

Duplicati/Library/Backend/pCloud/Model/pCloudFolderContent.cs

@@ -85,7 +85,7 @@ internal record pCloudFolderContent
     /// <summary>
     /// Folder ID if the item is a folder (null otherwise)
     /// </summary>
-    public long? folderid { get; init; }
+    public ulong folderid { get; init; }
 
     /// <summary>
     /// File ID if the item is a file (null otherwise)

Duplicati/Library/Backend/pCloud/pCloudBackend.cs

@@ -93,8 +93,19 @@ public class pCloudBackend : IStreamingBackend
     /// </summary>
     private ulong? _CachedFolderID;
 
+    /// <summary>
+    /// Name of the authentication parameter/option
+    /// </summary>
     private const string AUTHENTICATION_OPTION = "authid";
 
+    /// <summary>
+    /// Path separators (both Windows \ and unix /) to be used in path manipulation
+    /// </summary>
+    private static readonly char[] PATH_SEPARATORS = ['/', '\\'];
+
+    /// <summary>
+    /// List of pcloud Servers and their respective hostnames
+    /// </summary>
     private static readonly Dictionary<string, string> PCLOUD_SERVERS = new(StringComparer.OrdinalIgnoreCase)
     {
         { "pCloud Global", "api.pcloud.com" },
@@ -129,7 +140,8 @@ public class pCloudBackend : IStreamingBackend
         if (string.IsNullOrWhiteSpace(uri.Host))
             throw new UserInformationException(Strings.pCloudBackend.NoServerSpecified, "NopCloudServerSpecified");
 
-        _Path = uri.Path;
+        // Ensure that the path is in the correct format, without starting or tailing slashes
+        _Path = uri.Path.TrimStart(PATH_SEPARATORS).TrimEnd(PATH_SEPARATORS).Trim();
         _ServerUrl = uri.Host;
     }
 
@@ -154,31 +166,47 @@ public class pCloudBackend : IStreamingBackend
     /// Implementation of interface method for listing remote folder contents.
     /// The root parameter is used to list the root folder, as the pCloud API
     /// when using oauth tokens creates an isolated folder Applications/ApplicationName
-    ///
-    /// When root = true, this is the directory it will be listed.
-    /// When root = false, it will list the _Path directory if it exists, which is
-    /// where the Duplicati backup files are stored.
+    /// 
     /// </summary>
-    /// <param name="root">True to list absolute root, false to list _Path directory</param>
+    /// <param name="folderId">The folder ID to consider as root</param>
+    /// <param name="cancellationToken">Cancellation Token</param>
     /// <returns></returns>
-    private IEnumerable<IFileEntry> List(bool root)
+    private async Task<IEnumerable<IFileEntry>> List(ulong folderId, CancellationToken cancellationToken)
     {
-        if (_CachedFolderID == null && string.IsNullOrWhiteSpace(_Path) == false && root == false)
-            CreateFolderAsync(CancellationToken.None).Await();
+        var result = await ListWithMetadata(folderId, cancellationToken).ConfigureAwait(false);
+        return result.Select(item => new pCloudFileEntry
+        {
+            IsFolder = item.isfolder,
+            Name = item.name,
+            Size = item.size ?? 0,
+            LastAccess = DateTime.Parse(item.created),
+            LastModification = DateTime.Parse(item.modified)
+        })
+            .ToList();
+    }
 
-        ulong folderId = root ? 0 : _CachedFolderID ?? 0;
-
-        var timeoutToken = new CancellationTokenSource();
+    /// <summary>
+    /// Lists folders with pCloud metadata, necessary to obtain the folder IDs
+    /// </summary>
+    /// <param name="folderId">FolderId to be used as root.</param>
+    /// <param name="cancellationToken">Cancellation Token</param>
+    /// <returns></returns>
+    /// <exception cref="Exception"></exception>
+    private async Task<List<pCloudFolderContent>> ListWithMetadata(ulong folderId, CancellationToken cancellationToken)
+    {
+        using var timeoutToken = new CancellationTokenSource();
         timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS));
+        using var combinedTokens =
+            CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken);
 
         using var requestResources = CreateRequest($"/listfolder?folderid={folderId}", HttpMethod.Get);
 
-        using var response = requestResources.HttpClient
+        using var response = await requestResources.HttpClient
             .SendAsync(requestResources.RequestMessage, HttpCompletionOption.ResponseContentRead,
-                timeoutToken.Token).Await();
+                timeoutToken.Token).ConfigureAwait(false);
 
         response.EnsureSuccessStatusCode();
-        var content = response.Content.ReadAsStringAsync(timeoutToken.Token).Result;
+        var content = await response.Content.ReadAsStringAsync(timeoutToken.Token).ConfigureAwait(false);
         var listFolderResponse = JsonSerializer.Deserialize<pCloudListFolderResponse>(content);
 
         if (pCloudErrorList.ErrorMessages.TryGetValue(listFolderResponse.result, out var message))
@@ -187,30 +215,26 @@ public class pCloudBackend : IStreamingBackend
         if (listFolderResponse.result != 0)
             throw new Exception(Strings.pCloudBackend.FailedWithUnexpectedErrorCode("list", listFolderResponse.result));
 
-        if (listFolderResponse?.metadata?.contents == null)
-            return new List<IFileEntry>();
-
-        return listFolderResponse.metadata.contents
-            .Select<pCloudFolderContent, IFileEntry>(item => new pCloudFileEntry
-            {
-                IsFolder = item.isfolder,
-                Name = item.name,
-                Size = item.size ?? 0,
-                LastAccess = DateTime.Parse(item.created),
-                LastModification = DateTime.Parse(item.modified)
-            })
-            .ToList();
+        return listFolderResponse.metadata?.contents ?? [];
     }
 
     /// <summary>
     /// Implementation of interface method for listing remote folder contents
     /// </summary>
     /// <returns>List of IFileEntry with directory listing result</returns>
-    public IEnumerable<IFileEntry> List()
+    public async Task<IEnumerable<IFileEntry>> ListAsync(CancellationToken cancellationToken)
     {
-        return List(false);
+        _CachedFolderID ??= await GetFolderId(cancellationToken).ConfigureAwait(false);
+
+        return await List(_CachedFolderID.Value, cancellationToken).ConfigureAwait(false);
     }
 
+    /// <summary>
+    /// Wrapper method of legacy non async call to list files in the remote folder
+    /// </summary>
+    /// <returns></returns>
+    public IEnumerable<IFileEntry> List() => ListAsync(CancellationToken.None).Await();
+
     /// <summary>
     /// Upload files to remote location
     /// </summary>
@@ -236,9 +260,7 @@ public class pCloudBackend : IStreamingBackend
     /// <exception cref="Exception">Exceptions arising from either code execution</exception>
     public async Task PutAsync(string remotename, Stream input, CancellationToken cancellationToken)
     {
-        // If we don't have the folder ID cached, lets fetch it with the create folder api
-        if (_CachedFolderID == null)
-            await CreateFolderAsync(cancellationToken).ConfigureAwait(false);
+        _CachedFolderID ??= await GetFolderId(cancellationToken).ConfigureAwait(false);
 
         using var timeoutToken = new CancellationTokenSource();
         timeoutToken.CancelAfter(TimeSpan.FromSeconds(LONG_OPERATION_TIMEOUT_SECONDS));
@@ -250,6 +272,7 @@ public class pCloudBackend : IStreamingBackend
         using var requestResources =
             CreateRequest($"/uploadfile?folderid={_CachedFolderID}&filename={encodedPath}&nopartial=1",
                 HttpMethod.Post);
+
         requestResources.RequestMessage.Content = new StreamContent(input);
         requestResources.RequestMessage.Content.Headers.ContentLength = input.Length;
         requestResources.RequestMessage.Content.Headers.ContentType =
@@ -276,7 +299,7 @@ public class pCloudBackend : IStreamingBackend
     /// <param name="output">Destination stream to write to</param>
     /// <param name="cancellationToken">CancellationToken that is combined with internal timeout token</param>
     /// <exception cref="FileMissingException">FileMissingException when file is not found</exception>
-    /// <exception cref="Exception">Exceptions arising from either code execution</exception>
+    /// <exception cref="Exception">Exceptions arising from either code execution or FileMissingException</exception>
     public async Task GetAsync(string remotename, string localname, CancellationToken cancellationToken)
     {
         await using var fs = File.Open(localname,
@@ -290,26 +313,27 @@ public class pCloudBackend : IStreamingBackend
     /// 
     /// </summary>
     /// <param name="filename">Filename at remote, path is automatically concatenated if needed</param>
+    /// <param name="cancellationToken">Cancellation Token</param>
     /// <returns></returns>
     /// <exception cref="FileMissingException">FileMissingException when file is not found</exception>
     /// <exception cref="Exception">Exceptions arising from either code execution</exception>
-    private async Task<string> GetFileId(string filename)
+    private async Task<string> GetFileLink(string filename, CancellationToken cancellationToken)
     {
-        var path = string.IsNullOrWhiteSpace(_Path) ? "" : $"/{_Path}";
-        var encodedPath = Uri.EscapeDataString($"{path}/{filename}");
-
-        using var requestResources = CreateRequest($"/getfilelink?path={encodedPath}", HttpMethod.Get);
+        using var timeoutToken = new CancellationTokenSource();
+        timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS));
+        using var combinedTokens =
+            CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken);
+        
+        using var requestResources = CreateRequest($"/getfilelink?fileid={await GetFileId(filename, cancellationToken).ConfigureAwait(false)}", HttpMethod.Get);
 
         using var response = await requestResources.HttpClient.SendAsync(
             requestResources.RequestMessage,
-            HttpCompletionOption.ResponseContentRead).ConfigureAwait(false);
+            HttpCompletionOption.ResponseContentRead, combinedTokens.Token).ConfigureAwait(false);
 
         if (!response.IsSuccessStatusCode)
-        {
             throw new Exception($"Failed to get download link. Status: {response.StatusCode}");
-        }
 
-        var content = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
+        var content = await response.Content.ReadAsStringAsync(combinedTokens.Token).ConfigureAwait(false);
         var getFileIdResponse = JsonSerializer.Deserialize<pCloudDownloadResponse>(content);
 
         if (pCloudErrorList.ErrorMessages.TryGetValue(getFileIdResponse.result, out var message))
@@ -317,7 +341,7 @@ public class pCloudBackend : IStreamingBackend
 
         if (getFileIdResponse.result != 0)
             throw new Exception(
-                Strings.pCloudBackend.FailedWithUnexpectedErrorCode("getfileid", getFileIdResponse.result));
+                Strings.pCloudBackend.FailedWithUnexpectedErrorCode("getfilelink", getFileIdResponse.result));
 
         return $"https://{getFileIdResponse.hosts[0]}{getFileIdResponse.path}";
     }
@@ -329,7 +353,7 @@ public class pCloudBackend : IStreamingBackend
     /// <param name="output">Destination stream to write to</param>
     /// <param name="cancellationToken">CancellationToken that is combined with internal timeout token</param>
     /// <exception cref="FileMissingException">FileMissingException when file is not found</exception>
-    /// <exception cref="Exception">Exceptions arising from either code execution</exception>
+    /// <exception cref="Exception">Exceptions arising from either code execution or FileMissingException</exception>
     public async Task GetAsync(string remotename, Stream output, CancellationToken cancellationToken)
     {
         try
@@ -341,10 +365,10 @@ public class pCloudBackend : IStreamingBackend
 
             using var requestResources = CreateRequest(string.Empty, HttpMethod.Get);
 
-            requestResources.RequestMessage.RequestUri = new Uri(await GetFileId(remotename));
+            requestResources.RequestMessage.RequestUri = new Uri(await GetFileLink(remotename, cancellationToken).ConfigureAwait(false));
 
             await requestResources.HttpClient.DownloadFile(requestResources.RequestMessage, output, null,
-                timeoutToken.Token).ConfigureAwait(false);
+                combinedTokens.Token).ConfigureAwait(false);
         }
         catch (HttpRequestException wex)
         {
@@ -375,25 +399,22 @@ public class pCloudBackend : IStreamingBackend
     /// <returns></returns>
     /// <exception cref="FileMissingException">FileMissingException when file is not found</exception>
     /// <exception cref="Exception">Exceptions arising from either code execution or business logic when return code from pcloud indicates an error.</exception>
-    public Task DeleteAsync(string remotename, CancellationToken cancellationToken)
+    public async Task DeleteAsync(string remotename, CancellationToken cancellationToken)
     {
-        if (!String.IsNullOrWhiteSpace(_Path))
-            remotename = $"/{_Path}/{remotename}";
-
         using var timeoutToken = new CancellationTokenSource();
         timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS));
         using var combinedTokens =
             CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken);
 
-        using var requestResources = CreateRequest($"/deletefile?path={remotename}", HttpMethod.Get);
+        using var requestResources = CreateRequest($"/deletefile?fileid={await GetFileId(remotename, cancellationToken).ConfigureAwait(false)}", HttpMethod.Get);
 
-        using var response = requestResources.HttpClient.SendAsync(
+        using var response = await requestResources.HttpClient.SendAsync(
             requestResources.RequestMessage,
-            HttpCompletionOption.ResponseContentRead, combinedTokens.Token).Await();
+            HttpCompletionOption.ResponseContentRead, combinedTokens.Token).ConfigureAwait(false);
 
         response.EnsureSuccessStatusCode();
 
-        var content = response.Content.ReadAsStringAsync(combinedTokens.Token).Await();
+        var content = await response.Content.ReadAsStringAsync(combinedTokens.Token).ConfigureAwait(false);
         var deleteFileResponse = JsonSerializer.Deserialize<pCloudDeleteResponse>(content);
 
         // If no error code is matched, result was == 0 so it successfully created the folder
@@ -407,7 +428,6 @@ public class pCloudBackend : IStreamingBackend
             throw new Exception(
                 Strings.pCloudBackend.FailedWithUnexpectedErrorCode("delete", deleteFileResponse.result));
 
-        return Task.CompletedTask;
     }
 
     /// <summary>
@@ -418,19 +438,18 @@ public class pCloudBackend : IStreamingBackend
     public Task<string[]> GetDNSNamesAsync(CancellationToken cancellationToken) => Task.FromResult(new[] { _DnsName });
 
     /// <summary>
-    /// Performs basic backend conectivity tests, in this case, checks
-    /// that the configured path, if configured, exists
+    /// Tests backend connectivity by verifying the configured path exists
     /// </summary>
-    /// <param name="cancellationToken">CancellationToken, in this call not used.</param>
-    /// <returns></returns>
+    /// <param name="cancellationToken">The cancellation token (not used)</param>
     /// <exception cref="FolderMissingException">Thrown when configured path does not exist</exception>
-    public Task TestAsync(CancellationToken cancellationToken)
+    public async Task TestAsync(CancellationToken cancellationToken)
     {
-        // List the scoped root directory and check if _Path folder exists
-        if (!List(true).Any(x => x.IsFolder && x.Name == _Path) && !string.IsNullOrWhiteSpace(_Path))
-            throw new FolderMissingException();
+        if (string.IsNullOrWhiteSpace(_Path) || _Path.Split(PATH_SEPARATORS, StringSplitOptions.RemoveEmptyEntries).Length == 0)
+            return;
+
+        // This method will search for the folderId recursively and throw an exception if the folder is not found
+        await GetFolderId(cancellationToken).ConfigureAwait(false);
 
-        return Task.CompletedTask;
     }
 
     /// <summary>
@@ -439,40 +458,96 @@ public class pCloudBackend : IStreamingBackend
     /// <param name="cancellationToken">CancellationToken that will be combined with internal timeout token</param>
     /// <returns></returns>
     /// <exception cref="Exception"></exception>
-    public Task CreateFolderAsync(CancellationToken cancellationToken)
+    public async Task CreateFolderAsync(CancellationToken cancellationToken)
+    {
+        if (string.IsNullOrWhiteSpace(_Path) || _Path.Split(PATH_SEPARATORS, StringSplitOptions.RemoveEmptyEntries).Length == 0)
+        {
+            _CachedFolderID = 0;
+            return;
+        }
+
+        var currentId = 0UL;
+        foreach (var folder in _Path.Split(PATH_SEPARATORS, StringSplitOptions.RemoveEmptyEntries))
+            currentId = await CreateFolder(cancellationToken, currentId, folder).ConfigureAwait(false);
+
+        _CachedFolderID = currentId;
+    }
+
+    /// <summary>
+    /// Create remote folder in relation to the parent folder
+    /// </summary>
+    /// <param name="cancellationToken">Cancellation Token</param>
+    /// <param name="parentFolderId">Parent Folder ID</param>
+    /// <param name="folderName">Folder name</param>
+    /// <returns>The folderID of the newly created folder</returns>
+    private async Task<ulong> CreateFolder(CancellationToken cancellationToken, ulong parentFolderId, string folderName)
     {
         using var timeoutToken = new CancellationTokenSource();
         timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS));
         using var combinedTokens =
             CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken);
 
-        using var requestResources = CreateRequest($"/createfolderifnotexists?folderid=0&name={_Path}", HttpMethod.Get);
+        using var requestResources = CreateRequest($"/createfolderifnotexists?folderid={parentFolderId}&name={folderName}", HttpMethod.Get);
 
-        using var response = requestResources.HttpClient.SendAsync(
+        using var response = await requestResources.HttpClient.SendAsync(
             requestResources.RequestMessage,
-            HttpCompletionOption.ResponseContentRead, combinedTokens.Token).Await();
+            HttpCompletionOption.ResponseContentRead, combinedTokens.Token).ConfigureAwait(false);
 
         response.EnsureSuccessStatusCode();
 
-        var content = response.Content.ReadAsStringAsync(combinedTokens.Token).Await();
+        var content = await response.Content.ReadAsStringAsync(combinedTokens.Token).ConfigureAwait(false);
         var createFolderResponse = JsonSerializer.Deserialize<pCloudCreateFolderResponse>(content);
 
         if (pCloudErrorList.ErrorMessages.TryGetValue(createFolderResponse.result, out var message))
             throw new Exception(message);
-        /*
-         * If no error code is matched, result was == 0 so it successfully created the folder
-         * We must cache the folderID, as that will be used to upload files, reason being that
-         * for oauth clients, pCloud creates a scoped folder Applications/ApplicationName
-         * and the folders created with /createfolderifnotexists api are created under that folder.
-         *
-         * However, when uploading files, the folderid is required (and surprisingly if not provided
-         * it will create the file outside the scoped folder, at the root of the drive)
-         */
-        if (createFolderResponse.result == 0) _CachedFolderID = createFolderResponse.metadata?.folderid;
 
-        return Task.CompletedTask;
+        return createFolderResponse is { result: 0, metadata.folderid: var id }
+            ? id
+            : throw new Exception(Strings.pCloudBackend.FailedWithUnexpectedErrorCode("createfolder", createFolderResponse.result));
     }
 
+    /// <summary>
+    /// Returns the fileID by listing the folder and searching for the filename & metadata.
+    /// 
+    /// For operations such as delete/getfilelink using the fileID is more reliable than using direct path/filename concatenation.
+    /// </summary>
+    /// <param name="name">The filename</param>
+    /// <param name="cancellationToken">Cancellation Token</param>
+    /// <returns></returns>
+    /// <exception cref="FileNotFoundException"></exception>
+    private async Task<ulong> GetFileId(string name, CancellationToken cancellationToken)
+    {
+        _CachedFolderID ??= await GetFolderId(cancellationToken).ConfigureAwait(false);
+
+        var result = await ListWithMetadata((ulong)_CachedFolderID, cancellationToken).ConfigureAwait(false);
+
+        return result.FirstOrDefault(x => !x.isfolder && x.name == name)?.fileid ?? throw new FileNotFoundException(name);
+
+    }
+
+    /// <summary>
+    /// Returns the folder ID for the configured path, regardless of the depth
+    /// </summary>
+    /// <exception cref="FolderMissingException"></exception>
+    private async Task<ulong> GetFolderId(CancellationToken cancellationToken)
+    {
+        if (string.IsNullOrWhiteSpace(_Path) || _Path.Split(PATH_SEPARATORS, StringSplitOptions.RemoveEmptyEntries).Length == 0)
+            return 0UL;
+
+        var currentFolderId = 0UL;
+        foreach (var folder in _Path.Split(PATH_SEPARATORS, StringSplitOptions.RemoveEmptyEntries))
+        {
+            var folderContent = await ListWithMetadata(currentFolderId, cancellationToken).ConfigureAwait(false);
+            var matchingFolder = folderContent.FirstOrDefault(x => x.isfolder && x.name == folder);
+
+            if (matchingFolder?.folderid == null)
+                throw new FolderMissingException();
+
+            currentFolderId = matchingFolder.folderid;
+        }
+
+        return currentFolderId;
+    }
 
     /// <summary>
     /// Wrapper for the tupple of HttpClient and HttpRequestMessage used in web requests.

Duplicati/Library/Main/Operation/Backup/CountFilesHandler.cs

@@ -1,22 +1,22 @@
 // Copyright (C) 2024, The Duplicati Team
 // https://duplicati.com, hello@duplicati.com
-// 
-// Permission is hereby granted, free of charge, to any person obtaining a 
-// copy of this software and associated documentation files (the "Software"), 
-// to deal in the Software without restriction, including without limitation 
-// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
-// and/or sell copies of the Software, and to permit persons to whom the 
+//
+// Permission is hereby granted, free of charge, to any person obtaining a
+// copy of this software and associated documentation files (the "Software"),
+// to deal in the Software without restriction, including without limitation
+// the rights to use, copy, modify, merge, publish, distribute, sublicense,
+// and/or sell copies of the Software, and to permit persons to whom the
 // Software is furnished to do so, subject to the following conditions:
-// 
-// The above copyright notice and this permission notice shall be included in 
+//
+// The above copyright notice and this permission notice shall be included in
 // all copies or substantial portions of the Software.
-// 
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 // DEALINGS IN THE SOFTWARE.
 
 using System.Collections.Generic;
@@ -24,6 +24,7 @@ using System.Threading.Tasks;
 using Duplicati.Library.Utility;
 using Duplicati.Library.Snapshots;
 using CoCoL;
+using Duplicati.Library.Common.IO;
 
 namespace Duplicati.Library.Main.Operation.Backup
 {
@@ -57,7 +58,8 @@ namespace Duplicati.Library.Main.Operation.Backup
 
                             try
                             {
-                                size += snapshot.GetFileSize(path);
+                                if (SystemIO.IO_OS.FileExists(path))
+                                    size += snapshot.GetFileSize(path);
                             }
                             catch
                             {

Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs

@@ -22,7 +22,6 @@
 using CoCoL;
 using Duplicati.Library.Main.Operation.Common;
 using Duplicati.Library.Main.Volumes;
-using Duplicati.Library.Utility;
 using System;
 using System.Threading.Tasks;
 using static Duplicati.Library.Main.Operation.Common.BackendHandler;
@@ -106,8 +105,8 @@ namespace Duplicati.Library.Main.Operation.Backup
                         // even if they are already added as non-blocklist blocks, but filter out duplicates
                         if (indexvolume != null && isMandatoryBlocklistHash)
                         {
-                            // This can cause a race between workers, 
-                            // but the side-effect is that the index files are slightly larger                        
+                            // This can cause a race between workers,
+                            // but the side-effect is that the index files are slightly larger
                             if (newBlock || !await database.IsBlocklistHashKnownAsync(b.HashKey))
                             {
                                 blocklistHashesAdded++;
@@ -163,9 +162,16 @@ namespace Duplicati.Library.Main.Operation.Backup
                     if (ex.IsRetiredException())
                     {
                         // If we have collected data, merge all pending volumes into a single volume
-                        if (blockvolume != null && (blockvolume.SourceSize > 0 || blocklistHashesAdded > 0))
+                        if (blockvolume != null)
                         {
-                            await self.SpillPickup.WriteAsync(new SpillVolumeRequest(blockvolume, indexvolume));
+                            if (blockvolume.SourceSize > 0 || blocklistHashesAdded > 0)
+                            {
+                                await self.SpillPickup.WriteAsync(new SpillVolumeRequest(blockvolume, indexvolume));
+                            }
+                            else
+                            {
+                                await database.RemoveRemoteVolumeAsync(blockvolume.RemoteFilename);
+                            }
                         }
                     }
 

Duplicati/Library/Modules/Builtin/SendHttpMessage.cs

@@ -26,6 +26,8 @@ using System.Net.Http;
 using System.Net.Http.Headers;
 using System.Text;
 using System.Threading.Tasks;
+using Duplicati.Library.Utility;
+using Uri = System.Uri;
 
 namespace Duplicati.Library.Modules.Builtin
 {
@@ -101,7 +103,16 @@ namespace Duplicati.Library.Modules.Builtin
         /// Option used to set the maximum number of log lines
         /// </summary>
         private const string OPTION_MAX_LOG_LINES = "send-http-max-log-lines";
-
+        
+        /// <summary>
+        /// The option used to accept a specific SSL certificate hash
+        /// </summary>
+        private const string OPTION_ACCEPT_SPECIFIED_CERTIFICATE = "send-http-accept-specified-ssl-hash"; 
+        /// <summary>
+        /// The option used to accept any SSL certificate
+        /// </summary>
+        private const string OPTION_ACCEPT_ANY_CERTIFICATE = "send-http-accept-any-ssl-certificate"; 
+        
         #endregion
 
         #region Option defaults
@@ -132,6 +143,16 @@ namespace Duplicati.Library.Modules.Builtin
         /// The message parameter name
         /// </summary>
         private string m_extraParameters;
+        
+        /// <summary>
+        /// Option to accept any SSL certificate
+        /// </summary>
+        private bool m_acceptAnyCertificate;
+        
+        /// <summary>
+        /// Specific hashes to be accepted by the certificate validator
+        /// </summary>
+        private string[] m_acceptSpecificCertificates;
 
         #endregion
 
@@ -184,6 +205,9 @@ namespace Duplicati.Library.Modules.Builtin
 
                     new CommandLineArgument(OPTION_URL_FORM, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.SendhttpurlsformShort, Strings.SendHttpMessage.SendhttpurlsformLong),
                     new CommandLineArgument(OPTION_URL_JSON, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.SendhttpurlsjsonShort, Strings.SendHttpMessage.SendhttpurlsjsonLong),
+                    
+                    new CommandLineArgument(OPTION_ACCEPT_ANY_CERTIFICATE, CommandLineArgument.ArgumentType.Boolean, Strings.SendHttpMessage.AcceptAnyCertificateShort, Strings.SendHttpMessage.AcceptAnyCertificateLong),
+                    new CommandLineArgument(OPTION_ACCEPT_SPECIFIED_CERTIFICATE, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.AcceptSpecifiedCertificateShort, Strings.SendHttpMessage.AcceptSpecifiedCertificateLong),
                 });
             }
         }
@@ -242,6 +266,9 @@ namespace Duplicati.Library.Modules.Builtin
                 m_messageParameterName = DEFAULT_MESSAGE_PARAMETER_NAME;
 
             commandlineOptions.TryGetValue(OPTION_EXTRA_PARAMETERS, out m_extraParameters);
+            m_acceptAnyCertificate = commandlineOptions.ContainsKey(OPTION_ACCEPT_ANY_CERTIFICATE) && Utility.Utility.ParseBoolOption(commandlineOptions.AsReadOnly(), OPTION_ACCEPT_ANY_CERTIFICATE);
+            m_acceptSpecificCertificates = commandlineOptions.ContainsKey(OPTION_ACCEPT_SPECIFIED_CERTIFICATE) ? commandlineOptions[OPTION_ACCEPT_SPECIFIED_CERTIFICATE].Split([",", ";"], StringSplitOptions.RemoveEmptyEntries): null;
+
             return true;
         }
 
@@ -327,7 +354,10 @@ namespace Duplicati.Library.Modules.Builtin
             if (m_report_targets == null || m_cachedBodyResults == null)
                 return;
 
-            using var client = new HttpClient();
+            using HttpClientHandler httpHandler = new HttpClientHandler();
+            HttpClientHelper.ConfigureHandlerCertificateValidator(httpHandler, m_acceptAnyCertificate, m_acceptSpecificCertificates);
+
+            using var client = new HttpClient(httpHandler);
 
             Exception ex = null;
 

Duplicati/Library/Modules/Builtin/Strings.cs

@@ -225,6 +225,10 @@ You can supply multiple options with a comma separator, e.g. ""{0},{1}"". The sp
         public static string SendhttpurlsformShort { get { return LC.L(@"HTTP report URLs for sending form data"); } }
         public static string SendhttpurlsjsonLong { get { return LC.L(@"Use this option to set HTTP report URLs for sending JSON data. This option accepts multiple URLs, seperated by a semi-colon. All URLs will receive the same data. Note that this option ignores the format and verb settings."); } }
         public static string SendhttpurlsjsonShort { get { return LC.L(@"HTTP report URLs for sending JSON data"); } }
+        public static string AcceptAnyCertificateLong { get { return LC.L(@"Use this option to accept any server certificate, regardless of what errors it may have. Please use --{0} instead, whenever possible.", "accept-specified-ssl-hash"); } }
+        public static string AcceptAnyCertificateShort { get { return LC.L(@"Accept any server certificate"); } }
+        public static string AcceptSpecifiedCertificateLong { get { return LC.L(@"If your server certificate is reported as invalid (e.g. with self-signed certificates), you can supply the certificate hash (SHA1) to approve it anyway. The hash value must be entered in hex format without spaces or colons. You can enter multiple hashes separated by commas."); } }
+        public static string AcceptSpecifiedCertificateShort { get { return LC.L(@"Optionally accept a known SSL certificate"); } }
     }
 
     internal static class ReportHelper

nuget.config

@@ -1,12 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<configuration>
-    <config>
-        <add key="repositoryPath" value="packages" />
-    </config>
-    <packageSources>
-        <!--
-            This should only be temporary until the freebsd libs are uploaded to nuget.org
-        -->
-        <add key="Freebsd native libs testing repo" value="https://staticwell.com/nuget/index.json" />
-    </packageSources>
-</configuration>