From 8e9250d6884236cfae3467b1e2691a2c531c9007 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 30 Apr 2025 16:11:04 +0200 Subject: [PATCH 1/2] Added new destination test method. This new method also returns information about the remote destination contents to assist in showing warnings in the UI. --- .../Dto/V2/DestinationTestRequestDto.cs | 41 ++++ .../Dto/V2/DestinationTestResponseDto.cs | 125 +++++++++++ .../Endpoints/V2/DestinationVerify.cs | 202 ++++++++++++++++++ .../Services/SystemInfoProvider.cs | 9 +- 4 files changed, 376 insertions(+), 1 deletion(-) create mode 100644 Duplicati/WebserverCore/Dto/V2/DestinationTestRequestDto.cs create mode 100644 Duplicati/WebserverCore/Dto/V2/DestinationTestResponseDto.cs create mode 100644 Duplicati/WebserverCore/Endpoints/V2/DestinationVerify.cs diff --git a/Duplicati/WebserverCore/Dto/V2/DestinationTestRequestDto.cs b/Duplicati/WebserverCore/Dto/V2/DestinationTestRequestDto.cs new file mode 100644 index 000000000..60fe93426 --- /dev/null +++ b/Duplicati/WebserverCore/Dto/V2/DestinationTestRequestDto.cs @@ -0,0 +1,41 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +namespace Duplicati.WebserverCore.Dto.V2; + +/// +/// The list filesets request DTO +/// +public sealed record DestinationTestRequestDto +{ + /// + /// The destination url + /// + public required string DestinationUrl { get; init; } + + /// + /// Any additional options to pass to the destination + /// + public required Dictionary Options { get; init; } = new(); + /// + /// Whether to automatically create the destination if it does not exist + /// + public required bool AutoCreate { get; init; } = false; +} \ No newline at end of file diff --git a/Duplicati/WebserverCore/Dto/V2/DestinationTestResponseDto.cs b/Duplicati/WebserverCore/Dto/V2/DestinationTestResponseDto.cs new file mode 100644 index 000000000..97c6a5035 --- /dev/null +++ b/Duplicati/WebserverCore/Dto/V2/DestinationTestResponseDto.cs @@ -0,0 +1,125 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +namespace Duplicati.WebserverCore.Dto.V2; + +/// +/// The list filesets request DTO +/// +public sealed record DestinationTestResponseDto : ResponseEnvelope +{ + /// + /// Creates a new instance of the DestinationTestResponseDto with the given error and status code + /// + /// The error message + /// The status code + /// Flag indicating if the folder exists + /// The host certificate, if any + /// The reported host key, if any + /// The currently accepted host key, if any + /// The DestinationTestResponseDto instance + public static DestinationTestResponseDto Failure( + string error, + string statusCode, + bool? folderExists = null, + string? hostCertificate = null, + string? reportedHostKey = null, + string? acceptedHostKey = null + ) => new DestinationTestResponseDto() + { + Success = false, + Error = error, + StatusCode = statusCode, + Data = hostCertificate == null && reportedHostKey == null && acceptedHostKey == null + ? null + : new DestinationTestResult() + { + FolderExists = folderExists, + FolderIsEmpty = null, + FolderContainsBackupFiles = null, + FolderContainsEncryptedBackupFiles = null, + HostCertificate = hostCertificate, + ReportedHostKey = reportedHostKey, + AcceptedHostKey = acceptedHostKey + } + }; + + /// + /// Creates a new instance of the DestinationTestResponseDto with the given error and status code + /// + /// Flag indicating if there are any files + /// Flag indicating if there are any backups + /// Flag indicating if there are any encrypted files + /// The DestinationTestResponseDto instance + public static DestinationTestResponseDto Create(bool anyFiles, bool anyBackups, bool anyEncryptedFiles) + { + return new DestinationTestResponseDto() + { + Success = true, + Error = null, + StatusCode = "OK", + Data = new DestinationTestResult() + { + FolderExists = true, + FolderIsEmpty = !anyFiles, + FolderContainsBackupFiles = anyBackups, + FolderContainsEncryptedBackupFiles = anyEncryptedFiles, + HostCertificate = null, + ReportedHostKey = null, + AcceptedHostKey = null + } + }; + } +} + +/// +/// The destination test result DTO +/// +public sealed record DestinationTestResult +{ + /// + /// Flag indicating if the folder exists + /// + public required bool? FolderExists { get; init; } + /// + /// Flag indicating if the folder is empty + /// + public required bool? FolderIsEmpty { get; init; } + /// + /// Flag indicating if the folder contains backup files + /// + public required bool? FolderContainsBackupFiles { get; init; } + /// + /// Flag indicating if the folder contains encrypted backup files + /// + public required bool? FolderContainsEncryptedBackupFiles { get; init; } + /// + /// The host certificate, if any + /// + public string? HostCertificate { get; init; } + /// + /// The reported host key, if any + /// + public required string? ReportedHostKey { get; init; } + /// + /// The currently accepted host key, if any + /// + public required string? AcceptedHostKey { get; init; } +} \ No newline at end of file diff --git a/Duplicati/WebserverCore/Endpoints/V2/DestinationVerify.cs b/Duplicati/WebserverCore/Endpoints/V2/DestinationVerify.cs new file mode 100644 index 000000000..6e63db05b --- /dev/null +++ b/Duplicati/WebserverCore/Endpoints/V2/DestinationVerify.cs @@ -0,0 +1,202 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using Duplicati.Library.Interface; +using Duplicati.Library.Main; +using Duplicati.Library.Main.Volumes; +using Duplicati.Library.RestAPI; +using Duplicati.Server; +using Duplicati.Server.Database; +using Duplicati.WebserverCore.Abstractions; +using Duplicati.WebserverCore.Dto.V2; +using Microsoft.AspNetCore.Mvc; + +namespace Duplicati.WebserverCore.Endpoints.V2.Backup; + +public class DestinationVerify : IEndpointV2 +{ + public static void Map(RouteGroupBuilder group) + { + group.MapPost("/destination/test", ([FromServices] Connection connection, [FromBody] Dto.V2.DestinationTestRequestDto input, CancellationToken cancelToken) + => ExecuteTest(input, cancelToken)) + .RequireAuthorization(); + } + + private record TupleDisposeWrapper(IBackend Backend, IEnumerable Modules) : IDisposable + { + public void Dispose() + { + Backend.Dispose(); + foreach (var n in Modules) + if (n is IDisposable disposable) + disposable.Dispose(); + } + } + + private static Dictionary ParseUrlOptions(Library.Utility.Uri uri) + { + var qp = uri.QueryParameters; + + var opts = Runner.GetCommonOptions(); + foreach (var k in qp.Keys.Cast()) + opts[k] = qp[k]; + + return opts; + } + + private static IEnumerable ConfigureModules(IDictionary opts) + { + // TODO: This works because the generic modules are implemented + // with pre .NetCore logic, using static methods + // The modules are created to allow multipe dispose, + // which violates the .Net patterns + var modules = Library.DynamicLoader.GenericLoader.Modules.OfType().ToArray(); + foreach (var n in modules) + n.Configure(opts); + + return modules; + } + + private static async Task GetBackend(string url, CancellationToken cancelToken) + { + var uri = new Library.Utility.Uri(url); + var opts = ParseUrlOptions(uri); + + var tmp = new[] { uri }; + await SecretProviderHelper.ApplySecretProviderAsync([], tmp, opts, Library.Utility.TempFolder.SystemTempPath, FIXMEGlobal.SecretProvider, cancelToken); + url = tmp[0].ToString(); + + var modules = ConfigureModules(opts); + var backend = Library.DynamicLoader.BackendLoader.GetBackend(url, new Dictionary()); + return new TupleDisposeWrapper(backend, modules); + } + + + private static async Task ExecuteTest(DestinationTestRequestDto input, CancellationToken cancelToken) + { + TupleDisposeWrapper? wrapper = null; + + try + { + wrapper = await GetBackend(input.DestinationUrl, cancelToken); + + using (var b = wrapper.Backend) + { + try { await b.TestAsync(cancelToken).ConfigureAwait(false); } + catch (FolderMissingException) + { + if (!input.AutoCreate) + throw; + + await b.CreateFolderAsync(cancelToken).ConfigureAwait(false); + await b.TestAsync(cancelToken).ConfigureAwait(false); + } + + var anyFiles = false; + var anyBackups = false; + var anyEncryptedFiles = false; + await foreach (var f in b.ListAsync(cancelToken).ConfigureAwait(false)) + { + if (f.IsFolder) + continue; + + anyFiles = true; + var parsed = VolumeBase.ParseFilename(f.Name); + if (parsed != null) + { + anyBackups = true; + anyEncryptedFiles = !string.IsNullOrWhiteSpace(parsed.EncryptionModule); + break; + } + } + + return DestinationTestResponseDto.Create( + anyFiles, + anyBackups, + anyEncryptedFiles + ); + + } + } + catch (FolderMissingException) + { + if (input.AutoCreate) + return DestinationTestResponseDto.Failure( + "Failed to create folder", + "error-creating-folder", + folderExists: false + ); + + return DestinationTestResponseDto.Failure( + "Folder does not exist", + "missing-folder", + folderExists: false + ); + } + catch (Library.Utility.SslCertificateValidator.InvalidCertificateException icex) + { + if (string.IsNullOrWhiteSpace(icex.Certificate)) + return DestinationTestResponseDto.Failure( + icex.Message, + "CertError" + ); + + return DestinationTestResponseDto.Failure( + "Host certificate error", + "incorrect-cert", + hostCertificate: icex.Certificate + ); + } + catch (Library.Utility.HostKeyException hex) + { + if (string.IsNullOrWhiteSpace(hex.ReportedHostKey)) + return DestinationTestResponseDto.Failure( + hex.Message, + "KeyError" + ); + + return DestinationTestResponseDto.Failure( + "Host key error", + "incorrect-host-key", + reportedHostKey: hex.ReportedHostKey, + acceptedHostKey: hex.AcceptedHostKey + ); + } + catch (UserInformationException uex) + { + return DestinationTestResponseDto.Failure( + uex.Message, + uex.HelpID + ); + } + catch (Exception ex) + { + return DestinationTestResponseDto.Failure( + ex.Message, + "error" + ); + } + finally + { + wrapper?.Dispose(); + } + } +} \ No newline at end of file diff --git a/Duplicati/WebserverCore/Services/SystemInfoProvider.cs b/Duplicati/WebserverCore/Services/SystemInfoProvider.cs index 4523c6e0e..034464f08 100644 --- a/Duplicati/WebserverCore/Services/SystemInfoProvider.cs +++ b/Duplicati/WebserverCore/Services/SystemInfoProvider.cs @@ -37,7 +37,14 @@ public class SystemInfoProvider : ISystemInfoProvider /// /// The API extensions that are available /// - private static readonly string[] APIExtensions = ["v2:backup:list-filesets", "v2:backup:list-folder", "v2:backup:list-versions", "v2:backup:search"]; + private static readonly string[] APIExtensions = [ + "v2:backup:list-filesets", + "v2:backup:list-folder", + "v2:backup:list-versions", + "v2:backup:search", + "v2:destination:test" + ]; + /// /// The API scopes that are available /// From ab6ef6c60b0dbbd4f546216daa7b68b49ed101f0 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 30 Apr 2025 16:11:46 +0200 Subject: [PATCH 2/2] Fixed minor issues with the listing method --- Duplicati/Library/RestAPI/Runner.cs | 4 +- .../Endpoints/V2/BackupListing.cs | 39 +++++++++++-------- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/Duplicati/Library/RestAPI/Runner.cs b/Duplicati/Library/RestAPI/Runner.cs index 22be4824b..c55fe657d 100644 --- a/Duplicati/Library/RestAPI/Runner.cs +++ b/Duplicati/Library/RestAPI/Runner.cs @@ -192,12 +192,12 @@ namespace Duplicati.Server filters); } - public static IRunnerData CreateListFilesetsTask(Duplicati.Server.Serialization.Interface.IBackup backup) + public static IRunnerData CreateListFilesetsTask(Duplicati.Server.Serialization.Interface.IBackup backup, Dictionary extraOptions = null) { return CreateTask( DuplicatiOperation.ListFilesets, backup, - new Dictionary()); + extraOptions ?? new Dictionary()); } public static IRunnerData CreateListFolderContents(Duplicati.Server.Serialization.Interface.IBackup backup, string[] folders, DateTime time, int pageSize, int pageOffset) diff --git a/Duplicati/WebserverCore/Endpoints/V2/BackupListing.cs b/Duplicati/WebserverCore/Endpoints/V2/BackupListing.cs index d6c566df2..d013f73d5 100644 --- a/Duplicati/WebserverCore/Endpoints/V2/BackupListing.cs +++ b/Duplicati/WebserverCore/Endpoints/V2/BackupListing.cs @@ -85,24 +85,31 @@ public class BackupListing : IEndpointV2 // Retries will hang the http request extra["number-of-retries"] = "0"; - var r = Runner.Run(Runner.CreateListFilesetsTask(bk), false) as IListFilesetResults; - if (r == null) - throw new ServerErrorException("No result from list operation"); + try + { + var r = Runner.Run(Runner.CreateListFilesetsTask(bk, extra), false) as IListFilesetResults; + if (r == null) + throw new ServerErrorException("No result from list operation"); - if (r.EncryptedFiles.HasValue && r.EncryptedFiles.Value && bk.Settings.Any(x => string.Equals("--no-encryption", x.Name, StringComparison.OrdinalIgnoreCase))) - throw new ServerErrorException("encrypted-storage"); + if (r.EncryptedFiles.HasValue && r.EncryptedFiles.Value && bk.Settings.Any(x => string.Equals("--no-encryption", x.Name, StringComparison.OrdinalIgnoreCase))) + throw new UserInformationException("The remote storage is encrypted, but no passphrase is given", "EncryptedStorageNoPassphrase"); - return Dto.V2.ListFilesetsResponseDto.Create( - r.Filesets - .Select(x => new Dto.V2.ListFilesetsResponseItem() - { - Version = x.Version, - Time = x.Time, - IsFullBackup = x.IsFullBackup, - FileCount = x.FileCount, - FileSizes = x.FileSizes - }) - ); + return Dto.V2.ListFilesetsResponseDto.Create( + r.Filesets + .Select(x => new Dto.V2.ListFilesetsResponseItem() + { + Version = x.Version, + Time = x.Time, + IsFullBackup = x.IsFullBackup, + FileCount = x.FileCount, + FileSizes = x.FileSizes + }) + ); + } + catch (FolderMissingException fex) + { + throw new UserInformationException("Remote folder is missing", "FolderMissing", fex); + } } private static Dto.V2.ListFileVersionsOutputDto ExecuteListVersions(IBackup bk, Dto.V2.ListFileVersionsRequestDto input)